{ "schema_version": "1", "generated_at": "2026-07-17T09:10:39.557213+00:00", "vllm_version": "0.25.1", "vllm_recipes_sha": "5d1bf23", "accelerator_brands": { "h100": "nvidia", "h200": "nvidia", "b200": "nvidia", "gb200": "nvidia", "b300": "nvidia", "gb300": "nvidia", "dgx_station_gb300": "nvidia", "dgx_spark_gb10": "nvidia", "mi300x": "amd", "mi325x": "amd", "mi355x": "amd", "trillium": "google", "ironwood": "google", "xeon6": "intel", "xeon5": "intel", "rtxpro-6000-blackwell-server-edition": "nvidia" }, "index": { "h100": { "google/diffusiongemma-26B-A4B-it": [ { "name": "DiffusionGemma 26B-A4B IT", "huggingface_id": "google/diffusiongemma-26B-A4B-it", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic": [ { "name": "DiffusionGemma 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4": [ { "name": "DiffusionGemma 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-12B-it": [ { "name": "Gemma 4 12B IT", "huggingface_id": "google/gemma-4-12B-it", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "google/gemma-4-12B-it-qat-w4a16-ct": [ { "name": "Gemma 4 12B IT (w4a16)", "huggingface_id": "google/gemma-4-12B-it-qat-w4a16-ct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "LiquidAI/LFM2.5-1.2B-Base": [ { "name": "LFM2.5 1.2B Base", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Base", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Instruct": [ { "name": "LFM2.5 1.2B Instruct", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Instruct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP-202606": [ { "name": "LFM2.5 1.2B JP (202606)", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP-202606", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP": [ { "name": "LFM2.5 1.2B JP", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Thinking": [ { "name": "LFM2.5 1.2B Thinking", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Thinking", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-230M": [ { "name": "LFM2.5 230M", "huggingface_id": "LiquidAI/LFM2.5-230M", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-350M": [ { "name": "LFM2.5 350M", "huggingface_id": "LiquidAI/LFM2.5-350M", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-8B-A1B": [ { "name": "LFM2.5 8B A1B", "huggingface_id": "LiquidAI/LFM2.5-8B-A1B", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-1.6B": [ { "name": "LFM2.5 VL 1.6B", "huggingface_id": "LiquidAI/LFM2.5-VL-1.6B", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-450M": [ { "name": "LFM2.5 VL 450M", "huggingface_id": "LiquidAI/LFM2.5-VL-450M", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.1": [ { "name": "MiniMax-M2.1", "huggingface_id": "MiniMaxAI/MiniMax-M2.1", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.5": [ { "name": "MiniMax-M2.5", "huggingface_id": "MiniMaxAI/MiniMax-M2.5", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--max-num-seqs": "512", "--max-num-batched-tokens": "32768", "--kv-cache-dtype": "fp8", "--moe-backend": "triton", "--attention-backend": "FLASHINFER", "--enable-flashinfer-autotune": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "PYTHONNOUSERSITE": "1", "VLLM_USE_DEEP_GEMM": "0", "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--max-num-seqs": "512", "--max-num-batched-tokens": "32768", "--kv-cache-dtype": "fp8", "--moe-backend": "triton", "--attention-backend": "FLASHINFER", "--enable-flashinfer-autotune": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "PYTHONNOUSERSITE": "1", "VLLM_USE_DEEP_GEMM": "0", "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.5-NVFP4": [ { "name": "MiniMax-M2.5 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.5-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--max-num-seqs": "512", "--max-num-batched-tokens": "32768", "--kv-cache-dtype": "fp8", "--moe-backend": "triton", "--attention-backend": "FLASHINFER", "--enable-flashinfer-autotune": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "PYTHONNOUSERSITE": "1", "VLLM_USE_DEEP_GEMM": "0", "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--max-num-seqs": "512", "--max-num-batched-tokens": "32768", "--kv-cache-dtype": "fp8", "--moe-backend": "triton", "--attention-backend": "FLASHINFER", "--enable-flashinfer-autotune": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "PYTHONNOUSERSITE": "1", "VLLM_USE_DEEP_GEMM": "0", "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2": [ { "name": "MiniMax-M2", "huggingface_id": "MiniMaxAI/MiniMax-M2", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/MiniMax-M2-NVFP4": [ { "name": "MiniMax-M2 (nvfp4)", "huggingface_id": "RedHatAI/MiniMax-M2-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "OpenGVLab/InternVL3_5-8B": [ { "name": "InternVL3.5", "huggingface_id": "OpenGVLab/InternVL3_5-8B", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.2": [ { "name": "DeepSeek-V3.2", "huggingface_id": "deepseek-ai/DeepSeek-V3.2", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.2-NVFP4": [ { "name": "DeepSeek-V3.2 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.2-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "meta-llama/Llama-4-Scout-17B-16E-Instruct": [ { "name": "Llama-4-Scout", "huggingface_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192" }, "extra_envs": {} }, "balance": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8": [ { "name": "Llama-4-Scout (fp8)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--kv-cache-dtype": "fp8" }, "extra_envs": {} }, "balance": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4": [ { "name": "Llama-4-Scout (nvfp4)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "microsoft/Phi-4-mini-instruct": [ { "name": "Phi-4", "huggingface_id": "microsoft/Phi-4-mini-instruct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "microsoft/Phi-4-mini-reasoning": [ { "name": "Phi-4 (mini_reasoning)", "huggingface_id": "microsoft/Phi-4-mini-reasoning", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mini_reasoning" } ], "microsoft/Phi-4-reasoning": [ { "name": "Phi-4 (reasoning)", "huggingface_id": "microsoft/Phi-4-reasoning", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "reasoning" } ], "microsoft/Phi-4-multimodal-instruct": [ { "name": "Phi-4 (multimodal)", "huggingface_id": "microsoft/Phi-4-multimodal-instruct", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "multimodal" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512": [ { "name": "Mistral-Large-3-675B-Instruct", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4": [ { "name": "Mistral-Large-3-675B-Instruct (nvfp4)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8": [ { "name": "Mistral-Large-3-675B-Instruct (fp8)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Cosmos3-Nano": [ { "name": "Cosmos3-Nano", "huggingface_id": "nvidia/Cosmos3-Nano", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Image2Video": [ { "name": "Cosmos3-Super-Image2Video", "huggingface_id": "nvidia/Cosmos3-Super-Image2Video", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Text2Image": [ { "name": "Cosmos3-Super-Text2Image", "huggingface_id": "nvidia/Cosmos3-Super-Text2Image", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super": [ { "name": "Cosmos3-Super", "huggingface_id": "nvidia/Cosmos3-Super", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-4B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-4B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (base_bf16)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base_bf16" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2": [ { "name": "NVIDIA Nemotron-Nano-9B-v2", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (base)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (japanese)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "japanese" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (fp8)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (nvfp4)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5V": [ { "name": "GLM-4.5V", "huggingface_id": "zai-org/GLM-4.5V", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5V-FP8": [ { "name": "GLM-4.5V (fp8)", "huggingface_id": "zai-org/GLM-4.5V-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6V": [ { "name": "GLM-4.6V", "huggingface_id": "zai-org/GLM-4.6V", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6V-FP8": [ { "name": "GLM-4.6V (fp8)", "huggingface_id": "zai-org/GLM-4.6V-FP8", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-Image": [ { "name": "GLM-Image", "huggingface_id": "zai-org/GLM-Image", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/Glyph": [ { "name": "Glyph", "huggingface_id": "zai-org/Glyph", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--limit-mm-per-prompt.video": "0", "--reasoning-parser": "glm45", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "h100", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "h200": { "google/diffusiongemma-26B-A4B-it": [ { "name": "DiffusionGemma 26B-A4B IT", "huggingface_id": "google/diffusiongemma-26B-A4B-it", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic": [ { "name": "DiffusionGemma 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4": [ { "name": "DiffusionGemma 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "JetBrains/Mellum2-12B-A2.5B-Instruct": [ { "name": "Mellum2-12B-A2.5B-Instruct", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Instruct", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "JetBrains/Mellum2-12B-A2.5B-Thinking": [ { "name": "Mellum2-12B-A2.5B-Thinking", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Thinking", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Base": [ { "name": "LFM2.5 1.2B Base", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Base", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Instruct": [ { "name": "LFM2.5 1.2B Instruct", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Instruct", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP-202606": [ { "name": "LFM2.5 1.2B JP (202606)", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP-202606", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP": [ { "name": "LFM2.5 1.2B JP", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Thinking": [ { "name": "LFM2.5 1.2B Thinking", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Thinking", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-230M": [ { "name": "LFM2.5 230M", "huggingface_id": "LiquidAI/LFM2.5-230M", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-350M": [ { "name": "LFM2.5 350M", "huggingface_id": "LiquidAI/LFM2.5-350M", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-8B-A1B": [ { "name": "LFM2.5 8B A1B", "huggingface_id": "LiquidAI/LFM2.5-8B-A1B", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-1.6B": [ { "name": "LFM2.5 VL 1.6B", "huggingface_id": "LiquidAI/LFM2.5-VL-1.6B", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-450M": [ { "name": "LFM2.5 VL 450M", "huggingface_id": "LiquidAI/LFM2.5-VL-450M", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.5": [ { "name": "MiniMax-M2.5", "huggingface_id": "MiniMaxAI/MiniMax-M2.5", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--max-num-seqs": "512", "--max-num-batched-tokens": "32768", "--kv-cache-dtype": "fp8", "--moe-backend": "triton", "--attention-backend": "FLASHINFER", "--enable-flashinfer-autotune": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "PYTHONNOUSERSITE": "1", "VLLM_USE_DEEP_GEMM": "0", "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--max-num-seqs": "512", "--max-num-batched-tokens": "32768", "--kv-cache-dtype": "fp8", "--moe-backend": "triton", "--attention-backend": "FLASHINFER", "--enable-flashinfer-autotune": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "PYTHONNOUSERSITE": "1", "VLLM_USE_DEEP_GEMM": "0", "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.5-NVFP4": [ { "name": "MiniMax-M2.5 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.5-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--max-num-seqs": "512", "--max-num-batched-tokens": "32768", "--kv-cache-dtype": "fp8", "--moe-backend": "triton", "--attention-backend": "FLASHINFER", "--enable-flashinfer-autotune": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "PYTHONNOUSERSITE": "1", "VLLM_USE_DEEP_GEMM": "0", "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--max-num-seqs": "512", "--max-num-batched-tokens": "32768", "--kv-cache-dtype": "fp8", "--moe-backend": "triton", "--attention-backend": "FLASHINFER", "--enable-flashinfer-autotune": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "PYTHONNOUSERSITE": "1", "VLLM_USE_DEEP_GEMM": "0", "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2": [ { "name": "MiniMax-M2", "huggingface_id": "MiniMaxAI/MiniMax-M2", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/MiniMax-M2-NVFP4": [ { "name": "MiniMax-M2 (nvfp4)", "huggingface_id": "RedHatAI/MiniMax-M2-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M3": [ { "name": "MiniMax-M3", "huggingface_id": "MiniMaxAI/MiniMax-M3", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M3-MXFP8": [ { "name": "MiniMax-M3 (mxfp8)", "huggingface_id": "MiniMaxAI/MiniMax-M3-MXFP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp8" } ], "nvidia/MiniMax-M3-NVFP4": [ { "name": "MiniMax-M3 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M3-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "amd/MiniMax-M3-MXFP4": [ { "name": "MiniMax-M3 (mxfp4)", "huggingface_id": "amd/MiniMax-M3-MXFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp4" } ], "Qwen/Qwen3-235B-A22B-Instruct-2507": [ { "name": "Qwen3-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-235B-A22B-FP8": [ { "name": "Qwen3-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-235B-A22B-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4": [ { "name": "Qwen3-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct": [ { "name": "Qwen3-Coder-480B-A35B-Instruct", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_USE_DEEP_GEMM": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_DEEP_GEMM": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_USE_DEEP_GEMM": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_DEEP_GEMM": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_USE_DEEP_GEMM": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_DEEP_GEMM": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct": [ { "name": "Qwen3-Next-80B-A3B-Instruct", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8": [ { "name": "Qwen3-Next-80B-A3B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4": [ { "name": "Qwen3-Next-80B-A3B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B": [ { "name": "Qwen3.5-397B", "huggingface_id": "Qwen/Qwen3.5-397B-A17B", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Qwen3.5-397B-A17B-NVFP4": [ { "name": "Qwen3.5-397B (nvfp4)", "huggingface_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4": [ { "name": "Qwen3.5-397B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "XiaomiMiMo/MiMo-V2-Flash": [ { "name": "MiMo-V2-Flash", "huggingface_id": "XiaomiMiMo/MiMo-V2-Flash", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5-Pro": [ { "name": "MiMo-V2.5-Pro", "huggingface_id": "XiaomiMiMo/MiMo-V2.5-Pro", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5": [ { "name": "MiMo-V2.5", "huggingface_id": "XiaomiMiMo/MiMo-V2.5", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1": [ { "name": "DeepSeek-R1", "huggingface_id": "deepseek-ai/DeepSeek-R1", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1-0528": [ { "name": "DeepSeek-R1 (r1_0528)", "huggingface_id": "deepseek-ai/DeepSeek-R1-0528", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "r1_0528" } ], "nvidia/DeepSeek-R1-0528-NVFP4-v2": [ { "name": "DeepSeek-R1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-R1-0528-NVFP4-v2", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.1": [ { "name": "DeepSeek-V3.1", "huggingface_id": "deepseek-ai/DeepSeek-V3.1", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.1-NVFP4": [ { "name": "DeepSeek-V3.1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.1-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.2-Exp": [ { "name": "DeepSeek-V3.2-Exp", "huggingface_id": "deepseek-ai/DeepSeek-V3.2-Exp", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-V3.2": [ { "name": "DeepSeek-V3.2", "huggingface_id": "deepseek-ai/DeepSeek-V3.2", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.2-NVFP4": [ { "name": "DeepSeek-V3.2 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.2-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3": [ { "name": "DeepSeek-V3", "huggingface_id": "deepseek-ai/DeepSeek-V3", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3-FP4": [ { "name": "DeepSeek-V3 (fp4)", "huggingface_id": "nvidia/DeepSeek-V3-FP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--gpu-memory-utilization": "0.95", "--max-num-seqs": "16", "--max-num-batched-tokens": "16384", "--no-enable-flashinfer-autotune": "", "--compilation-config": "{\"mode\": 0, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--gpu-memory-utilization": "0.95", "--max-num-seqs": "16", "--max-num-batched-tokens": "16384", "--no-enable-flashinfer-autotune": "", "--compilation-config": "{\"mode\": 0, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--gpu-memory-utilization": "0.95", "--max-num-seqs": "16", "--max-num-batched-tokens": "16384", "--no-enable-flashinfer-autotune": "", "--compilation-config": "{\"mode\": 0, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--gpu-memory-utilization": "0.95", "--max-num-seqs": "16", "--max-num-batched-tokens": "16384", "--no-enable-flashinfer-autotune": "", "--compilation-config": "{\"mode\": 0, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--gpu-memory-utilization": "0.95", "--max-num-seqs": "16", "--max-num-batched-tokens": "16384", "--no-enable-flashinfer-autotune": "", "--compilation-config": "{\"mode\": 0, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--gpu-memory-utilization": "0.95", "--max-num-seqs": "16", "--max-num-batched-tokens": "16384", "--no-enable-flashinfer-autotune": "", "--compilation-config": "{\"mode\": 0, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "inclusionAI/Ling-2.6-flash": [ { "name": "Ling-2.6-flash", "huggingface_id": "inclusionAI/Ling-2.6-flash", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-1T-FP8": [ { "name": "Ring-1T-FP8", "huggingface_id": "inclusionAI/Ring-1T-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S2-Preview": [ { "name": "Intern-S2-Preview", "huggingface_id": "internlm/Intern-S2-Preview", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S2-Preview-FP8": [ { "name": "Intern-S2-Preview (fp8)", "huggingface_id": "internlm/Intern-S2-Preview-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Ministral-3-14B-Instruct-2512": [ { "name": "Ministral-3-Instruct", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-8B-Instruct-2512": [ { "name": "Ministral-3-Instruct (8b)", "huggingface_id": "mistralai/Ministral-3-8B-Instruct-2512", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "8b" } ], "mistralai/Ministral-3-3B-Instruct-2512": [ { "name": "Ministral-3-Instruct (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Instruct-2512", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Instruct-2512-FP8": [ { "name": "Ministral-3-Instruct (fp8)", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Ministral-3-8B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning", "huggingface_id": "mistralai/Ministral-3-8B-Reasoning-2512", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-3B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Reasoning-2512", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (14b)", "huggingface_id": "mistralai/Ministral-3-14B-Reasoning-2512", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "14b" } ], "moonshotai/Kimi-K2-Instruct": [ { "name": "Kimi-K2-Instruct", "huggingface_id": "moonshotai/Kimi-K2-Instruct", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Thinking": [ { "name": "Kimi-K2-Thinking", "huggingface_id": "moonshotai/Kimi-K2-Thinking", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2-Thinking-NVFP4": [ { "name": "Kimi-K2-Thinking (nvfp4)", "huggingface_id": "nvidia/Kimi-K2-Thinking-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.5": [ { "name": "Kimi-K2.5", "huggingface_id": "moonshotai/Kimi-K2.5", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.5-NVFP4": [ { "name": "Kimi-K2.5 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.5-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.7-Code": [ { "name": "Kimi-K2.7-Code", "huggingface_id": "moonshotai/Kimi-K2.7-Code", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Nano": [ { "name": "Cosmos3-Nano", "huggingface_id": "nvidia/Cosmos3-Nano", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Image2Video": [ { "name": "Cosmos3-Super-Image2Video", "huggingface_id": "nvidia/Cosmos3-Super-Image2Video", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Text2Image": [ { "name": "Cosmos3-Super-Text2Image", "huggingface_id": "nvidia/Cosmos3-Super-Text2Image", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super": [ { "name": "Cosmos3-Super", "huggingface_id": "nvidia/Cosmos3-Super", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-4B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-4B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (base_bf16)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base_bf16" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2": [ { "name": "NVIDIA Nemotron-Nano-9B-v2", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (base)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (japanese)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "japanese" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (fp8)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (nvfp4)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS-2.1": [ { "name": "Laguna XS-2.1", "huggingface_id": "poolside/Laguna-XS-2.1", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS-2.1-FP8": [ { "name": "Laguna XS-2.1 (fp8)", "huggingface_id": "poolside/Laguna-XS-2.1-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS-2.1-NVFP4": [ { "name": "Laguna XS-2.1 (nvfp4)", "huggingface_id": "poolside/Laguna-XS-2.1-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS-2.1-INT4": [ { "name": "Laguna XS-2.1 (int4)", "huggingface_id": "poolside/Laguna-XS-2.1-INT4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "poolside/Laguna-XS.2": [ { "name": "Laguna XS.2", "huggingface_id": "poolside/Laguna-XS.2", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS.2-FP8": [ { "name": "Laguna XS.2 (fp8)", "huggingface_id": "poolside/Laguna-XS.2-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS.2-NVFP4": [ { "name": "Laguna XS.2 (nvfp4)", "huggingface_id": "poolside/Laguna-XS.2-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS.2-INT4": [ { "name": "Laguna XS.2 (int4)", "huggingface_id": "poolside/Laguna-XS.2-INT4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stepfun-ai/Step-3.5-Flash": [ { "name": "Step-3.5-Flash", "huggingface_id": "stepfun-ai/Step-3.5-Flash", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.5-Flash-FP8": [ { "name": "Step-3.5-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.5-Flash-INT4": [ { "name": "Step-3.5-Flash (int4)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stepfun-ai/Step-3.5-Flash-INT8": [ { "name": "Step-3.5-Flash (int8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "stepfun-ai/Step-3.7-Flash": [ { "name": "Step-3.7-Flash", "huggingface_id": "stepfun-ai/Step-3.7-Flash", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.7-Flash-FP8": [ { "name": "Step-3.7-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.7-Flash-NVFP4": [ { "name": "Step-3.7-Flash (nvfp4)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "tencent/Hy3-preview": [ { "name": "Hy3-preview", "huggingface_id": "tencent/Hy3-preview", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-4.5": [ { "name": "GLM-4.5", "huggingface_id": "zai-org/GLM-4.5", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5-FP8": [ { "name": "GLM-4.5 (fp8)", "huggingface_id": "zai-org/GLM-4.5-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.5V": [ { "name": "GLM-4.5V", "huggingface_id": "zai-org/GLM-4.5V", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5V-FP8": [ { "name": "GLM-4.5V (fp8)", "huggingface_id": "zai-org/GLM-4.5V-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6": [ { "name": "GLM-4.6", "huggingface_id": "zai-org/GLM-4.6", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6-FP8": [ { "name": "GLM-4.6 (fp8)", "huggingface_id": "zai-org/GLM-4.6-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6V": [ { "name": "GLM-4.6V", "huggingface_id": "zai-org/GLM-4.6V", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6V-FP8": [ { "name": "GLM-4.6V (fp8)", "huggingface_id": "zai-org/GLM-4.6V-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.7": [ { "name": "GLM-4.7", "huggingface_id": "zai-org/GLM-4.7", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.7-FP8": [ { "name": "GLM-4.7 (fp8)", "huggingface_id": "zai-org/GLM-4.7-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-4.7-NVFP4": [ { "name": "GLM-4.7 (nvfp4)", "huggingface_id": "nvidia/GLM-4.7-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.1": [ { "name": "GLM-5.1", "huggingface_id": "zai-org/GLM-5.1", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5.1-FP8": [ { "name": "GLM-5.1 (fp8)", "huggingface_id": "zai-org/GLM-5.1-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5.1-NVFP4": [ { "name": "GLM-5.1 (nvfp4)", "huggingface_id": "nvidia/GLM-5.1-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5": [ { "name": "GLM-5", "huggingface_id": "zai-org/GLM-5", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5-FP8": [ { "name": "GLM-5 (fp8)", "huggingface_id": "zai-org/GLM-5-FP8", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5-NVFP4": [ { "name": "GLM-5 (nvfp4)", "huggingface_id": "nvidia/GLM-5-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "h200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "b200": { "LiquidAI/LFM2.5-1.2B-Base": [ { "name": "LFM2.5 1.2B Base", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Base", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Instruct": [ { "name": "LFM2.5 1.2B Instruct", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Instruct", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP-202606": [ { "name": "LFM2.5 1.2B JP (202606)", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP-202606", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP": [ { "name": "LFM2.5 1.2B JP", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Thinking": [ { "name": "LFM2.5 1.2B Thinking", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Thinking", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-230M": [ { "name": "LFM2.5 230M", "huggingface_id": "LiquidAI/LFM2.5-230M", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-350M": [ { "name": "LFM2.5 350M", "huggingface_id": "LiquidAI/LFM2.5-350M", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-8B-A1B": [ { "name": "LFM2.5 8B A1B", "huggingface_id": "LiquidAI/LFM2.5-8B-A1B", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-1.6B": [ { "name": "LFM2.5 VL 1.6B", "huggingface_id": "LiquidAI/LFM2.5-VL-1.6B", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-450M": [ { "name": "LFM2.5 VL 450M", "huggingface_id": "LiquidAI/LFM2.5-VL-450M", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": { "VLLM_USE_OINK_OPS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M3": [ { "name": "MiniMax-M3", "huggingface_id": "MiniMaxAI/MiniMax-M3", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M3-MXFP8": [ { "name": "MiniMax-M3 (mxfp8)", "huggingface_id": "MiniMaxAI/MiniMax-M3-MXFP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp8" } ], "nvidia/MiniMax-M3-NVFP4": [ { "name": "MiniMax-M3 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M3-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "amd/MiniMax-M3-MXFP4": [ { "name": "MiniMax-M3 (mxfp4)", "huggingface_id": "amd/MiniMax-M3-MXFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp4" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-R1": [ { "name": "DeepSeek-R1", "huggingface_id": "deepseek-ai/DeepSeek-R1", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1-0528": [ { "name": "DeepSeek-R1 (r1_0528)", "huggingface_id": "deepseek-ai/DeepSeek-R1-0528", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "r1_0528" } ], "nvidia/DeepSeek-R1-0528-NVFP4-v2": [ { "name": "DeepSeek-R1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-R1-0528-NVFP4-v2", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3": [ { "name": "DeepSeek-V3", "huggingface_id": "deepseek-ai/DeepSeek-V3", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3-FP4": [ { "name": "DeepSeek-V3 (fp4)", "huggingface_id": "nvidia/DeepSeek-V3-FP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"fuse_attn_quant\":true,\"eliminate_noops\":true}}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"fuse_attn_quant\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"fuse_attn_quant\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "meta-llama/Llama-4-Scout-17B-16E-Instruct": [ { "name": "Llama-4-Scout", "huggingface_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } }, "balance": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8": [ { "name": "Llama-4-Scout (fp8)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } }, "balance": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4": [ { "name": "Llama-4-Scout (nvfp4)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512": [ { "name": "Mistral-Large-3-675B-Instruct", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4": [ { "name": "Mistral-Large-3-675B-Instruct (nvfp4)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8": [ { "name": "Mistral-Large-3-675B-Instruct (fp8)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-4B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-4B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (base_bf16)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base_bf16" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B (fp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2": [ { "name": "NVIDIA Nemotron-Nano-9B-v2", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (base)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (japanese)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "japanese" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (fp8)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (nvfp4)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": "1" } }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": "1" } }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": "1" } }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.5-Flash": [ { "name": "Step-3.5-Flash", "huggingface_id": "stepfun-ai/Step-3.5-Flash", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.5-Flash-FP8": [ { "name": "Step-3.5-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.5-Flash-INT4": [ { "name": "Step-3.5-Flash (int4)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "0" } } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stepfun-ai/Step-3.5-Flash-INT8": [ { "name": "Step-3.5-Flash (int8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "0" } } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "stepfun-ai/Step-3.7-Flash": [ { "name": "Step-3.7-Flash", "huggingface_id": "stepfun-ai/Step-3.7-Flash", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.7-Flash-FP8": [ { "name": "Step-3.7-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.7-Flash-NVFP4": [ { "name": "Step-3.7-Flash (nvfp4)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-5.2-FP8": [ { "name": "GLM-5.2", "huggingface_id": "zai-org/GLM-5.2-FP8", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/GLM-5.2-NVFP4": [ { "name": "GLM-5.2 (nvfp4)", "huggingface_id": "nvidia/GLM-5.2-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2": [ { "name": "GLM-5.2 (bf16)", "huggingface_id": "zai-org/GLM-5.2", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "b200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "gb200": { "Qwen/Qwen3.5-397B-A17B": [ { "name": "Qwen3.5-397B", "huggingface_id": "Qwen/Qwen3.5-397B-A17B", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Qwen3.5-397B-A17B-NVFP4": [ { "name": "Qwen3.5-397B (nvfp4)", "huggingface_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4": [ { "name": "Qwen3.5-397B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "deepseek-ai/DeepSeek-R1": [ { "name": "DeepSeek-R1", "huggingface_id": "deepseek-ai/DeepSeek-R1", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1-0528": [ { "name": "DeepSeek-R1 (r1_0528)", "huggingface_id": "deepseek-ai/DeepSeek-R1-0528", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "r1_0528" } ], "nvidia/DeepSeek-R1-0528-NVFP4-v2": [ { "name": "DeepSeek-R1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-R1-0528-NVFP4-v2", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3": [ { "name": "DeepSeek-V3", "huggingface_id": "deepseek-ai/DeepSeek-V3", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3-FP4": [ { "name": "DeepSeek-V3 (fp4)", "huggingface_id": "nvidia/DeepSeek-V3-FP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "internlm/Intern-S2-Preview": [ { "name": "Intern-S2-Preview", "huggingface_id": "internlm/Intern-S2-Preview", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S2-Preview-FP8": [ { "name": "Intern-S2-Preview (fp8)", "huggingface_id": "internlm/Intern-S2-Preview-FP8", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"fuse_attn_quant\":true,\"eliminate_noops\":true}}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"fuse_attn_quant\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"fuse_attn_quant\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "meta-llama/Llama-4-Scout-17B-16E-Instruct": [ { "name": "Llama-4-Scout", "huggingface_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } }, "balance": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8": [ { "name": "Llama-4-Scout (fp8)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } }, "balance": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4": [ { "name": "Llama-4-Scout (nvfp4)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--async-scheduling": "", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"pass_config\":{\"fuse_allreduce_rms\":true,\"eliminate_noops\":true}}", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Ministral-3-8B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning", "huggingface_id": "mistralai/Ministral-3-8B-Reasoning-2512", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-3B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Reasoning-2512", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (14b)", "huggingface_id": "mistralai/Ministral-3-14B-Reasoning-2512", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "14b" } ], "moonshotai/Kimi-K2.5": [ { "name": "Kimi-K2.5", "huggingface_id": "moonshotai/Kimi-K2.5", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-config.use_trtllm_ragged_deepseek_prefill=True": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-config.use_trtllm_ragged_deepseek_prefill=True": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.5-NVFP4": [ { "name": "Kimi-K2.5 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.5-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-config.use_trtllm_ragged_deepseek_prefill=True": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-config.use_trtllm_ragged_deepseek_prefill=True": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-config": "{\"mla_prefill_backend\":\"TRTLLM_RAGGED\",\"use_prefill_query_quantization\":true}", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-config": "{\"mla_prefill_backend\":\"TRTLLM_RAGGED\",\"use_prefill_query_quantization\":true}", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-config": "{\"mla_prefill_backend\":\"TRTLLM_RAGGED\",\"use_prefill_query_quantization\":true}", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-config": "{\"mla_prefill_backend\":\"TRTLLM_RAGGED\",\"use_prefill_query_quantization\":true}", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/Cosmos3-Super-Text2Image": [ { "name": "Cosmos3-Super-Text2Image", "huggingface_id": "nvidia/Cosmos3-Super-Text2Image", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "gb200", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "b300": { "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "inclusionAI/Ling-2.6-1T": [ { "name": "Ling-2.6-1T", "huggingface_id": "inclusionAI/Ling-2.6-1T", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-config": "{\"mla_prefill_backend\":\"TRTLLM_RAGGED\",\"use_prefill_query_quantization\":true}", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-config": "{\"mla_prefill_backend\":\"TRTLLM_RAGGED\",\"use_prefill_query_quantization\":true}", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-config": "{\"mla_prefill_backend\":\"TRTLLM_RAGGED\",\"use_prefill_query_quantization\":true}", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-config": "{\"mla_prefill_backend\":\"TRTLLM_RAGGED\",\"use_prefill_query_quantization\":true}", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2-FP8": [ { "name": "GLM-5.2", "huggingface_id": "zai-org/GLM-5.2-FP8", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/GLM-5.2-NVFP4": [ { "name": "GLM-5.2 (nvfp4)", "huggingface_id": "nvidia/GLM-5.2-NVFP4", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2": [ { "name": "GLM-5.2 (bf16)", "huggingface_id": "zai-org/GLM-5.2", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "b300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "gb300": { "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "tencent/Hy3": [ { "name": "Hy3", "huggingface_id": "tencent/Hy3", "accelerators": "gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" } }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-FP8": [ { "name": "Hy3 (fp8)", "huggingface_id": "tencent/Hy3-FP8", "accelerators": "gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" } }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "dgx_station_gb300": { "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--attention_config.use_fp4_indexer_cache=True": "", "--moe-backend": "deep_gemm_mega_moe", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (base_bf16)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base_bf16" } ], "stepfun-ai/Step-3.7-Flash": [ { "name": "Step-3.7-Flash", "huggingface_id": "stepfun-ai/Step-3.7-Flash", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.7-Flash-FP8": [ { "name": "Step-3.7-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-FP8", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.7-Flash-NVFP4": [ { "name": "Step-3.7-Flash (nvfp4)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-NVFP4", "accelerators": "dgx_station_gb300", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ] }, "dgx_spark_gb10": { "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "dgx_spark_gb10", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "dgx_spark_gb10", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "dgx_spark_gb10", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "accelerators": "dgx_spark_gb10", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "accelerators": "dgx_spark_gb10", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "accelerators": "dgx_spark_gb10", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (base_bf16)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "accelerators": "dgx_spark_gb10", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base_bf16" } ] }, "mi300x": { "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "MiniMaxAI/MiniMax-M2.1": [ { "name": "MiniMax-M2.1", "huggingface_id": "MiniMaxAI/MiniMax-M2.1", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.5": [ { "name": "MiniMax-M2.5", "huggingface_id": "MiniMaxAI/MiniMax-M2.5", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.5-NVFP4": [ { "name": "MiniMax-M2.5 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.5-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2": [ { "name": "MiniMax-M2", "huggingface_id": "MiniMaxAI/MiniMax-M2", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/MiniMax-M2-NVFP4": [ { "name": "MiniMax-M2 (nvfp4)", "huggingface_id": "RedHatAI/MiniMax-M2-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M3": [ { "name": "MiniMax-M3", "huggingface_id": "MiniMaxAI/MiniMax-M3", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M3-MXFP8": [ { "name": "MiniMax-M3 (mxfp8)", "huggingface_id": "MiniMaxAI/MiniMax-M3-MXFP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } } }, "default_strategy": "latency-focus", "variant_name": "mxfp8" } ], "nvidia/MiniMax-M3-NVFP4": [ { "name": "MiniMax-M3 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M3-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "amd/MiniMax-M3-MXFP4": [ { "name": "MiniMax-M3 (mxfp4)", "huggingface_id": "amd/MiniMax-M3-MXFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } } }, "default_strategy": "latency-focus", "variant_name": "mxfp4" } ], "OpenGVLab/InternVL3_5-8B": [ { "name": "InternVL3.5", "huggingface_id": "OpenGVLab/InternVL3_5-8B", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct": [ { "name": "Qwen2.5-VL-72B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--limit-mm-per-prompt": "{\"image\":2,\"video\":0}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-72B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct-AWQ", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--limit-mm-per-prompt": "{\"image\":2,\"video\":0}", "--quantization": "awq" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-235B-A22B-Instruct-2507": [ { "name": "Qwen3-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-235B-A22B-FP8": [ { "name": "Qwen3-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-235B-A22B-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4": [ { "name": "Qwen3-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct": [ { "name": "Qwen3-Coder-480B-A35B-Instruct", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct": [ { "name": "Qwen3-Next-80B-A3B-Instruct", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8": [ { "name": "Qwen3-Next-80B-A3B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4": [ { "name": "Qwen3-Next-80B-A3B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B": [ { "name": "Qwen3.5-397B", "huggingface_id": "Qwen/Qwen3.5-397B-A17B", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Qwen3.5-397B-A17B-NVFP4": [ { "name": "Qwen3.5-397B (nvfp4)", "huggingface_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4": [ { "name": "Qwen3.5-397B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3Guard-Gen-8B": [ { "name": "Qwen3Guard-Gen-8B", "huggingface_id": "Qwen/Qwen3Guard-Gen-8B", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3Guard-Gen-4B": [ { "name": "Qwen3Guard-Gen-8B (small_4b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-4B", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "small_4b" } ], "Qwen/Qwen3Guard-Gen-0.6B": [ { "name": "Qwen3Guard-Gen-8B (tiny_0_6b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-0.6B", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "tiny_0_6b" } ], "XiaomiMiMo/MiMo-V2-Flash": [ { "name": "MiMo-V2-Flash", "huggingface_id": "XiaomiMiMo/MiMo-V2-Flash", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-28B-A3B-PT": [ { "name": "ERNIE-4.5-VL", "huggingface_id": "baidu/ERNIE-4.5-VL-28B-A3B-PT", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-424B-A47B-PT": [ { "name": "ERNIE-4.5-VL (424b)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b" }, { "name": "ERNIE-4.5-VL (424b_fp8)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b_fp8" } ], "deepseek-ai/DeepSeek-V3": [ { "name": "DeepSeek-V3", "huggingface_id": "deepseek-ai/DeepSeek-V3", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "ghcr.io/embeddedllm/vllm-rocm:ubuntu22.04_rocm6.4.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default", "extra_envs": { "VLLM_RPC_TIMEOUT": "1800000", "SAFETENSORS_FAST_GPU": "1" } } ], "nvidia/DeepSeek-V3-FP4": [ { "name": "DeepSeek-V3 (fp4)", "huggingface_id": "nvidia/DeepSeek-V3-FP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "inclusionAI/Ling-2.6-1T": [ { "name": "Ling-2.6-1T", "huggingface_id": "inclusionAI/Ling-2.6-1T", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ling-2.6-flash": [ { "name": "Ling-2.6-flash", "huggingface_id": "inclusionAI/Ling-2.6-flash", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-1T-FP8": [ { "name": "Ring-1T-FP8", "huggingface_id": "inclusionAI/Ring-1T-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1": [ { "name": "Intern-S1", "huggingface_id": "internlm/Intern-S1", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1-FP8": [ { "name": "Intern-S1 (fp8)", "huggingface_id": "internlm/Intern-S1-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Ministral-3-14B-Instruct-2512": [ { "name": "Ministral-3-Instruct", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-8B-Instruct-2512": [ { "name": "Ministral-3-Instruct (8b)", "huggingface_id": "mistralai/Ministral-3-8B-Instruct-2512", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "8b" } ], "mistralai/Ministral-3-3B-Instruct-2512": [ { "name": "Ministral-3-Instruct (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Instruct-2512", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Instruct-2512-FP8": [ { "name": "Ministral-3-Instruct (fp8)", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Ministral-3-8B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning", "huggingface_id": "mistralai/Ministral-3-8B-Reasoning-2512", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-3B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Reasoning-2512", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (14b)", "huggingface_id": "mistralai/Ministral-3-14B-Reasoning-2512", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "14b" } ], "mistralai/Mistral-Medium-3.5-128B": [ { "name": "Mistral-Medium-3.5", "huggingface_id": "mistralai/Mistral-Medium-3.5-128B", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"mistralai/Mistral-Medium-3.5-128B-EAGLE\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Instruct": [ { "name": "Kimi-K2-Instruct", "huggingface_id": "moonshotai/Kimi-K2-Instruct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2.5": [ { "name": "Kimi-K2.5", "huggingface_id": "moonshotai/Kimi-K2.5", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.5-NVFP4": [ { "name": "Kimi-K2.5 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.5-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.7-Code": [ { "name": "Kimi-K2.7-Code", "huggingface_id": "moonshotai/Kimi-K2.7-Code", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct": [ { "name": "Hunyuan-A13B-Instruct", "huggingface_id": "tencent/Hunyuan-A13B-Instruct", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct-FP8": [ { "name": "Hunyuan-A13B-Instruct (fp8)", "huggingface_id": "tencent/Hunyuan-A13B-Instruct-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "tencent/Hy3-preview": [ { "name": "Hy3-preview", "huggingface_id": "tencent/Hy3-preview", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3": [ { "name": "Hy3", "huggingface_id": "tencent/Hy3", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-FP8": [ { "name": "Hy3 (fp8)", "huggingface_id": "tencent/Hy3-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-4.5": [ { "name": "GLM-4.5", "huggingface_id": "zai-org/GLM-4.5", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5-FP8": [ { "name": "GLM-4.5 (fp8)", "huggingface_id": "zai-org/GLM-4.5-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6": [ { "name": "GLM-4.6", "huggingface_id": "zai-org/GLM-4.6", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6-FP8": [ { "name": "GLM-4.6 (fp8)", "huggingface_id": "zai-org/GLM-4.6-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.7": [ { "name": "GLM-4.7", "huggingface_id": "zai-org/GLM-4.7", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.7-FP8": [ { "name": "GLM-4.7 (fp8)", "huggingface_id": "zai-org/GLM-4.7-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-4.7-NVFP4": [ { "name": "GLM-4.7 (nvfp4)", "huggingface_id": "nvidia/GLM-4.7-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2-FP8": [ { "name": "GLM-5.2", "huggingface_id": "zai-org/GLM-5.2-FP8", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/GLM-5.2-NVFP4": [ { "name": "GLM-5.2 (nvfp4)", "huggingface_id": "nvidia/GLM-5.2-NVFP4", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2": [ { "name": "GLM-5.2 (bf16)", "huggingface_id": "zai-org/GLM-5.2", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/Glyph": [ { "name": "Glyph", "huggingface_id": "zai-org/Glyph", "accelerators": "mi300x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--limit-mm-per-prompt.video": "0", "--reasoning-parser": "glm45", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "mi325x": { "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "MiniMaxAI/MiniMax-M2.1": [ { "name": "MiniMax-M2.1", "huggingface_id": "MiniMaxAI/MiniMax-M2.1", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.5": [ { "name": "MiniMax-M2.5", "huggingface_id": "MiniMaxAI/MiniMax-M2.5", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.5-NVFP4": [ { "name": "MiniMax-M2.5 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.5-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2": [ { "name": "MiniMax-M2", "huggingface_id": "MiniMaxAI/MiniMax-M2", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/MiniMax-M2-NVFP4": [ { "name": "MiniMax-M2 (nvfp4)", "huggingface_id": "RedHatAI/MiniMax-M2-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "OpenGVLab/InternVL3_5-8B": [ { "name": "InternVL3.5", "huggingface_id": "OpenGVLab/InternVL3_5-8B", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct": [ { "name": "Qwen2.5-VL-72B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--limit-mm-per-prompt": "{\"image\":2,\"video\":0}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-72B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct-AWQ", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--limit-mm-per-prompt": "{\"image\":2,\"video\":0}", "--quantization": "awq" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-235B-A22B-Instruct-2507": [ { "name": "Qwen3-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-235B-A22B-FP8": [ { "name": "Qwen3-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-235B-A22B-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4": [ { "name": "Qwen3-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct": [ { "name": "Qwen3-Coder-480B-A35B-Instruct", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct": [ { "name": "Qwen3-Next-80B-A3B-Instruct", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8": [ { "name": "Qwen3-Next-80B-A3B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4": [ { "name": "Qwen3-Next-80B-A3B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B": [ { "name": "Qwen3.5-397B", "huggingface_id": "Qwen/Qwen3.5-397B-A17B", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Qwen3.5-397B-A17B-NVFP4": [ { "name": "Qwen3.5-397B (nvfp4)", "huggingface_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4": [ { "name": "Qwen3.5-397B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "XiaomiMiMo/MiMo-V2-Flash": [ { "name": "MiMo-V2-Flash", "huggingface_id": "XiaomiMiMo/MiMo-V2-Flash", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-28B-A3B-PT": [ { "name": "ERNIE-4.5-VL", "huggingface_id": "baidu/ERNIE-4.5-VL-28B-A3B-PT", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-424B-A47B-PT": [ { "name": "ERNIE-4.5-VL (424b)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b" }, { "name": "ERNIE-4.5-VL (424b_fp8)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b_fp8" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "inclusionAI/Ring-1T-FP8": [ { "name": "Ring-1T-FP8", "huggingface_id": "inclusionAI/Ring-1T-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1": [ { "name": "Intern-S1", "huggingface_id": "internlm/Intern-S1", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1-FP8": [ { "name": "Intern-S1 (fp8)", "huggingface_id": "internlm/Intern-S1-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Ministral-3-14B-Instruct-2512": [ { "name": "Ministral-3-Instruct", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-8B-Instruct-2512": [ { "name": "Ministral-3-Instruct (8b)", "huggingface_id": "mistralai/Ministral-3-8B-Instruct-2512", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "8b" } ], "mistralai/Ministral-3-3B-Instruct-2512": [ { "name": "Ministral-3-Instruct (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Instruct-2512", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Instruct-2512-FP8": [ { "name": "Ministral-3-Instruct (fp8)", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "moonshotai/Kimi-K2-Instruct": [ { "name": "Kimi-K2-Instruct", "huggingface_id": "moonshotai/Kimi-K2-Instruct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2.5": [ { "name": "Kimi-K2.5", "huggingface_id": "moonshotai/Kimi-K2.5", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.5-NVFP4": [ { "name": "Kimi-K2.5 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.5-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.7-Code": [ { "name": "Kimi-K2.7-Code", "huggingface_id": "moonshotai/Kimi-K2.7-Code", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct": [ { "name": "Hunyuan-A13B-Instruct", "huggingface_id": "tencent/Hunyuan-A13B-Instruct", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct-FP8": [ { "name": "Hunyuan-A13B-Instruct (fp8)", "huggingface_id": "tencent/Hunyuan-A13B-Instruct-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "tencent/Hy3-preview": [ { "name": "Hy3-preview", "huggingface_id": "tencent/Hy3-preview", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-4.5": [ { "name": "GLM-4.5", "huggingface_id": "zai-org/GLM-4.5", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5-FP8": [ { "name": "GLM-4.5 (fp8)", "huggingface_id": "zai-org/GLM-4.5-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6": [ { "name": "GLM-4.6", "huggingface_id": "zai-org/GLM-4.6", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6-FP8": [ { "name": "GLM-4.6 (fp8)", "huggingface_id": "zai-org/GLM-4.6-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.7": [ { "name": "GLM-4.7", "huggingface_id": "zai-org/GLM-4.7", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.7-FP8": [ { "name": "GLM-4.7 (fp8)", "huggingface_id": "zai-org/GLM-4.7-FP8", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-4.7-NVFP4": [ { "name": "GLM-4.7 (nvfp4)", "huggingface_id": "nvidia/GLM-4.7-NVFP4", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/Glyph": [ { "name": "Glyph", "huggingface_id": "zai-org/Glyph", "accelerators": "mi325x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--limit-mm-per-prompt.video": "0", "--reasoning-parser": "glm45", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "mi355x": { "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "MiniMaxAI/MiniMax-M2.1": [ { "name": "MiniMax-M2.1", "huggingface_id": "MiniMaxAI/MiniMax-M2.1", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.5": [ { "name": "MiniMax-M2.5", "huggingface_id": "MiniMaxAI/MiniMax-M2.5", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.5-NVFP4": [ { "name": "MiniMax-M2.5 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.5-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2": [ { "name": "MiniMax-M2", "huggingface_id": "MiniMaxAI/MiniMax-M2", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/MiniMax-M2-NVFP4": [ { "name": "MiniMax-M2 (nvfp4)", "huggingface_id": "RedHatAI/MiniMax-M2-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M3": [ { "name": "MiniMax-M3", "huggingface_id": "MiniMaxAI/MiniMax-M3", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M3-MXFP8": [ { "name": "MiniMax-M3 (mxfp8)", "huggingface_id": "MiniMaxAI/MiniMax-M3-MXFP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } } }, "default_strategy": "latency-focus", "variant_name": "mxfp8" } ], "nvidia/MiniMax-M3-NVFP4": [ { "name": "MiniMax-M3 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M3-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "amd/MiniMax-M3-MXFP4": [ { "name": "MiniMax-M3 (mxfp4)", "huggingface_id": "amd/MiniMax-M3-MXFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } } }, "default_strategy": "latency-focus", "variant_name": "mxfp4" } ], "OpenGVLab/InternVL3_5-8B": [ { "name": "InternVL3.5", "huggingface_id": "OpenGVLab/InternVL3_5-8B", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct": [ { "name": "Qwen2.5-VL-72B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--limit-mm-per-prompt": "{\"image\":2,\"video\":0}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-72B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct-AWQ", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--limit-mm-per-prompt": "{\"image\":2,\"video\":0}", "--quantization": "awq" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-235B-A22B-Instruct-2507": [ { "name": "Qwen3-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-235B-A22B-FP8": [ { "name": "Qwen3-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-235B-A22B-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4": [ { "name": "Qwen3-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct": [ { "name": "Qwen3-Coder-480B-A35B-Instruct", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct": [ { "name": "Qwen3-Next-80B-A3B-Instruct", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8": [ { "name": "Qwen3-Next-80B-A3B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4": [ { "name": "Qwen3-Next-80B-A3B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B": [ { "name": "Qwen3.5-397B", "huggingface_id": "Qwen/Qwen3.5-397B-A17B", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Qwen3.5-397B-A17B-NVFP4": [ { "name": "Qwen3.5-397B (nvfp4)", "huggingface_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4": [ { "name": "Qwen3.5-397B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "XiaomiMiMo/MiMo-V2-Flash": [ { "name": "MiMo-V2-Flash", "huggingface_id": "XiaomiMiMo/MiMo-V2-Flash", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-28B-A3B-PT": [ { "name": "ERNIE-4.5-VL", "huggingface_id": "baidu/ERNIE-4.5-VL-28B-A3B-PT", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-424B-A47B-PT": [ { "name": "ERNIE-4.5-VL (424b)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b" }, { "name": "ERNIE-4.5-VL (424b_fp8)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b_fp8" } ], "deepseek-ai/DeepSeek-V3": [ { "name": "DeepSeek-V3", "huggingface_id": "deepseek-ai/DeepSeek-V3", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default", "extra_envs": { "VLLM_RPC_TIMEOUT": "1800000" } } ], "nvidia/DeepSeek-V3-FP4": [ { "name": "DeepSeek-V3 (fp4)", "huggingface_id": "nvidia/DeepSeek-V3-FP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "inclusionAI/Ring-1T-FP8": [ { "name": "Ring-1T-FP8", "huggingface_id": "inclusionAI/Ring-1T-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1": [ { "name": "Intern-S1", "huggingface_id": "internlm/Intern-S1", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1-FP8": [ { "name": "Intern-S1 (fp8)", "huggingface_id": "internlm/Intern-S1-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Ministral-3-14B-Instruct-2512": [ { "name": "Ministral-3-Instruct", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-8B-Instruct-2512": [ { "name": "Ministral-3-Instruct (8b)", "huggingface_id": "mistralai/Ministral-3-8B-Instruct-2512", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "8b" } ], "mistralai/Ministral-3-3B-Instruct-2512": [ { "name": "Ministral-3-Instruct (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Instruct-2512", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Instruct-2512-FP8": [ { "name": "Ministral-3-Instruct (fp8)", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "moonshotai/Kimi-K2-Instruct": [ { "name": "Kimi-K2-Instruct", "huggingface_id": "moonshotai/Kimi-K2-Instruct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2.5": [ { "name": "Kimi-K2.5", "huggingface_id": "moonshotai/Kimi-K2.5", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.5-NVFP4": [ { "name": "Kimi-K2.5 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.5-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.7-Code": [ { "name": "Kimi-K2.7-Code", "huggingface_id": "moonshotai/Kimi-K2.7-Code", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct": [ { "name": "Hunyuan-A13B-Instruct", "huggingface_id": "tencent/Hunyuan-A13B-Instruct", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct-FP8": [ { "name": "Hunyuan-A13B-Instruct (fp8)", "huggingface_id": "tencent/Hunyuan-A13B-Instruct-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "tencent/Hy3-preview": [ { "name": "Hy3-preview", "huggingface_id": "tencent/Hy3-preview", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3": [ { "name": "Hy3", "huggingface_id": "tencent/Hy3", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-FP8": [ { "name": "Hy3 (fp8)", "huggingface_id": "tencent/Hy3-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-4.5": [ { "name": "GLM-4.5", "huggingface_id": "zai-org/GLM-4.5", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5-FP8": [ { "name": "GLM-4.5 (fp8)", "huggingface_id": "zai-org/GLM-4.5-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6": [ { "name": "GLM-4.6", "huggingface_id": "zai-org/GLM-4.6", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6-FP8": [ { "name": "GLM-4.6 (fp8)", "huggingface_id": "zai-org/GLM-4.6-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.7": [ { "name": "GLM-4.7", "huggingface_id": "zai-org/GLM-4.7", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.7-FP8": [ { "name": "GLM-4.7 (fp8)", "huggingface_id": "zai-org/GLM-4.7-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-4.7-NVFP4": [ { "name": "GLM-4.7 (nvfp4)", "huggingface_id": "nvidia/GLM-4.7-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2-FP8": [ { "name": "GLM-5.2", "huggingface_id": "zai-org/GLM-5.2-FP8", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/GLM-5.2-NVFP4": [ { "name": "GLM-5.2 (nvfp4)", "huggingface_id": "nvidia/GLM-5.2-NVFP4", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2": [ { "name": "GLM-5.2 (bf16)", "huggingface_id": "zai-org/GLM-5.2", "accelerators": "mi355x", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ] }, "trillium": { "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "Qwen/Qwen2.5-VL-7B-Instruct": [ { "name": "Qwen2.5-VL-7B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-7B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-7B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct-AWQ", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-32B": [ { "name": "Qwen3-32B", "huggingface_id": "Qwen/Qwen3-32B", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-32B-FP8": [ { "name": "Qwen3-32B (fp8)", "huggingface_id": "Qwen/Qwen3-32B-FP8", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-32B-AWQ": [ { "name": "Qwen3-32B (awq)", "huggingface_id": "Qwen/Qwen3-32B-AWQ", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-4B": [ { "name": "Qwen3-4B", "huggingface_id": "Qwen/Qwen3-4B", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-4B-FP8": [ { "name": "Qwen3-4B (fp8)", "huggingface_id": "Qwen/Qwen3-4B-FP8", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "trillium", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ] }, "ironwood": { "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "Qwen/Qwen3-32B": [ { "name": "Qwen3-32B", "huggingface_id": "Qwen/Qwen3-32B", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-32B-FP8": [ { "name": "Qwen3-32B (fp8)", "huggingface_id": "Qwen/Qwen3-32B-FP8", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-32B-AWQ": [ { "name": "Qwen3-32B (awq)", "huggingface_id": "Qwen/Qwen3-32B-AWQ", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct": [ { "name": "Qwen3-Coder-480B-A35B-Instruct", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "ironwood", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ] }, "xeon6": { "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_CPU_KVCACHE_SPACE": "40", "VLLM_CPU_ATTN_SPLIT_KV": "0" } } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "Qwen/Qwen3-4B": [ { "name": "Qwen3-4B", "huggingface_id": "Qwen/Qwen3-4B", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-4B-FP8": [ { "name": "Qwen3-4B (fp8)", "huggingface_id": "Qwen/Qwen3-4B-FP8", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-35B-A3B": [ { "name": "Qwen3.5-35B-A3B", "huggingface_id": "Qwen/Qwen3.5-35B-A3B", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-35B-A3B-FP8": [ { "name": "Qwen3.5-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-FP8", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4": [ { "name": "Qwen3.5-35B-A3B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-4B": [ { "name": "Qwen3.5-4B", "huggingface_id": "Qwen/Qwen3.5-4B", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "meta-llama/Llama-4-Scout-17B-16E-Instruct": [ { "name": "Llama-4-Scout", "huggingface_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} }, "balance": { "extra_args": { "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8": [ { "name": "Llama-4-Scout (fp8)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4": [ { "name": "Llama-4-Scout (nvfp4)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4", "accelerators": "xeon6", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ] }, "xeon5": {}, "rtxpro-6000-blackwell-server-edition": { "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "rtxpro-6000-blackwell-server-edition", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--gpu-memory-utilization": "0.8" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "", "--gpu-memory-utilization": "0.8" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "rtxpro-6000-blackwell-server-edition", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}", "--gpu-memory-utilization": "0.8" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "amd": { "ByteDance-Seed/Seed-OSS-36B-Instruct": [ { "name": "Seed-OSS-36B", "huggingface_id": "ByteDance-Seed/Seed-OSS-36B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "seed_oss" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "google/diffusiongemma-26B-A4B-it": [ { "name": "DiffusionGemma 26B-A4B IT", "huggingface_id": "google/diffusiongemma-26B-A4B-it", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic": [ { "name": "DiffusionGemma 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4": [ { "name": "DiffusionGemma 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-12B-it": [ { "name": "Gemma 4 12B IT", "huggingface_id": "google/gemma-4-12B-it", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "google/gemma-4-12B-it-qat-w4a16-ct": [ { "name": "Gemma 4 12B IT (w4a16)", "huggingface_id": "google/gemma-4-12B-it-qat-w4a16-ct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/translategemma-27b-it": [ { "name": "TranslateGemma 27B IT", "huggingface_id": "google/translategemma-27b-it", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Infomaniak-AI/vllm-translategemma-27b-it": [ { "name": "TranslateGemma 27B IT (vllm_optimized)", "huggingface_id": "Infomaniak-AI/vllm-translategemma-27b-it", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "vllm_optimized" } ], "google/translategemma-4b-it": [ { "name": "TranslateGemma 27B IT (small_4b)", "huggingface_id": "google/translategemma-4b-it", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "small_4b" } ], "JetBrains/Mellum2-12B-A2.5B-Instruct": [ { "name": "Mellum2-12B-A2.5B-Instruct", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "JetBrains/Mellum2-12B-A2.5B-Thinking": [ { "name": "Mellum2-12B-A2.5B-Thinking", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Thinking", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Base": [ { "name": "LFM2.5 1.2B Base", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Base", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Instruct": [ { "name": "LFM2.5 1.2B Instruct", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP-202606": [ { "name": "LFM2.5 1.2B JP (202606)", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP-202606", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP": [ { "name": "LFM2.5 1.2B JP", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Thinking": [ { "name": "LFM2.5 1.2B Thinking", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Thinking", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-230M": [ { "name": "LFM2.5 230M", "huggingface_id": "LiquidAI/LFM2.5-230M", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-350M": [ { "name": "LFM2.5 350M", "huggingface_id": "LiquidAI/LFM2.5-350M", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-8B-A1B": [ { "name": "LFM2.5 8B A1B", "huggingface_id": "LiquidAI/LFM2.5-8B-A1B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-1.6B": [ { "name": "LFM2.5 VL 1.6B", "huggingface_id": "LiquidAI/LFM2.5-VL-1.6B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-450M": [ { "name": "LFM2.5 VL 450M", "huggingface_id": "LiquidAI/LFM2.5-VL-450M", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.1": [ { "name": "MiniMax-M2.1", "huggingface_id": "MiniMaxAI/MiniMax-M2.1", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.5": [ { "name": "MiniMax-M2.5", "huggingface_id": "MiniMaxAI/MiniMax-M2.5", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.5-NVFP4": [ { "name": "MiniMax-M2.5 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.5-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2": [ { "name": "MiniMax-M2", "huggingface_id": "MiniMaxAI/MiniMax-M2", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/MiniMax-M2-NVFP4": [ { "name": "MiniMax-M2 (nvfp4)", "huggingface_id": "RedHatAI/MiniMax-M2-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--attention-backend": "ROCM_AITER_FA", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M3": [ { "name": "MiniMax-M3", "huggingface_id": "MiniMaxAI/MiniMax-M3", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M3-MXFP8": [ { "name": "MiniMax-M3 (mxfp8)", "huggingface_id": "MiniMaxAI/MiniMax-M3-MXFP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } } }, "default_strategy": "latency-focus", "variant_name": "mxfp8" } ], "nvidia/MiniMax-M3-NVFP4": [ { "name": "MiniMax-M3 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M3-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "amd/MiniMax-M3-MXFP4": [ { "name": "MiniMax-M3 (mxfp4)", "huggingface_id": "amd/MiniMax-M3-MXFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } }, "balance": { "extra_args": { "--block-size": "128", "--attention-backend": "TRITON_ATTN", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0" } } }, "default_strategy": "latency-focus", "variant_name": "mxfp4" } ], "OpenGVLab/InternVL3_5-8B": [ { "name": "InternVL3.5", "huggingface_id": "OpenGVLab/InternVL3_5-8B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-SoundEffect": [ { "name": "MOSS-SoundEffect", "huggingface_id": "OpenMOSS-Team/MOSS-SoundEffect", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_sound_effect.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTS-Realtime": [ { "name": "MOSS-TTS-Realtime", "huggingface_id": "OpenMOSS-Team/MOSS-TTS-Realtime", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTS": [ { "name": "MOSS-TTS", "huggingface_id": "OpenMOSS-Team/MOSS-TTS", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_tts.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTSD-v1.0": [ { "name": "MOSS-TTSD-v1.0", "huggingface_id": "OpenMOSS-Team/MOSS-TTSD-v1.0", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_ttsd.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-Transcribe-Diarize": [ { "name": "MOSS-Transcribe-Diarize", "huggingface_id": "OpenMOSS-Team/MOSS-Transcribe-Diarize", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-VoiceGenerator": [ { "name": "MOSS-VoiceGenerator", "huggingface_id": "OpenMOSS-Team/MOSS-VoiceGenerator", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_voice_generator.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL-1.5": [ { "name": "PaddleOCR-VL-1.5", "huggingface_id": "PaddlePaddle/PaddleOCR-VL-1.5", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL-1.6": [ { "name": "PaddleOCR-VL-1.6", "huggingface_id": "PaddlePaddle/PaddleOCR-VL-1.6", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL": [ { "name": "PaddleOCR-VL", "huggingface_id": "PaddlePaddle/PaddleOCR-VL", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen-Image": [ { "name": "Qwen-Image", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" }, { "name": "Qwen-Image (fp8)", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "fp8", "--ignored-layers": "img_mlp" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" }, { "name": "Qwen-Image (int8)", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "int8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "Qwen/Qwen2.5-VL-72B-Instruct": [ { "name": "Qwen2.5-VL-72B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--limit-mm-per-prompt": "{\"image\":2,\"video\":0}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-72B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct-AWQ", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--limit-mm-per-prompt": "{\"image\":2,\"video\":0}", "--quantization": "awq" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen2.5-VL-7B-Instruct": [ { "name": "Qwen2.5-VL-7B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-7B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-7B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct-AWQ", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-235B-A22B-Instruct-2507": [ { "name": "Qwen3-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-235B-A22B-FP8": [ { "name": "Qwen3-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-235B-A22B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4": [ { "name": "Qwen3-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-32B": [ { "name": "Qwen3-32B", "huggingface_id": "Qwen/Qwen3-32B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-32B-FP8": [ { "name": "Qwen3-32B (fp8)", "huggingface_id": "Qwen/Qwen3-32B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-32B-AWQ": [ { "name": "Qwen3-32B (awq)", "huggingface_id": "Qwen/Qwen3-32B-AWQ", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-4B": [ { "name": "Qwen3-4B", "huggingface_id": "Qwen/Qwen3-4B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-4B-FP8": [ { "name": "Qwen3-4B (fp8)", "huggingface_id": "Qwen/Qwen3-4B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-ASR-1.7B": [ { "name": "Qwen3-ASR-1.7B", "huggingface_id": "Qwen/Qwen3-ASR-1.7B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct": [ { "name": "Qwen3-Coder-480B-A35B-Instruct", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct": [ { "name": "Qwen3-Next-80B-A3B-Instruct", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8": [ { "name": "Qwen3-Next-80B-A3B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4": [ { "name": "Qwen3-Next-80B-A3B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (voicedesign-1.7b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "voicedesign-1.7b" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-Base": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (base-1.7b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base-1.7b" } ], "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (customvoice-0.6b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "customvoice-0.6b" } ], "Qwen/Qwen3-TTS-12Hz-0.6B-Base": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (base-0.6b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-0.6B-Base", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base-0.6b" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--async-scheduling": "", "--enable-chunked-prefill": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-0.8B": [ { "name": "Qwen3.5-0.8B", "huggingface_id": "Qwen/Qwen3.5-0.8B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-122B-A10B": [ { "name": "Qwen3.5-122B-A10B", "huggingface_id": "Qwen/Qwen3.5-122B-A10B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-122B-A10B-FP8": [ { "name": "Qwen3.5-122B-A10B (fp8)", "huggingface_id": "Qwen/Qwen3.5-122B-A10B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-122B-A10B-GPTQ-Int4": [ { "name": "Qwen3.5-122B-A10B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-122B-A10B-GPTQ-Int4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-27B": [ { "name": "Qwen3.5-27B", "huggingface_id": "Qwen/Qwen3.5-27B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-27B-FP8": [ { "name": "Qwen3.5-27B (fp8)", "huggingface_id": "Qwen/Qwen3.5-27B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-27B-GPTQ-Int4": [ { "name": "Qwen3.5-27B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-27B-GPTQ-Int4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-2B": [ { "name": "Qwen3.5-2B", "huggingface_id": "Qwen/Qwen3.5-2B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-35B-A3B": [ { "name": "Qwen3.5-35B-A3B", "huggingface_id": "Qwen/Qwen3.5-35B-A3B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-35B-A3B-FP8": [ { "name": "Qwen3.5-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4": [ { "name": "Qwen3.5-35B-A3B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-397B-A17B": [ { "name": "Qwen3.5-397B", "huggingface_id": "Qwen/Qwen3.5-397B-A17B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Qwen3.5-397B-A17B-NVFP4": [ { "name": "Qwen3.5-397B (nvfp4)", "huggingface_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4": [ { "name": "Qwen3.5-397B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-4B": [ { "name": "Qwen3.5-4B", "huggingface_id": "Qwen/Qwen3.5-4B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-9B": [ { "name": "Qwen3.5-9B", "huggingface_id": "Qwen/Qwen3.5-9B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-27B": [ { "name": "Qwen3.6-27B", "huggingface_id": "Qwen/Qwen3.6-27B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-27B-FP8": [ { "name": "Qwen3.6-27B (fp8)", "huggingface_id": "Qwen/Qwen3.6-27B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "512", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-27B-NVFP4": [ { "name": "Qwen3.6-27B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-27B-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3Guard-Gen-8B": [ { "name": "Qwen3Guard-Gen-8B", "huggingface_id": "Qwen/Qwen3Guard-Gen-8B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3Guard-Gen-4B": [ { "name": "Qwen3Guard-Gen-8B (small_4b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-4B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "small_4b" } ], "Qwen/Qwen3Guard-Gen-0.6B": [ { "name": "Qwen3Guard-Gen-8B (tiny_0_6b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-0.6B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "tiny_0_6b" } ], "Wan-AI/Wan2.2-T2V-A14B-Diffusers": [ { "name": "Wan2.2", "huggingface_id": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2-Flash": [ { "name": "MiMo-V2-Flash", "huggingface_id": "XiaomiMiMo/MiMo-V2-Flash", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5-Pro": [ { "name": "MiMo-V2.5-Pro", "huggingface_id": "XiaomiMiMo/MiMo-V2.5-Pro", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5": [ { "name": "MiMo-V2.5", "huggingface_id": "XiaomiMiMo/MiMo-V2.5", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "arcee-ai/Trinity-Large-Thinking": [ { "name": "Trinity-Large-Thinking", "huggingface_id": "arcee-ai/Trinity-Large-Thinking", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--dtype": "bfloat16", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--dtype": "bfloat16", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "arcee-ai/Trinity-Large-Thinking-NVFP4": [ { "name": "Trinity-Large-Thinking (nvfp4)", "huggingface_id": "arcee-ai/Trinity-Large-Thinking-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--dtype": "bfloat16", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--dtype": "bfloat16", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "baidu/ERNIE-4.5-21B-A3B-PT": [ { "name": "ERNIE-4.5", "huggingface_id": "baidu/ERNIE-4.5-21B-A3B-PT", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--gpu-memory-utilization": "0.9", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--gpu-memory-utilization": "0.9", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-300B-A47B-PT": [ { "name": "ERNIE-4.5 (300b)", "huggingface_id": "baidu/ERNIE-4.5-300B-A47B-PT", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--gpu-memory-utilization": "0.95", "--quantization": "fp8", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--gpu-memory-utilization": "0.95", "--quantization": "fp8", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "300b" } ], "baidu/ERNIE-4.5-VL-28B-A3B-PT": [ { "name": "ERNIE-4.5-VL", "huggingface_id": "baidu/ERNIE-4.5-VL-28B-A3B-PT", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-424B-A47B-PT": [ { "name": "ERNIE-4.5-VL (424b)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b" }, { "name": "ERNIE-4.5-VL (424b_fp8)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b_fp8" } ], "baidu/Unlimited-OCR": [ { "name": "Unlimited-OCR", "huggingface_id": "baidu/Unlimited-OCR", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.unlimited_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "bosonai/higgs-audio-v3-tts-4b": [ { "name": "Higgs Audio V3 TTS", "huggingface_id": "bosonai/higgs-audio-v3-tts-4b", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-OCR-2": [ { "name": "DeepSeek-OCR-2", "huggingface_id": "deepseek-ai/DeepSeek-OCR-2", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.deepseek_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-OCR": [ { "name": "DeepSeek-OCR", "huggingface_id": "deepseek-ai/DeepSeek-OCR", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.deepseek_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1": [ { "name": "DeepSeek-R1", "huggingface_id": "deepseek-ai/DeepSeek-R1", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default", "extra_envs": { "VLLM_RPC_TIMEOUT": "1800000" } } ], "deepseek-ai/DeepSeek-R1-0528": [ { "name": "DeepSeek-R1 (r1_0528)", "huggingface_id": "deepseek-ai/DeepSeek-R1-0528", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "r1_0528" } ], "nvidia/DeepSeek-R1-0528-NVFP4-v2": [ { "name": "DeepSeek-R1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-R1-0528-NVFP4-v2", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.1": [ { "name": "DeepSeek-V3.1", "huggingface_id": "deepseek-ai/DeepSeek-V3.1", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.1-NVFP4": [ { "name": "DeepSeek-V3.1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.1-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.2-Exp": [ { "name": "DeepSeek-V3.2-Exp", "huggingface_id": "deepseek-ai/DeepSeek-V3.2-Exp", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--block-size": "1", "--kv-cache-dtype": "bfloat16", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "32768", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_RPC_TIMEOUT": "18000000" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--block-size": "1", "--kv-cache-dtype": "bfloat16", "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "32768", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_RPC_TIMEOUT": "18000000" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-V3.2": [ { "name": "DeepSeek-V3.2", "huggingface_id": "deepseek-ai/DeepSeek-V3.2", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.2-NVFP4": [ { "name": "DeepSeek-V3.2 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.2-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3": [ { "name": "DeepSeek-V3", "huggingface_id": "deepseek-ai/DeepSeek-V3", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default", "extra_envs": { "VLLM_RPC_TIMEOUT": "1800000" } } ], "nvidia/DeepSeek-V3-FP4": [ { "name": "DeepSeek-V3 (fp4)", "huggingface_id": "nvidia/DeepSeek-V3-FP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "SAFETENSORS_FAST_GPU": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--distributed-executor-backend": "mp", "--gpu-memory-utilization": "0.9", "--max-num-seqs": "512", "--max-num-batched-tokens": "8192", "--compilation-config": "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "fishaudio/s2-pro": [ { "name": "Fish Speech S2 Pro", "huggingface_id": "fishaudio/s2-pro", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ling-2.6-1T": [ { "name": "Ling-2.6-1T", "huggingface_id": "inclusionAI/Ling-2.6-1T", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ling-2.6-flash": [ { "name": "Ling-2.6-flash", "huggingface_id": "inclusionAI/Ling-2.6-flash", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ming-omni-tts-0.5B": [ { "name": "Ming-omni-tts-0.5B", "huggingface_id": "inclusionAI/Ming-omni-tts-0.5B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--enforce-eager": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-1T-FP8": [ { "name": "Ring-1T-FP8", "huggingface_id": "inclusionAI/Ring-1T-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-2.6-1T": [ { "name": "Ring-2.6-1T", "huggingface_id": "inclusionAI/Ring-2.6-1T", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1": [ { "name": "Intern-S1", "huggingface_id": "internlm/Intern-S1", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1-FP8": [ { "name": "Intern-S1 (fp8)", "huggingface_id": "internlm/Intern-S1-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "internlm/Intern-S2-Preview": [ { "name": "Intern-S2-Preview", "huggingface_id": "internlm/Intern-S2-Preview", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S2-Preview-FP8": [ { "name": "Intern-S2-Preview (fp8)", "huggingface_id": "internlm/Intern-S2-Preview-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "jinaai/jina-embeddings-v5-text-small-retrieval": [ { "name": "Jina Embeddings v5 Text Small", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-retrieval", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "jinaai/jina-embeddings-v5-text-small-text-matching": [ { "name": "Jina Embeddings v5 Text Small (text_matching)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-text-matching", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "text_matching" } ], "jinaai/jina-embeddings-v5-text-small-classification": [ { "name": "Jina Embeddings v5 Text Small (classification)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-classification", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "classification" } ], "jinaai/jina-embeddings-v5-text-small-clustering": [ { "name": "Jina Embeddings v5 Text Small (clustering)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-clustering", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "clustering" } ], "jinaai/jina-reranker-m0": [ { "name": "Jina Reranker m0", "huggingface_id": "jinaai/jina-reranker-m0", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--gpu-memory-utilization": "0.75", "--max-num-seqs": "32" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "meituan-longcat/LongCat-Image-Edit": [ { "name": "LongCat-Image-Edit", "huggingface_id": "meituan-longcat/LongCat-Image-Edit", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "meta-llama/Llama-4-Scout-17B-16E-Instruct": [ { "name": "Llama-4-Scout", "huggingface_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "16384", "--max-num-seqs": "64" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "16384", "--max-num-seqs": "64", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8": [ { "name": "Llama-4-Scout (fp8)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "16384", "--max-num-seqs": "64", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "16384", "--max-num-seqs": "64", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4": [ { "name": "Llama-4-Scout (nvfp4)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "16384", "--max-num-seqs": "64", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--no-enable-prefix-caching": "", "--max-num-batched-tokens": "16384", "--max-num-seqs": "64", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "microsoft/Phi-4-mini-instruct": [ { "name": "Phi-4", "huggingface_id": "microsoft/Phi-4-mini-instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "microsoft/Phi-4-mini-reasoning": [ { "name": "Phi-4 (mini_reasoning)", "huggingface_id": "microsoft/Phi-4-mini-reasoning", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mini_reasoning" } ], "microsoft/Phi-4-reasoning": [ { "name": "Phi-4 (reasoning)", "huggingface_id": "microsoft/Phi-4-reasoning", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "reasoning" } ], "microsoft/Phi-4-multimodal-instruct": [ { "name": "Phi-4 (multimodal)", "huggingface_id": "microsoft/Phi-4-multimodal-instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "multimodal" } ], "mistralai/Ministral-3-14B-Instruct-2512": [ { "name": "Ministral-3-Instruct", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-8B-Instruct-2512": [ { "name": "Ministral-3-Instruct (8b)", "huggingface_id": "mistralai/Ministral-3-8B-Instruct-2512", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "8b" } ], "mistralai/Ministral-3-3B-Instruct-2512": [ { "name": "Ministral-3-Instruct (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Instruct-2512", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Instruct-2512-FP8": [ { "name": "Ministral-3-Instruct (fp8)", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Ministral-3-8B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning", "huggingface_id": "mistralai/Ministral-3-8B-Reasoning-2512", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-3B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Reasoning-2512", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (14b)", "huggingface_id": "mistralai/Ministral-3-14B-Reasoning-2512", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "14b" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512": [ { "name": "Mistral-Large-3-675B-Instruct", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4": [ { "name": "Mistral-Large-3-675B-Instruct (nvfp4)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8": [ { "name": "Mistral-Large-3-675B-Instruct (fp8)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Mistral-Medium-3.5-128B": [ { "name": "Mistral-Medium-3.5", "huggingface_id": "mistralai/Mistral-Medium-3.5-128B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"mistralai/Mistral-Medium-3.5-128B-EAGLE\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Small-4-119B-2603": [ { "name": "Mistral-Small-4-119B", "huggingface_id": "mistralai/Mistral-Small-4-119B-2603", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "FLASH_ATTN_MLA", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--attention-backend": "FLASH_ATTN_MLA", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Small-4-119B-2603-NVFP4": [ { "name": "Mistral-Small-4-119B (nvfp4)", "huggingface_id": "mistralai/Mistral-Small-4-119B-2603-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "TRITON_MLA", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--attention-backend": "TRITON_MLA", "--no-enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Voxtral-4B-TTS-2603": [ { "name": "Voxtral-4B-TTS-2603", "huggingface_id": "mistralai/Voxtral-4B-TTS-2603", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Voxtral-Mini-4B-Realtime-2602": [ { "name": "Voxtral-Mini-4B-Realtime-2602", "huggingface_id": "mistralai/Voxtral-Mini-4B-Realtime-2602", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer-mode": "mistral", "--compilation_config": "{\"cudagraph_mode\": \"PIECEWISE\"}" }, "extra_envs": { "VLLM_DISABLE_COMPILE_CACHE": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Instruct": [ { "name": "Kimi-K2-Instruct", "huggingface_id": "moonshotai/Kimi-K2-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": { "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Thinking": [ { "name": "Kimi-K2-Thinking", "huggingface_id": "moonshotai/Kimi-K2-Thinking", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2-Thinking-NVFP4": [ { "name": "Kimi-K2-Thinking (nvfp4)", "huggingface_id": "nvidia/Kimi-K2-Thinking-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.5": [ { "name": "Kimi-K2.5", "huggingface_id": "moonshotai/Kimi-K2.5", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.5-NVFP4": [ { "name": "Kimi-K2.5 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.5-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.7-Code": [ { "name": "Kimi-K2.7-Code", "huggingface_id": "moonshotai/Kimi-K2.7-Code", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-Linear-48B-A3B-Instruct": [ { "name": "Kimi-Linear-48B-A3B-Instruct", "huggingface_id": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Nano": [ { "name": "Cosmos3-Nano", "huggingface_id": "nvidia/Cosmos3-Nano", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Image2Video": [ { "name": "Cosmos3-Super-Image2Video", "huggingface_id": "nvidia/Cosmos3-Super-Image2Video", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Text2Image": [ { "name": "Cosmos3-Super-Text2Image", "huggingface_id": "nvidia/Cosmos3-Super-Text2Image", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super": [ { "name": "Cosmos3-Super", "huggingface_id": "nvidia/Cosmos3-Super", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-4B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-4B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (base_bf16)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base_bf16" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B (fp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2": [ { "name": "NVIDIA Nemotron-Nano-9B-v2", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (base)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (japanese)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "japanese" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (fp8)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (nvfp4)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } }, "balance": { "extra_args": { "--attention-backend": "ROCM_AITER_UNIFIED_ATTN", "-cc.pass_config.fuse_rope_kvcache=True": "", "-cc.use_inductor_graph_partition=True": "", "--gpu-memory-utilization": "0.95", "--block-size=64": "", "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "HSA_NO_SCRATCH_RECLAIM": "1", "AMDGCN_USE_BUFFER_OPS": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/MiniCPM-V-4.6": [ { "name": "MiniCPM-V 4.6", "huggingface_id": "openbmb/MiniCPM-V-4.6", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/MiniCPM5-1B": [ { "name": "MiniCPM5-1B", "huggingface_id": "openbmb/MiniCPM5-1B", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/VoxCPM2": [ { "name": "VoxCPM2", "huggingface_id": "openbmb/VoxCPM2", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "pfnet/plamo-2-translate": [ { "name": "PLaMo 2 Translate", "huggingface_id": "pfnet/plamo-2-translate", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "pfnet/plamo-3-nict-31b-base": [ { "name": "PLaMo 3 NICT 31B Base", "huggingface_id": "pfnet/plamo-3-nict-31b-base", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-M.1": [ { "name": "Laguna M.1", "huggingface_id": "poolside/Laguna-M.1", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-M.1-FP8": [ { "name": "Laguna M.1 (fp8)", "huggingface_id": "poolside/Laguna-M.1-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-M.1-NVFP4": [ { "name": "Laguna M.1 (nvfp4)", "huggingface_id": "poolside/Laguna-M.1-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS-2.1": [ { "name": "Laguna XS-2.1", "huggingface_id": "poolside/Laguna-XS-2.1", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS-2.1-FP8": [ { "name": "Laguna XS-2.1 (fp8)", "huggingface_id": "poolside/Laguna-XS-2.1-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS-2.1-NVFP4": [ { "name": "Laguna XS-2.1 (nvfp4)", "huggingface_id": "poolside/Laguna-XS-2.1-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS-2.1-INT4": [ { "name": "Laguna XS-2.1 (int4)", "huggingface_id": "poolside/Laguna-XS-2.1-INT4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "poolside/Laguna-XS.2": [ { "name": "Laguna XS.2", "huggingface_id": "poolside/Laguna-XS.2", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS.2-FP8": [ { "name": "Laguna XS.2 (fp8)", "huggingface_id": "poolside/Laguna-XS.2-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS.2-NVFP4": [ { "name": "Laguna XS.2 (nvfp4)", "huggingface_id": "poolside/Laguna-XS.2-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS.2-INT4": [ { "name": "Laguna XS.2 (int4)", "huggingface_id": "poolside/Laguna-XS.2-INT4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stabilityai/stable-audio-open-1.0": [ { "name": "Stable Audio Open", "huggingface_id": "stabilityai/stable-audio-open-1.0", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enforce-eager": "", "--gpu-memory-utilization": "0.9" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stabilityai/stable-diffusion-3.5-medium": [ { "name": "Stable Diffusion 3.5", "huggingface_id": "stabilityai/stable-diffusion-3.5-medium", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stabilityai/stable-diffusion-3.5-large": [ { "name": "Stable Diffusion 3.5 (large)", "huggingface_id": "stabilityai/stable-diffusion-3.5-large", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "large" } ], "stabilityai/stable-diffusion-3.5-large-turbo": [ { "name": "Stable Diffusion 3.5 (large_turbo)", "huggingface_id": "stabilityai/stable-diffusion-3.5-large-turbo", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "large_turbo" } ], "stepfun-ai/Step-3.5-Flash": [ { "name": "Step-3.5-Flash", "huggingface_id": "stepfun-ai/Step-3.5-Flash", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.5-Flash-FP8": [ { "name": "Step-3.5-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.5-Flash-INT4": [ { "name": "Step-3.5-Flash (int4)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stepfun-ai/Step-3.5-Flash-INT8": [ { "name": "Step-3.5-Flash (int8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "stepfun-ai/Step-3.7-Flash": [ { "name": "Step-3.7-Flash", "huggingface_id": "stepfun-ai/Step-3.7-Flash", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.7-Flash-FP8": [ { "name": "Step-3.7-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.7-Flash-NVFP4": [ { "name": "Step-3.7-Flash (nvfp4)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "tencent/Hunyuan-A13B-Instruct": [ { "name": "Hunyuan-A13B-Instruct", "huggingface_id": "tencent/Hunyuan-A13B-Instruct", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct-FP8": [ { "name": "Hunyuan-A13B-Instruct (fp8)", "huggingface_id": "tencent/Hunyuan-A13B-Instruct-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "tencent/HunyuanOCR": [ { "name": "HunyuanOCR", "huggingface_id": "tencent/HunyuanOCR", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-preview": [ { "name": "Hy3-preview", "huggingface_id": "tencent/Hy3-preview", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3": [ { "name": "Hy3", "huggingface_id": "tencent/Hy3", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-FP8": [ { "name": "Hy3 (fp8)", "huggingface_id": "tencent/Hy3-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-4.5": [ { "name": "GLM-4.5", "huggingface_id": "zai-org/GLM-4.5", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5-FP8": [ { "name": "GLM-4.5 (fp8)", "huggingface_id": "zai-org/GLM-4.5-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.5V": [ { "name": "GLM-4.5V", "huggingface_id": "zai-org/GLM-4.5V", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5V-FP8": [ { "name": "GLM-4.5V (fp8)", "huggingface_id": "zai-org/GLM-4.5V-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6": [ { "name": "GLM-4.6", "huggingface_id": "zai-org/GLM-4.6", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6-FP8": [ { "name": "GLM-4.6 (fp8)", "huggingface_id": "zai-org/GLM-4.6-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6V": [ { "name": "GLM-4.6V", "huggingface_id": "zai-org/GLM-4.6V", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6V-FP8": [ { "name": "GLM-4.6V (fp8)", "huggingface_id": "zai-org/GLM-4.6V-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.7": [ { "name": "GLM-4.7", "huggingface_id": "zai-org/GLM-4.7", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.7-FP8": [ { "name": "GLM-4.7 (fp8)", "huggingface_id": "zai-org/GLM-4.7-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-4.7-NVFP4": [ { "name": "GLM-4.7 (nvfp4)", "huggingface_id": "nvidia/GLM-4.7-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.1": [ { "name": "GLM-5.1", "huggingface_id": "zai-org/GLM-5.1", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5.1-FP8": [ { "name": "GLM-5.1 (fp8)", "huggingface_id": "zai-org/GLM-5.1-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5.1-NVFP4": [ { "name": "GLM-5.1 (nvfp4)", "huggingface_id": "nvidia/GLM-5.1-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2-FP8": [ { "name": "GLM-5.2", "huggingface_id": "zai-org/GLM-5.2-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/GLM-5.2-NVFP4": [ { "name": "GLM-5.2 (nvfp4)", "huggingface_id": "nvidia/GLM-5.2-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2": [ { "name": "GLM-5.2 (bf16)", "huggingface_id": "zai-org/GLM-5.2", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--linear-backend": "aiter", "--moe-backend": "aiter", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-5": [ { "name": "GLM-5", "huggingface_id": "zai-org/GLM-5", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5-FP8": [ { "name": "GLM-5 (fp8)", "huggingface_id": "zai-org/GLM-5-FP8", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5-NVFP4": [ { "name": "GLM-5 (nvfp4)", "huggingface_id": "nvidia/GLM-5-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-ASR-Nano-2512": [ { "name": "GLM-ASR-Nano-2512", "huggingface_id": "zai-org/GLM-ASR-Nano-2512", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-GA": [ { "name": "GLM-GA", "huggingface_id": "zai-org/GLM-GA", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--allowed-local-media-path": "/", "--mm-processor-cache-type": "shm", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "glm4_6v" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-Image": [ { "name": "GLM-Image", "huggingface_id": "zai-org/GLM-Image", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-OCR": [ { "name": "GLM-OCR", "huggingface_id": "zai-org/GLM-OCR", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-TTS": [ { "name": "GLM-TTS", "huggingface_id": "zai-org/GLM-TTS", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/Glyph": [ { "name": "Glyph", "huggingface_id": "zai-org/Glyph", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--limit-mm-per-prompt.video": "0", "--reasoning-parser": "glm45", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "amd", "provider": "vllm_amd", "custom_image": "vllm/vllm-openai-rocm:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "google": { "ByteDance-Seed/Seed-OSS-36B-Instruct": [ { "name": "Seed-OSS-36B", "huggingface_id": "ByteDance-Seed/Seed-OSS-36B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "seed_oss" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "google/diffusiongemma-26B-A4B-it": [ { "name": "DiffusionGemma 26B-A4B IT", "huggingface_id": "google/diffusiongemma-26B-A4B-it", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic": [ { "name": "DiffusionGemma 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4": [ { "name": "DiffusionGemma 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-12B-it": [ { "name": "Gemma 4 12B IT", "huggingface_id": "google/gemma-4-12B-it", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "google/gemma-4-12B-it-qat-w4a16-ct": [ { "name": "Gemma 4 12B IT (w4a16)", "huggingface_id": "google/gemma-4-12B-it-qat-w4a16-ct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/translategemma-27b-it": [ { "name": "TranslateGemma 27B IT", "huggingface_id": "google/translategemma-27b-it", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Infomaniak-AI/vllm-translategemma-27b-it": [ { "name": "TranslateGemma 27B IT (vllm_optimized)", "huggingface_id": "Infomaniak-AI/vllm-translategemma-27b-it", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "vllm_optimized" } ], "google/translategemma-4b-it": [ { "name": "TranslateGemma 27B IT (small_4b)", "huggingface_id": "google/translategemma-4b-it", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "small_4b" } ], "JetBrains/Mellum2-12B-A2.5B-Instruct": [ { "name": "Mellum2-12B-A2.5B-Instruct", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "JetBrains/Mellum2-12B-A2.5B-Thinking": [ { "name": "Mellum2-12B-A2.5B-Thinking", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Thinking", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Base": [ { "name": "LFM2.5 1.2B Base", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Base", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Instruct": [ { "name": "LFM2.5 1.2B Instruct", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP-202606": [ { "name": "LFM2.5 1.2B JP (202606)", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP-202606", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP": [ { "name": "LFM2.5 1.2B JP", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Thinking": [ { "name": "LFM2.5 1.2B Thinking", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Thinking", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-230M": [ { "name": "LFM2.5 230M", "huggingface_id": "LiquidAI/LFM2.5-230M", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-350M": [ { "name": "LFM2.5 350M", "huggingface_id": "LiquidAI/LFM2.5-350M", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-8B-A1B": [ { "name": "LFM2.5 8B A1B", "huggingface_id": "LiquidAI/LFM2.5-8B-A1B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-1.6B": [ { "name": "LFM2.5 VL 1.6B", "huggingface_id": "LiquidAI/LFM2.5-VL-1.6B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-450M": [ { "name": "LFM2.5 VL 450M", "huggingface_id": "LiquidAI/LFM2.5-VL-450M", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.1": [ { "name": "MiniMax-M2.1", "huggingface_id": "MiniMaxAI/MiniMax-M2.1", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.5": [ { "name": "MiniMax-M2.5", "huggingface_id": "MiniMaxAI/MiniMax-M2.5", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.5-NVFP4": [ { "name": "MiniMax-M2.5 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.5-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2": [ { "name": "MiniMax-M2", "huggingface_id": "MiniMaxAI/MiniMax-M2", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/MiniMax-M2-NVFP4": [ { "name": "MiniMax-M2 (nvfp4)", "huggingface_id": "RedHatAI/MiniMax-M2-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M3": [ { "name": "MiniMax-M3", "huggingface_id": "MiniMaxAI/MiniMax-M3", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M3-MXFP8": [ { "name": "MiniMax-M3 (mxfp8)", "huggingface_id": "MiniMaxAI/MiniMax-M3-MXFP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp8" } ], "nvidia/MiniMax-M3-NVFP4": [ { "name": "MiniMax-M3 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M3-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "amd/MiniMax-M3-MXFP4": [ { "name": "MiniMax-M3 (mxfp4)", "huggingface_id": "amd/MiniMax-M3-MXFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp4" } ], "OpenGVLab/InternVL3_5-8B": [ { "name": "InternVL3.5", "huggingface_id": "OpenGVLab/InternVL3_5-8B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-SoundEffect": [ { "name": "MOSS-SoundEffect", "huggingface_id": "OpenMOSS-Team/MOSS-SoundEffect", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_sound_effect.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTS-Realtime": [ { "name": "MOSS-TTS-Realtime", "huggingface_id": "OpenMOSS-Team/MOSS-TTS-Realtime", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTS": [ { "name": "MOSS-TTS", "huggingface_id": "OpenMOSS-Team/MOSS-TTS", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_tts.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTSD-v1.0": [ { "name": "MOSS-TTSD-v1.0", "huggingface_id": "OpenMOSS-Team/MOSS-TTSD-v1.0", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_ttsd.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-Transcribe-Diarize": [ { "name": "MOSS-Transcribe-Diarize", "huggingface_id": "OpenMOSS-Team/MOSS-Transcribe-Diarize", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-VoiceGenerator": [ { "name": "MOSS-VoiceGenerator", "huggingface_id": "OpenMOSS-Team/MOSS-VoiceGenerator", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_voice_generator.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL-1.5": [ { "name": "PaddleOCR-VL-1.5", "huggingface_id": "PaddlePaddle/PaddleOCR-VL-1.5", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL-1.6": [ { "name": "PaddleOCR-VL-1.6", "huggingface_id": "PaddlePaddle/PaddleOCR-VL-1.6", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL": [ { "name": "PaddleOCR-VL", "huggingface_id": "PaddlePaddle/PaddleOCR-VL", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen-Image": [ { "name": "Qwen-Image", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" }, { "name": "Qwen-Image (fp8)", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "fp8", "--ignored-layers": "img_mlp" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" }, { "name": "Qwen-Image (int8)", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "int8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "Qwen/Qwen2.5-VL-72B-Instruct": [ { "name": "Qwen2.5-VL-72B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-72B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct-AWQ", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen2.5-VL-7B-Instruct": [ { "name": "Qwen2.5-VL-7B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-7B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-7B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct-AWQ", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-235B-A22B-Instruct-2507": [ { "name": "Qwen3-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-235B-A22B-FP8": [ { "name": "Qwen3-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-235B-A22B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4": [ { "name": "Qwen3-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-32B": [ { "name": "Qwen3-32B", "huggingface_id": "Qwen/Qwen3-32B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-32B-FP8": [ { "name": "Qwen3-32B (fp8)", "huggingface_id": "Qwen/Qwen3-32B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-32B-AWQ": [ { "name": "Qwen3-32B (awq)", "huggingface_id": "Qwen/Qwen3-32B-AWQ", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-4B": [ { "name": "Qwen3-4B", "huggingface_id": "Qwen/Qwen3-4B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-4B-FP8": [ { "name": "Qwen3-4B (fp8)", "huggingface_id": "Qwen/Qwen3-4B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-ASR-1.7B": [ { "name": "Qwen3-ASR-1.7B", "huggingface_id": "Qwen/Qwen3-ASR-1.7B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct": [ { "name": "Qwen3-Coder-480B-A35B-Instruct", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct": [ { "name": "Qwen3-Next-80B-A3B-Instruct", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8": [ { "name": "Qwen3-Next-80B-A3B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4": [ { "name": "Qwen3-Next-80B-A3B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (voicedesign-1.7b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "voicedesign-1.7b" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-Base": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (base-1.7b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base-1.7b" } ], "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (customvoice-0.6b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "customvoice-0.6b" } ], "Qwen/Qwen3-TTS-12Hz-0.6B-Base": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (base-0.6b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-0.6B-Base", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base-0.6b" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-0.8B": [ { "name": "Qwen3.5-0.8B", "huggingface_id": "Qwen/Qwen3.5-0.8B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-122B-A10B": [ { "name": "Qwen3.5-122B-A10B", "huggingface_id": "Qwen/Qwen3.5-122B-A10B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-122B-A10B-FP8": [ { "name": "Qwen3.5-122B-A10B (fp8)", "huggingface_id": "Qwen/Qwen3.5-122B-A10B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-122B-A10B-GPTQ-Int4": [ { "name": "Qwen3.5-122B-A10B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-122B-A10B-GPTQ-Int4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-27B": [ { "name": "Qwen3.5-27B", "huggingface_id": "Qwen/Qwen3.5-27B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-27B-FP8": [ { "name": "Qwen3.5-27B (fp8)", "huggingface_id": "Qwen/Qwen3.5-27B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-27B-GPTQ-Int4": [ { "name": "Qwen3.5-27B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-27B-GPTQ-Int4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-2B": [ { "name": "Qwen3.5-2B", "huggingface_id": "Qwen/Qwen3.5-2B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-35B-A3B": [ { "name": "Qwen3.5-35B-A3B", "huggingface_id": "Qwen/Qwen3.5-35B-A3B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-35B-A3B-FP8": [ { "name": "Qwen3.5-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4": [ { "name": "Qwen3.5-35B-A3B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-397B-A17B": [ { "name": "Qwen3.5-397B", "huggingface_id": "Qwen/Qwen3.5-397B-A17B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Qwen3.5-397B-A17B-NVFP4": [ { "name": "Qwen3.5-397B (nvfp4)", "huggingface_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4": [ { "name": "Qwen3.5-397B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-4B": [ { "name": "Qwen3.5-4B", "huggingface_id": "Qwen/Qwen3.5-4B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-9B": [ { "name": "Qwen3.5-9B", "huggingface_id": "Qwen/Qwen3.5-9B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-27B": [ { "name": "Qwen3.6-27B", "huggingface_id": "Qwen/Qwen3.6-27B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-27B-FP8": [ { "name": "Qwen3.6-27B (fp8)", "huggingface_id": "Qwen/Qwen3.6-27B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "512", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-27B-NVFP4": [ { "name": "Qwen3.6-27B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-27B-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3Guard-Gen-8B": [ { "name": "Qwen3Guard-Gen-8B", "huggingface_id": "Qwen/Qwen3Guard-Gen-8B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3Guard-Gen-4B": [ { "name": "Qwen3Guard-Gen-8B (small_4b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-4B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "small_4b" } ], "Qwen/Qwen3Guard-Gen-0.6B": [ { "name": "Qwen3Guard-Gen-8B (tiny_0_6b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-0.6B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "tiny_0_6b" } ], "Wan-AI/Wan2.2-T2V-A14B-Diffusers": [ { "name": "Wan2.2", "huggingface_id": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} }, "balance": { "extra_args": { "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2-Flash": [ { "name": "MiMo-V2-Flash", "huggingface_id": "XiaomiMiMo/MiMo-V2-Flash", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5-Pro": [ { "name": "MiMo-V2.5-Pro", "huggingface_id": "XiaomiMiMo/MiMo-V2.5-Pro", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5": [ { "name": "MiMo-V2.5", "huggingface_id": "XiaomiMiMo/MiMo-V2.5", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "arcee-ai/Trinity-Large-Thinking": [ { "name": "Trinity-Large-Thinking", "huggingface_id": "arcee-ai/Trinity-Large-Thinking", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--dtype": "bfloat16", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--dtype": "bfloat16", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "arcee-ai/Trinity-Large-Thinking-NVFP4": [ { "name": "Trinity-Large-Thinking (nvfp4)", "huggingface_id": "arcee-ai/Trinity-Large-Thinking-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--dtype": "bfloat16", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--dtype": "bfloat16", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "baidu/ERNIE-4.5-21B-A3B-PT": [ { "name": "ERNIE-4.5", "huggingface_id": "baidu/ERNIE-4.5-21B-A3B-PT", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-300B-A47B-PT": [ { "name": "ERNIE-4.5 (300b)", "huggingface_id": "baidu/ERNIE-4.5-300B-A47B-PT", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--gpu-memory-utilization": "0.95", "--quantization": "fp8", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--gpu-memory-utilization": "0.95", "--quantization": "fp8", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "300b" } ], "baidu/ERNIE-4.5-VL-28B-A3B-PT": [ { "name": "ERNIE-4.5-VL", "huggingface_id": "baidu/ERNIE-4.5-VL-28B-A3B-PT", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-424B-A47B-PT": [ { "name": "ERNIE-4.5-VL (424b)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b" }, { "name": "ERNIE-4.5-VL (424b_fp8)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b_fp8" } ], "baidu/Unlimited-OCR": [ { "name": "Unlimited-OCR", "huggingface_id": "baidu/Unlimited-OCR", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.unlimited_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "bosonai/higgs-audio-v3-tts-4b": [ { "name": "Higgs Audio V3 TTS", "huggingface_id": "bosonai/higgs-audio-v3-tts-4b", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-OCR-2": [ { "name": "DeepSeek-OCR-2", "huggingface_id": "deepseek-ai/DeepSeek-OCR-2", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.deepseek_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-OCR": [ { "name": "DeepSeek-OCR", "huggingface_id": "deepseek-ai/DeepSeek-OCR", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.deepseek_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1": [ { "name": "DeepSeek-R1", "huggingface_id": "deepseek-ai/DeepSeek-R1", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1-0528": [ { "name": "DeepSeek-R1 (r1_0528)", "huggingface_id": "deepseek-ai/DeepSeek-R1-0528", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "r1_0528" } ], "nvidia/DeepSeek-R1-0528-NVFP4-v2": [ { "name": "DeepSeek-R1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-R1-0528-NVFP4-v2", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.1": [ { "name": "DeepSeek-V3.1", "huggingface_id": "deepseek-ai/DeepSeek-V3.1", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.1-NVFP4": [ { "name": "DeepSeek-V3.1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.1-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.2-Exp": [ { "name": "DeepSeek-V3.2-Exp", "huggingface_id": "deepseek-ai/DeepSeek-V3.2-Exp", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-V3.2": [ { "name": "DeepSeek-V3.2", "huggingface_id": "deepseek-ai/DeepSeek-V3.2", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.2-NVFP4": [ { "name": "DeepSeek-V3.2 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.2-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3": [ { "name": "DeepSeek-V3", "huggingface_id": "deepseek-ai/DeepSeek-V3", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3-FP4": [ { "name": "DeepSeek-V3 (fp4)", "huggingface_id": "nvidia/DeepSeek-V3-FP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "fishaudio/s2-pro": [ { "name": "Fish Speech S2 Pro", "huggingface_id": "fishaudio/s2-pro", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ling-2.6-1T": [ { "name": "Ling-2.6-1T", "huggingface_id": "inclusionAI/Ling-2.6-1T", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ling-2.6-flash": [ { "name": "Ling-2.6-flash", "huggingface_id": "inclusionAI/Ling-2.6-flash", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ming-omni-tts-0.5B": [ { "name": "Ming-omni-tts-0.5B", "huggingface_id": "inclusionAI/Ming-omni-tts-0.5B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--enforce-eager": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-1T-FP8": [ { "name": "Ring-1T-FP8", "huggingface_id": "inclusionAI/Ring-1T-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-2.6-1T": [ { "name": "Ring-2.6-1T", "huggingface_id": "inclusionAI/Ring-2.6-1T", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1": [ { "name": "Intern-S1", "huggingface_id": "internlm/Intern-S1", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1-FP8": [ { "name": "Intern-S1 (fp8)", "huggingface_id": "internlm/Intern-S1-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "internlm/Intern-S2-Preview": [ { "name": "Intern-S2-Preview", "huggingface_id": "internlm/Intern-S2-Preview", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S2-Preview-FP8": [ { "name": "Intern-S2-Preview (fp8)", "huggingface_id": "internlm/Intern-S2-Preview-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "jinaai/jina-embeddings-v5-text-small-retrieval": [ { "name": "Jina Embeddings v5 Text Small", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-retrieval", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "jinaai/jina-embeddings-v5-text-small-text-matching": [ { "name": "Jina Embeddings v5 Text Small (text_matching)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-text-matching", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "text_matching" } ], "jinaai/jina-embeddings-v5-text-small-classification": [ { "name": "Jina Embeddings v5 Text Small (classification)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-classification", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "classification" } ], "jinaai/jina-embeddings-v5-text-small-clustering": [ { "name": "Jina Embeddings v5 Text Small (clustering)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-clustering", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "clustering" } ], "jinaai/jina-reranker-m0": [ { "name": "Jina Reranker m0", "huggingface_id": "jinaai/jina-reranker-m0", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--gpu-memory-utilization": "0.75", "--max-num-seqs": "32" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "meituan-longcat/LongCat-Image-Edit": [ { "name": "LongCat-Image-Edit", "huggingface_id": "meituan-longcat/LongCat-Image-Edit", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "meta-llama/Llama-4-Scout-17B-16E-Instruct": [ { "name": "Llama-4-Scout", "huggingface_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} }, "balance": { "extra_args": { "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8": [ { "name": "Llama-4-Scout (fp8)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4": [ { "name": "Llama-4-Scout (nvfp4)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "microsoft/Phi-4-mini-instruct": [ { "name": "Phi-4", "huggingface_id": "microsoft/Phi-4-mini-instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "microsoft/Phi-4-mini-reasoning": [ { "name": "Phi-4 (mini_reasoning)", "huggingface_id": "microsoft/Phi-4-mini-reasoning", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mini_reasoning" } ], "microsoft/Phi-4-reasoning": [ { "name": "Phi-4 (reasoning)", "huggingface_id": "microsoft/Phi-4-reasoning", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "reasoning" } ], "microsoft/Phi-4-multimodal-instruct": [ { "name": "Phi-4 (multimodal)", "huggingface_id": "microsoft/Phi-4-multimodal-instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "multimodal" } ], "mistralai/Ministral-3-14B-Instruct-2512": [ { "name": "Ministral-3-Instruct", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-8B-Instruct-2512": [ { "name": "Ministral-3-Instruct (8b)", "huggingface_id": "mistralai/Ministral-3-8B-Instruct-2512", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "8b" } ], "mistralai/Ministral-3-3B-Instruct-2512": [ { "name": "Ministral-3-Instruct (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Instruct-2512", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Instruct-2512-FP8": [ { "name": "Ministral-3-Instruct (fp8)", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Ministral-3-8B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning", "huggingface_id": "mistralai/Ministral-3-8B-Reasoning-2512", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-3B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Reasoning-2512", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (14b)", "huggingface_id": "mistralai/Ministral-3-14B-Reasoning-2512", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "14b" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512": [ { "name": "Mistral-Large-3-675B-Instruct", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4": [ { "name": "Mistral-Large-3-675B-Instruct (nvfp4)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8": [ { "name": "Mistral-Large-3-675B-Instruct (fp8)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Mistral-Medium-3.5-128B": [ { "name": "Mistral-Medium-3.5", "huggingface_id": "mistralai/Mistral-Medium-3.5-128B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"mistralai/Mistral-Medium-3.5-128B-EAGLE\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Small-4-119B-2603": [ { "name": "Mistral-Small-4-119B", "huggingface_id": "mistralai/Mistral-Small-4-119B-2603", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "FLASH_ATTN_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--attention-backend": "FLASH_ATTN_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Small-4-119B-2603-NVFP4": [ { "name": "Mistral-Small-4-119B (nvfp4)", "huggingface_id": "mistralai/Mistral-Small-4-119B-2603-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "TRITON_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--attention-backend": "TRITON_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Voxtral-4B-TTS-2603": [ { "name": "Voxtral-4B-TTS-2603", "huggingface_id": "mistralai/Voxtral-4B-TTS-2603", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Voxtral-Mini-4B-Realtime-2602": [ { "name": "Voxtral-Mini-4B-Realtime-2602", "huggingface_id": "mistralai/Voxtral-Mini-4B-Realtime-2602", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer-mode": "mistral", "--compilation_config": "{\"cudagraph_mode\": \"PIECEWISE\"}" }, "extra_envs": { "VLLM_DISABLE_COMPILE_CACHE": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Instruct": [ { "name": "Kimi-K2-Instruct", "huggingface_id": "moonshotai/Kimi-K2-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Thinking": [ { "name": "Kimi-K2-Thinking", "huggingface_id": "moonshotai/Kimi-K2-Thinking", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2-Thinking-NVFP4": [ { "name": "Kimi-K2-Thinking (nvfp4)", "huggingface_id": "nvidia/Kimi-K2-Thinking-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.5": [ { "name": "Kimi-K2.5", "huggingface_id": "moonshotai/Kimi-K2.5", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.5-NVFP4": [ { "name": "Kimi-K2.5 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.5-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.7-Code": [ { "name": "Kimi-K2.7-Code", "huggingface_id": "moonshotai/Kimi-K2.7-Code", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-Linear-48B-A3B-Instruct": [ { "name": "Kimi-Linear-48B-A3B-Instruct", "huggingface_id": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Nano": [ { "name": "Cosmos3-Nano", "huggingface_id": "nvidia/Cosmos3-Nano", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Image2Video": [ { "name": "Cosmos3-Super-Image2Video", "huggingface_id": "nvidia/Cosmos3-Super-Image2Video", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Text2Image": [ { "name": "Cosmos3-Super-Text2Image", "huggingface_id": "nvidia/Cosmos3-Super-Text2Image", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super": [ { "name": "Cosmos3-Super", "huggingface_id": "nvidia/Cosmos3-Super", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-4B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-4B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (base_bf16)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base_bf16" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B (fp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2": [ { "name": "NVIDIA Nemotron-Nano-9B-v2", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (base)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (japanese)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "japanese" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (fp8)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (nvfp4)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/MiniCPM-V-4.6": [ { "name": "MiniCPM-V 4.6", "huggingface_id": "openbmb/MiniCPM-V-4.6", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/MiniCPM5-1B": [ { "name": "MiniCPM5-1B", "huggingface_id": "openbmb/MiniCPM5-1B", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/VoxCPM2": [ { "name": "VoxCPM2", "huggingface_id": "openbmb/VoxCPM2", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "pfnet/plamo-2-translate": [ { "name": "PLaMo 2 Translate", "huggingface_id": "pfnet/plamo-2-translate", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "pfnet/plamo-3-nict-31b-base": [ { "name": "PLaMo 3 NICT 31B Base", "huggingface_id": "pfnet/plamo-3-nict-31b-base", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-M.1": [ { "name": "Laguna M.1", "huggingface_id": "poolside/Laguna-M.1", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-M.1-FP8": [ { "name": "Laguna M.1 (fp8)", "huggingface_id": "poolside/Laguna-M.1-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-M.1-NVFP4": [ { "name": "Laguna M.1 (nvfp4)", "huggingface_id": "poolside/Laguna-M.1-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS-2.1": [ { "name": "Laguna XS-2.1", "huggingface_id": "poolside/Laguna-XS-2.1", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS-2.1-FP8": [ { "name": "Laguna XS-2.1 (fp8)", "huggingface_id": "poolside/Laguna-XS-2.1-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS-2.1-NVFP4": [ { "name": "Laguna XS-2.1 (nvfp4)", "huggingface_id": "poolside/Laguna-XS-2.1-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS-2.1-INT4": [ { "name": "Laguna XS-2.1 (int4)", "huggingface_id": "poolside/Laguna-XS-2.1-INT4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "poolside/Laguna-XS.2": [ { "name": "Laguna XS.2", "huggingface_id": "poolside/Laguna-XS.2", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS.2-FP8": [ { "name": "Laguna XS.2 (fp8)", "huggingface_id": "poolside/Laguna-XS.2-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS.2-NVFP4": [ { "name": "Laguna XS.2 (nvfp4)", "huggingface_id": "poolside/Laguna-XS.2-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS.2-INT4": [ { "name": "Laguna XS.2 (int4)", "huggingface_id": "poolside/Laguna-XS.2-INT4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stabilityai/stable-audio-open-1.0": [ { "name": "Stable Audio Open", "huggingface_id": "stabilityai/stable-audio-open-1.0", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enforce-eager": "", "--gpu-memory-utilization": "0.9" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stabilityai/stable-diffusion-3.5-medium": [ { "name": "Stable Diffusion 3.5", "huggingface_id": "stabilityai/stable-diffusion-3.5-medium", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stabilityai/stable-diffusion-3.5-large": [ { "name": "Stable Diffusion 3.5 (large)", "huggingface_id": "stabilityai/stable-diffusion-3.5-large", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "large" } ], "stabilityai/stable-diffusion-3.5-large-turbo": [ { "name": "Stable Diffusion 3.5 (large_turbo)", "huggingface_id": "stabilityai/stable-diffusion-3.5-large-turbo", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "large_turbo" } ], "stepfun-ai/Step-3.5-Flash": [ { "name": "Step-3.5-Flash", "huggingface_id": "stepfun-ai/Step-3.5-Flash", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.5-Flash-FP8": [ { "name": "Step-3.5-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.5-Flash-INT4": [ { "name": "Step-3.5-Flash (int4)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stepfun-ai/Step-3.5-Flash-INT8": [ { "name": "Step-3.5-Flash (int8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "stepfun-ai/Step-3.7-Flash": [ { "name": "Step-3.7-Flash", "huggingface_id": "stepfun-ai/Step-3.7-Flash", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.7-Flash-FP8": [ { "name": "Step-3.7-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.7-Flash-NVFP4": [ { "name": "Step-3.7-Flash (nvfp4)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "tencent/Hunyuan-A13B-Instruct": [ { "name": "Hunyuan-A13B-Instruct", "huggingface_id": "tencent/Hunyuan-A13B-Instruct", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct-FP8": [ { "name": "Hunyuan-A13B-Instruct (fp8)", "huggingface_id": "tencent/Hunyuan-A13B-Instruct-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "tencent/HunyuanOCR": [ { "name": "HunyuanOCR", "huggingface_id": "tencent/HunyuanOCR", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-preview": [ { "name": "Hy3-preview", "huggingface_id": "tencent/Hy3-preview", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3": [ { "name": "Hy3", "huggingface_id": "tencent/Hy3", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-FP8": [ { "name": "Hy3 (fp8)", "huggingface_id": "tencent/Hy3-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-4.5": [ { "name": "GLM-4.5", "huggingface_id": "zai-org/GLM-4.5", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5-FP8": [ { "name": "GLM-4.5 (fp8)", "huggingface_id": "zai-org/GLM-4.5-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.5V": [ { "name": "GLM-4.5V", "huggingface_id": "zai-org/GLM-4.5V", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5V-FP8": [ { "name": "GLM-4.5V (fp8)", "huggingface_id": "zai-org/GLM-4.5V-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6": [ { "name": "GLM-4.6", "huggingface_id": "zai-org/GLM-4.6", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6-FP8": [ { "name": "GLM-4.6 (fp8)", "huggingface_id": "zai-org/GLM-4.6-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6V": [ { "name": "GLM-4.6V", "huggingface_id": "zai-org/GLM-4.6V", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6V-FP8": [ { "name": "GLM-4.6V (fp8)", "huggingface_id": "zai-org/GLM-4.6V-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.7": [ { "name": "GLM-4.7", "huggingface_id": "zai-org/GLM-4.7", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.7-FP8": [ { "name": "GLM-4.7 (fp8)", "huggingface_id": "zai-org/GLM-4.7-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-4.7-NVFP4": [ { "name": "GLM-4.7 (nvfp4)", "huggingface_id": "nvidia/GLM-4.7-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.1": [ { "name": "GLM-5.1", "huggingface_id": "zai-org/GLM-5.1", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5.1-FP8": [ { "name": "GLM-5.1 (fp8)", "huggingface_id": "zai-org/GLM-5.1-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5.1-NVFP4": [ { "name": "GLM-5.1 (nvfp4)", "huggingface_id": "nvidia/GLM-5.1-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2-FP8": [ { "name": "GLM-5.2", "huggingface_id": "zai-org/GLM-5.2-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/GLM-5.2-NVFP4": [ { "name": "GLM-5.2 (nvfp4)", "huggingface_id": "nvidia/GLM-5.2-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2": [ { "name": "GLM-5.2 (bf16)", "huggingface_id": "zai-org/GLM-5.2", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-5": [ { "name": "GLM-5", "huggingface_id": "zai-org/GLM-5", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5-FP8": [ { "name": "GLM-5 (fp8)", "huggingface_id": "zai-org/GLM-5-FP8", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5-NVFP4": [ { "name": "GLM-5 (nvfp4)", "huggingface_id": "nvidia/GLM-5-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-ASR-Nano-2512": [ { "name": "GLM-ASR-Nano-2512", "huggingface_id": "zai-org/GLM-ASR-Nano-2512", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-GA": [ { "name": "GLM-GA", "huggingface_id": "zai-org/GLM-GA", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--allowed-local-media-path": "/", "--mm-processor-cache-type": "shm", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "glm4_6v" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-Image": [ { "name": "GLM-Image", "huggingface_id": "zai-org/GLM-Image", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-OCR": [ { "name": "GLM-OCR", "huggingface_id": "zai-org/GLM-OCR", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-TTS": [ { "name": "GLM-TTS", "huggingface_id": "zai-org/GLM-TTS", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/Glyph": [ { "name": "Glyph", "huggingface_id": "zai-org/Glyph", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--limit-mm-per-prompt.video": "0", "--reasoning-parser": "glm45", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "google", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "intel": { "ByteDance-Seed/Seed-OSS-36B-Instruct": [ { "name": "Seed-OSS-36B", "huggingface_id": "ByteDance-Seed/Seed-OSS-36B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "seed_oss" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "google/diffusiongemma-26B-A4B-it": [ { "name": "DiffusionGemma 26B-A4B IT", "huggingface_id": "google/diffusiongemma-26B-A4B-it", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic": [ { "name": "DiffusionGemma 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4": [ { "name": "DiffusionGemma 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-12B-it": [ { "name": "Gemma 4 12B IT", "huggingface_id": "google/gemma-4-12B-it", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "google/gemma-4-12B-it-qat-w4a16-ct": [ { "name": "Gemma 4 12B IT (w4a16)", "huggingface_id": "google/gemma-4-12B-it-qat-w4a16-ct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/translategemma-27b-it": [ { "name": "TranslateGemma 27B IT", "huggingface_id": "google/translategemma-27b-it", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Infomaniak-AI/vllm-translategemma-27b-it": [ { "name": "TranslateGemma 27B IT (vllm_optimized)", "huggingface_id": "Infomaniak-AI/vllm-translategemma-27b-it", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "vllm_optimized" } ], "google/translategemma-4b-it": [ { "name": "TranslateGemma 27B IT (small_4b)", "huggingface_id": "google/translategemma-4b-it", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "small_4b" } ], "JetBrains/Mellum2-12B-A2.5B-Instruct": [ { "name": "Mellum2-12B-A2.5B-Instruct", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "JetBrains/Mellum2-12B-A2.5B-Thinking": [ { "name": "Mellum2-12B-A2.5B-Thinking", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Thinking", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Base": [ { "name": "LFM2.5 1.2B Base", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Base", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Instruct": [ { "name": "LFM2.5 1.2B Instruct", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP-202606": [ { "name": "LFM2.5 1.2B JP (202606)", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP-202606", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP": [ { "name": "LFM2.5 1.2B JP", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Thinking": [ { "name": "LFM2.5 1.2B Thinking", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Thinking", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-230M": [ { "name": "LFM2.5 230M", "huggingface_id": "LiquidAI/LFM2.5-230M", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-350M": [ { "name": "LFM2.5 350M", "huggingface_id": "LiquidAI/LFM2.5-350M", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-8B-A1B": [ { "name": "LFM2.5 8B A1B", "huggingface_id": "LiquidAI/LFM2.5-8B-A1B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-1.6B": [ { "name": "LFM2.5 VL 1.6B", "huggingface_id": "LiquidAI/LFM2.5-VL-1.6B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-450M": [ { "name": "LFM2.5 VL 450M", "huggingface_id": "LiquidAI/LFM2.5-VL-450M", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.1": [ { "name": "MiniMax-M2.1", "huggingface_id": "MiniMaxAI/MiniMax-M2.1", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.5": [ { "name": "MiniMax-M2.5", "huggingface_id": "MiniMaxAI/MiniMax-M2.5", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.5-NVFP4": [ { "name": "MiniMax-M2.5 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.5-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2": [ { "name": "MiniMax-M2", "huggingface_id": "MiniMaxAI/MiniMax-M2", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/MiniMax-M2-NVFP4": [ { "name": "MiniMax-M2 (nvfp4)", "huggingface_id": "RedHatAI/MiniMax-M2-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M3": [ { "name": "MiniMax-M3", "huggingface_id": "MiniMaxAI/MiniMax-M3", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M3-MXFP8": [ { "name": "MiniMax-M3 (mxfp8)", "huggingface_id": "MiniMaxAI/MiniMax-M3-MXFP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp8" } ], "nvidia/MiniMax-M3-NVFP4": [ { "name": "MiniMax-M3 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M3-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "amd/MiniMax-M3-MXFP4": [ { "name": "MiniMax-M3 (mxfp4)", "huggingface_id": "amd/MiniMax-M3-MXFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp4" } ], "OpenGVLab/InternVL3_5-8B": [ { "name": "InternVL3.5", "huggingface_id": "OpenGVLab/InternVL3_5-8B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-SoundEffect": [ { "name": "MOSS-SoundEffect", "huggingface_id": "OpenMOSS-Team/MOSS-SoundEffect", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_sound_effect.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTS-Realtime": [ { "name": "MOSS-TTS-Realtime", "huggingface_id": "OpenMOSS-Team/MOSS-TTS-Realtime", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTS": [ { "name": "MOSS-TTS", "huggingface_id": "OpenMOSS-Team/MOSS-TTS", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_tts.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTSD-v1.0": [ { "name": "MOSS-TTSD-v1.0", "huggingface_id": "OpenMOSS-Team/MOSS-TTSD-v1.0", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_ttsd.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-Transcribe-Diarize": [ { "name": "MOSS-Transcribe-Diarize", "huggingface_id": "OpenMOSS-Team/MOSS-Transcribe-Diarize", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-VoiceGenerator": [ { "name": "MOSS-VoiceGenerator", "huggingface_id": "OpenMOSS-Team/MOSS-VoiceGenerator", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_voice_generator.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL-1.5": [ { "name": "PaddleOCR-VL-1.5", "huggingface_id": "PaddlePaddle/PaddleOCR-VL-1.5", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL-1.6": [ { "name": "PaddleOCR-VL-1.6", "huggingface_id": "PaddlePaddle/PaddleOCR-VL-1.6", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL": [ { "name": "PaddleOCR-VL", "huggingface_id": "PaddlePaddle/PaddleOCR-VL", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen-Image": [ { "name": "Qwen-Image", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" }, { "name": "Qwen-Image (fp8)", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "fp8", "--ignored-layers": "img_mlp" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" }, { "name": "Qwen-Image (int8)", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "int8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "Qwen/Qwen2.5-VL-72B-Instruct": [ { "name": "Qwen2.5-VL-72B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-72B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct-AWQ", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen2.5-VL-7B-Instruct": [ { "name": "Qwen2.5-VL-7B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-7B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-7B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct-AWQ", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-235B-A22B-Instruct-2507": [ { "name": "Qwen3-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-235B-A22B-FP8": [ { "name": "Qwen3-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-235B-A22B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4": [ { "name": "Qwen3-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-32B": [ { "name": "Qwen3-32B", "huggingface_id": "Qwen/Qwen3-32B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-32B-FP8": [ { "name": "Qwen3-32B (fp8)", "huggingface_id": "Qwen/Qwen3-32B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-32B-AWQ": [ { "name": "Qwen3-32B (awq)", "huggingface_id": "Qwen/Qwen3-32B-AWQ", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-4B": [ { "name": "Qwen3-4B", "huggingface_id": "Qwen/Qwen3-4B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-4B-FP8": [ { "name": "Qwen3-4B (fp8)", "huggingface_id": "Qwen/Qwen3-4B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-ASR-1.7B": [ { "name": "Qwen3-ASR-1.7B", "huggingface_id": "Qwen/Qwen3-ASR-1.7B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct": [ { "name": "Qwen3-Coder-480B-A35B-Instruct", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct": [ { "name": "Qwen3-Next-80B-A3B-Instruct", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8": [ { "name": "Qwen3-Next-80B-A3B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4": [ { "name": "Qwen3-Next-80B-A3B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (voicedesign-1.7b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "voicedesign-1.7b" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-Base": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (base-1.7b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base-1.7b" } ], "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (customvoice-0.6b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "customvoice-0.6b" } ], "Qwen/Qwen3-TTS-12Hz-0.6B-Base": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (base-0.6b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-0.6B-Base", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base-0.6b" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-0.8B": [ { "name": "Qwen3.5-0.8B", "huggingface_id": "Qwen/Qwen3.5-0.8B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-122B-A10B": [ { "name": "Qwen3.5-122B-A10B", "huggingface_id": "Qwen/Qwen3.5-122B-A10B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-122B-A10B-FP8": [ { "name": "Qwen3.5-122B-A10B (fp8)", "huggingface_id": "Qwen/Qwen3.5-122B-A10B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-122B-A10B-GPTQ-Int4": [ { "name": "Qwen3.5-122B-A10B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-122B-A10B-GPTQ-Int4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-27B": [ { "name": "Qwen3.5-27B", "huggingface_id": "Qwen/Qwen3.5-27B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-27B-FP8": [ { "name": "Qwen3.5-27B (fp8)", "huggingface_id": "Qwen/Qwen3.5-27B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-27B-GPTQ-Int4": [ { "name": "Qwen3.5-27B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-27B-GPTQ-Int4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-2B": [ { "name": "Qwen3.5-2B", "huggingface_id": "Qwen/Qwen3.5-2B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-35B-A3B": [ { "name": "Qwen3.5-35B-A3B", "huggingface_id": "Qwen/Qwen3.5-35B-A3B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-35B-A3B-FP8": [ { "name": "Qwen3.5-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4": [ { "name": "Qwen3.5-35B-A3B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-397B-A17B": [ { "name": "Qwen3.5-397B", "huggingface_id": "Qwen/Qwen3.5-397B-A17B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Qwen3.5-397B-A17B-NVFP4": [ { "name": "Qwen3.5-397B (nvfp4)", "huggingface_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4": [ { "name": "Qwen3.5-397B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-4B": [ { "name": "Qwen3.5-4B", "huggingface_id": "Qwen/Qwen3.5-4B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-9B": [ { "name": "Qwen3.5-9B", "huggingface_id": "Qwen/Qwen3.5-9B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-27B": [ { "name": "Qwen3.6-27B", "huggingface_id": "Qwen/Qwen3.6-27B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-27B-FP8": [ { "name": "Qwen3.6-27B (fp8)", "huggingface_id": "Qwen/Qwen3.6-27B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "512", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-27B-NVFP4": [ { "name": "Qwen3.6-27B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-27B-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3Guard-Gen-8B": [ { "name": "Qwen3Guard-Gen-8B", "huggingface_id": "Qwen/Qwen3Guard-Gen-8B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3Guard-Gen-4B": [ { "name": "Qwen3Guard-Gen-8B (small_4b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-4B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "small_4b" } ], "Qwen/Qwen3Guard-Gen-0.6B": [ { "name": "Qwen3Guard-Gen-8B (tiny_0_6b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-0.6B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "tiny_0_6b" } ], "Wan-AI/Wan2.2-T2V-A14B-Diffusers": [ { "name": "Wan2.2", "huggingface_id": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} }, "balance": { "extra_args": { "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2-Flash": [ { "name": "MiMo-V2-Flash", "huggingface_id": "XiaomiMiMo/MiMo-V2-Flash", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5-Pro": [ { "name": "MiMo-V2.5-Pro", "huggingface_id": "XiaomiMiMo/MiMo-V2.5-Pro", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5": [ { "name": "MiMo-V2.5", "huggingface_id": "XiaomiMiMo/MiMo-V2.5", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "arcee-ai/Trinity-Large-Thinking": [ { "name": "Trinity-Large-Thinking", "huggingface_id": "arcee-ai/Trinity-Large-Thinking", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--dtype": "bfloat16", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--dtype": "bfloat16", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "arcee-ai/Trinity-Large-Thinking-NVFP4": [ { "name": "Trinity-Large-Thinking (nvfp4)", "huggingface_id": "arcee-ai/Trinity-Large-Thinking-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--dtype": "bfloat16", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--dtype": "bfloat16", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "baidu/ERNIE-4.5-21B-A3B-PT": [ { "name": "ERNIE-4.5", "huggingface_id": "baidu/ERNIE-4.5-21B-A3B-PT", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-300B-A47B-PT": [ { "name": "ERNIE-4.5 (300b)", "huggingface_id": "baidu/ERNIE-4.5-300B-A47B-PT", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--gpu-memory-utilization": "0.95", "--quantization": "fp8", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--gpu-memory-utilization": "0.95", "--quantization": "fp8", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "300b" } ], "baidu/ERNIE-4.5-VL-28B-A3B-PT": [ { "name": "ERNIE-4.5-VL", "huggingface_id": "baidu/ERNIE-4.5-VL-28B-A3B-PT", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-424B-A47B-PT": [ { "name": "ERNIE-4.5-VL (424b)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b" }, { "name": "ERNIE-4.5-VL (424b_fp8)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b_fp8" } ], "baidu/Unlimited-OCR": [ { "name": "Unlimited-OCR", "huggingface_id": "baidu/Unlimited-OCR", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.unlimited_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "bosonai/higgs-audio-v3-tts-4b": [ { "name": "Higgs Audio V3 TTS", "huggingface_id": "bosonai/higgs-audio-v3-tts-4b", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-OCR-2": [ { "name": "DeepSeek-OCR-2", "huggingface_id": "deepseek-ai/DeepSeek-OCR-2", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.deepseek_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-OCR": [ { "name": "DeepSeek-OCR", "huggingface_id": "deepseek-ai/DeepSeek-OCR", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.deepseek_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1": [ { "name": "DeepSeek-R1", "huggingface_id": "deepseek-ai/DeepSeek-R1", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1-0528": [ { "name": "DeepSeek-R1 (r1_0528)", "huggingface_id": "deepseek-ai/DeepSeek-R1-0528", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "r1_0528" } ], "nvidia/DeepSeek-R1-0528-NVFP4-v2": [ { "name": "DeepSeek-R1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-R1-0528-NVFP4-v2", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.1": [ { "name": "DeepSeek-V3.1", "huggingface_id": "deepseek-ai/DeepSeek-V3.1", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.1-NVFP4": [ { "name": "DeepSeek-V3.1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.1-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.2-Exp": [ { "name": "DeepSeek-V3.2-Exp", "huggingface_id": "deepseek-ai/DeepSeek-V3.2-Exp", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-V3.2": [ { "name": "DeepSeek-V3.2", "huggingface_id": "deepseek-ai/DeepSeek-V3.2", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.2-NVFP4": [ { "name": "DeepSeek-V3.2 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.2-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3": [ { "name": "DeepSeek-V3", "huggingface_id": "deepseek-ai/DeepSeek-V3", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3-FP4": [ { "name": "DeepSeek-V3 (fp4)", "huggingface_id": "nvidia/DeepSeek-V3-FP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "fishaudio/s2-pro": [ { "name": "Fish Speech S2 Pro", "huggingface_id": "fishaudio/s2-pro", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ling-2.6-1T": [ { "name": "Ling-2.6-1T", "huggingface_id": "inclusionAI/Ling-2.6-1T", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ling-2.6-flash": [ { "name": "Ling-2.6-flash", "huggingface_id": "inclusionAI/Ling-2.6-flash", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ming-omni-tts-0.5B": [ { "name": "Ming-omni-tts-0.5B", "huggingface_id": "inclusionAI/Ming-omni-tts-0.5B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--enforce-eager": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-1T-FP8": [ { "name": "Ring-1T-FP8", "huggingface_id": "inclusionAI/Ring-1T-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-2.6-1T": [ { "name": "Ring-2.6-1T", "huggingface_id": "inclusionAI/Ring-2.6-1T", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1": [ { "name": "Intern-S1", "huggingface_id": "internlm/Intern-S1", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1-FP8": [ { "name": "Intern-S1 (fp8)", "huggingface_id": "internlm/Intern-S1-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "internlm/Intern-S2-Preview": [ { "name": "Intern-S2-Preview", "huggingface_id": "internlm/Intern-S2-Preview", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S2-Preview-FP8": [ { "name": "Intern-S2-Preview (fp8)", "huggingface_id": "internlm/Intern-S2-Preview-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "jinaai/jina-embeddings-v5-text-small-retrieval": [ { "name": "Jina Embeddings v5 Text Small", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-retrieval", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "jinaai/jina-embeddings-v5-text-small-text-matching": [ { "name": "Jina Embeddings v5 Text Small (text_matching)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-text-matching", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "text_matching" } ], "jinaai/jina-embeddings-v5-text-small-classification": [ { "name": "Jina Embeddings v5 Text Small (classification)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-classification", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "classification" } ], "jinaai/jina-embeddings-v5-text-small-clustering": [ { "name": "Jina Embeddings v5 Text Small (clustering)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-clustering", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "clustering" } ], "jinaai/jina-reranker-m0": [ { "name": "Jina Reranker m0", "huggingface_id": "jinaai/jina-reranker-m0", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--gpu-memory-utilization": "0.75", "--max-num-seqs": "32" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "meituan-longcat/LongCat-Image-Edit": [ { "name": "LongCat-Image-Edit", "huggingface_id": "meituan-longcat/LongCat-Image-Edit", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "meta-llama/Llama-4-Scout-17B-16E-Instruct": [ { "name": "Llama-4-Scout", "huggingface_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} }, "balance": { "extra_args": { "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8": [ { "name": "Llama-4-Scout (fp8)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4": [ { "name": "Llama-4-Scout (nvfp4)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "microsoft/Phi-4-mini-instruct": [ { "name": "Phi-4", "huggingface_id": "microsoft/Phi-4-mini-instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "microsoft/Phi-4-mini-reasoning": [ { "name": "Phi-4 (mini_reasoning)", "huggingface_id": "microsoft/Phi-4-mini-reasoning", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mini_reasoning" } ], "microsoft/Phi-4-reasoning": [ { "name": "Phi-4 (reasoning)", "huggingface_id": "microsoft/Phi-4-reasoning", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "reasoning" } ], "microsoft/Phi-4-multimodal-instruct": [ { "name": "Phi-4 (multimodal)", "huggingface_id": "microsoft/Phi-4-multimodal-instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "multimodal" } ], "mistralai/Ministral-3-14B-Instruct-2512": [ { "name": "Ministral-3-Instruct", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-8B-Instruct-2512": [ { "name": "Ministral-3-Instruct (8b)", "huggingface_id": "mistralai/Ministral-3-8B-Instruct-2512", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "8b" } ], "mistralai/Ministral-3-3B-Instruct-2512": [ { "name": "Ministral-3-Instruct (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Instruct-2512", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Instruct-2512-FP8": [ { "name": "Ministral-3-Instruct (fp8)", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Ministral-3-8B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning", "huggingface_id": "mistralai/Ministral-3-8B-Reasoning-2512", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-3B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Reasoning-2512", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (14b)", "huggingface_id": "mistralai/Ministral-3-14B-Reasoning-2512", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "14b" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512": [ { "name": "Mistral-Large-3-675B-Instruct", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4": [ { "name": "Mistral-Large-3-675B-Instruct (nvfp4)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8": [ { "name": "Mistral-Large-3-675B-Instruct (fp8)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Mistral-Medium-3.5-128B": [ { "name": "Mistral-Medium-3.5", "huggingface_id": "mistralai/Mistral-Medium-3.5-128B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"mistralai/Mistral-Medium-3.5-128B-EAGLE\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Small-4-119B-2603": [ { "name": "Mistral-Small-4-119B", "huggingface_id": "mistralai/Mistral-Small-4-119B-2603", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "FLASH_ATTN_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--attention-backend": "FLASH_ATTN_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Small-4-119B-2603-NVFP4": [ { "name": "Mistral-Small-4-119B (nvfp4)", "huggingface_id": "mistralai/Mistral-Small-4-119B-2603-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "TRITON_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--attention-backend": "TRITON_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Voxtral-4B-TTS-2603": [ { "name": "Voxtral-4B-TTS-2603", "huggingface_id": "mistralai/Voxtral-4B-TTS-2603", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Voxtral-Mini-4B-Realtime-2602": [ { "name": "Voxtral-Mini-4B-Realtime-2602", "huggingface_id": "mistralai/Voxtral-Mini-4B-Realtime-2602", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer-mode": "mistral", "--compilation_config": "{\"cudagraph_mode\": \"PIECEWISE\"}" }, "extra_envs": { "VLLM_DISABLE_COMPILE_CACHE": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Instruct": [ { "name": "Kimi-K2-Instruct", "huggingface_id": "moonshotai/Kimi-K2-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Thinking": [ { "name": "Kimi-K2-Thinking", "huggingface_id": "moonshotai/Kimi-K2-Thinking", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2-Thinking-NVFP4": [ { "name": "Kimi-K2-Thinking (nvfp4)", "huggingface_id": "nvidia/Kimi-K2-Thinking-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.5": [ { "name": "Kimi-K2.5", "huggingface_id": "moonshotai/Kimi-K2.5", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.5-NVFP4": [ { "name": "Kimi-K2.5 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.5-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.7-Code": [ { "name": "Kimi-K2.7-Code", "huggingface_id": "moonshotai/Kimi-K2.7-Code", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-Linear-48B-A3B-Instruct": [ { "name": "Kimi-Linear-48B-A3B-Instruct", "huggingface_id": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Nano": [ { "name": "Cosmos3-Nano", "huggingface_id": "nvidia/Cosmos3-Nano", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Image2Video": [ { "name": "Cosmos3-Super-Image2Video", "huggingface_id": "nvidia/Cosmos3-Super-Image2Video", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Text2Image": [ { "name": "Cosmos3-Super-Text2Image", "huggingface_id": "nvidia/Cosmos3-Super-Text2Image", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super": [ { "name": "Cosmos3-Super", "huggingface_id": "nvidia/Cosmos3-Super", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-4B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-4B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (base_bf16)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base_bf16" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B (fp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2": [ { "name": "NVIDIA Nemotron-Nano-9B-v2", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (base)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (japanese)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "japanese" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (fp8)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (nvfp4)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/MiniCPM-V-4.6": [ { "name": "MiniCPM-V 4.6", "huggingface_id": "openbmb/MiniCPM-V-4.6", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/MiniCPM5-1B": [ { "name": "MiniCPM5-1B", "huggingface_id": "openbmb/MiniCPM5-1B", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/VoxCPM2": [ { "name": "VoxCPM2", "huggingface_id": "openbmb/VoxCPM2", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "pfnet/plamo-2-translate": [ { "name": "PLaMo 2 Translate", "huggingface_id": "pfnet/plamo-2-translate", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "pfnet/plamo-3-nict-31b-base": [ { "name": "PLaMo 3 NICT 31B Base", "huggingface_id": "pfnet/plamo-3-nict-31b-base", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-M.1": [ { "name": "Laguna M.1", "huggingface_id": "poolside/Laguna-M.1", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-M.1-FP8": [ { "name": "Laguna M.1 (fp8)", "huggingface_id": "poolside/Laguna-M.1-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-M.1-NVFP4": [ { "name": "Laguna M.1 (nvfp4)", "huggingface_id": "poolside/Laguna-M.1-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS-2.1": [ { "name": "Laguna XS-2.1", "huggingface_id": "poolside/Laguna-XS-2.1", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS-2.1-FP8": [ { "name": "Laguna XS-2.1 (fp8)", "huggingface_id": "poolside/Laguna-XS-2.1-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS-2.1-NVFP4": [ { "name": "Laguna XS-2.1 (nvfp4)", "huggingface_id": "poolside/Laguna-XS-2.1-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS-2.1-INT4": [ { "name": "Laguna XS-2.1 (int4)", "huggingface_id": "poolside/Laguna-XS-2.1-INT4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "poolside/Laguna-XS.2": [ { "name": "Laguna XS.2", "huggingface_id": "poolside/Laguna-XS.2", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS.2-FP8": [ { "name": "Laguna XS.2 (fp8)", "huggingface_id": "poolside/Laguna-XS.2-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS.2-NVFP4": [ { "name": "Laguna XS.2 (nvfp4)", "huggingface_id": "poolside/Laguna-XS.2-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS.2-INT4": [ { "name": "Laguna XS.2 (int4)", "huggingface_id": "poolside/Laguna-XS.2-INT4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stabilityai/stable-audio-open-1.0": [ { "name": "Stable Audio Open", "huggingface_id": "stabilityai/stable-audio-open-1.0", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enforce-eager": "", "--gpu-memory-utilization": "0.9" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stabilityai/stable-diffusion-3.5-medium": [ { "name": "Stable Diffusion 3.5", "huggingface_id": "stabilityai/stable-diffusion-3.5-medium", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stabilityai/stable-diffusion-3.5-large": [ { "name": "Stable Diffusion 3.5 (large)", "huggingface_id": "stabilityai/stable-diffusion-3.5-large", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "large" } ], "stabilityai/stable-diffusion-3.5-large-turbo": [ { "name": "Stable Diffusion 3.5 (large_turbo)", "huggingface_id": "stabilityai/stable-diffusion-3.5-large-turbo", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "large_turbo" } ], "stepfun-ai/Step-3.5-Flash": [ { "name": "Step-3.5-Flash", "huggingface_id": "stepfun-ai/Step-3.5-Flash", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.5-Flash-FP8": [ { "name": "Step-3.5-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.5-Flash-INT4": [ { "name": "Step-3.5-Flash (int4)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stepfun-ai/Step-3.5-Flash-INT8": [ { "name": "Step-3.5-Flash (int8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "stepfun-ai/Step-3.7-Flash": [ { "name": "Step-3.7-Flash", "huggingface_id": "stepfun-ai/Step-3.7-Flash", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.7-Flash-FP8": [ { "name": "Step-3.7-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.7-Flash-NVFP4": [ { "name": "Step-3.7-Flash (nvfp4)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "tencent/Hunyuan-A13B-Instruct": [ { "name": "Hunyuan-A13B-Instruct", "huggingface_id": "tencent/Hunyuan-A13B-Instruct", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct-FP8": [ { "name": "Hunyuan-A13B-Instruct (fp8)", "huggingface_id": "tencent/Hunyuan-A13B-Instruct-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "tencent/HunyuanOCR": [ { "name": "HunyuanOCR", "huggingface_id": "tencent/HunyuanOCR", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-preview": [ { "name": "Hy3-preview", "huggingface_id": "tencent/Hy3-preview", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3": [ { "name": "Hy3", "huggingface_id": "tencent/Hy3", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-FP8": [ { "name": "Hy3 (fp8)", "huggingface_id": "tencent/Hy3-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-4.5": [ { "name": "GLM-4.5", "huggingface_id": "zai-org/GLM-4.5", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5-FP8": [ { "name": "GLM-4.5 (fp8)", "huggingface_id": "zai-org/GLM-4.5-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.5V": [ { "name": "GLM-4.5V", "huggingface_id": "zai-org/GLM-4.5V", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5V-FP8": [ { "name": "GLM-4.5V (fp8)", "huggingface_id": "zai-org/GLM-4.5V-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6": [ { "name": "GLM-4.6", "huggingface_id": "zai-org/GLM-4.6", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6-FP8": [ { "name": "GLM-4.6 (fp8)", "huggingface_id": "zai-org/GLM-4.6-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6V": [ { "name": "GLM-4.6V", "huggingface_id": "zai-org/GLM-4.6V", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6V-FP8": [ { "name": "GLM-4.6V (fp8)", "huggingface_id": "zai-org/GLM-4.6V-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.7": [ { "name": "GLM-4.7", "huggingface_id": "zai-org/GLM-4.7", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.7-FP8": [ { "name": "GLM-4.7 (fp8)", "huggingface_id": "zai-org/GLM-4.7-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-4.7-NVFP4": [ { "name": "GLM-4.7 (nvfp4)", "huggingface_id": "nvidia/GLM-4.7-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.1": [ { "name": "GLM-5.1", "huggingface_id": "zai-org/GLM-5.1", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5.1-FP8": [ { "name": "GLM-5.1 (fp8)", "huggingface_id": "zai-org/GLM-5.1-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5.1-NVFP4": [ { "name": "GLM-5.1 (nvfp4)", "huggingface_id": "nvidia/GLM-5.1-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2-FP8": [ { "name": "GLM-5.2", "huggingface_id": "zai-org/GLM-5.2-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/GLM-5.2-NVFP4": [ { "name": "GLM-5.2 (nvfp4)", "huggingface_id": "nvidia/GLM-5.2-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2": [ { "name": "GLM-5.2 (bf16)", "huggingface_id": "zai-org/GLM-5.2", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-5": [ { "name": "GLM-5", "huggingface_id": "zai-org/GLM-5", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5-FP8": [ { "name": "GLM-5 (fp8)", "huggingface_id": "zai-org/GLM-5-FP8", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5-NVFP4": [ { "name": "GLM-5 (nvfp4)", "huggingface_id": "nvidia/GLM-5-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-ASR-Nano-2512": [ { "name": "GLM-ASR-Nano-2512", "huggingface_id": "zai-org/GLM-ASR-Nano-2512", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-GA": [ { "name": "GLM-GA", "huggingface_id": "zai-org/GLM-GA", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--allowed-local-media-path": "/", "--mm-processor-cache-type": "shm", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "glm4_6v" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-Image": [ { "name": "GLM-Image", "huggingface_id": "zai-org/GLM-Image", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-OCR": [ { "name": "GLM-OCR", "huggingface_id": "zai-org/GLM-OCR", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-TTS": [ { "name": "GLM-TTS", "huggingface_id": "zai-org/GLM-TTS", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/Glyph": [ { "name": "Glyph", "huggingface_id": "zai-org/Glyph", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--limit-mm-per-prompt.video": "0", "--reasoning-parser": "glm45", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "intel", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] }, "nvidia": { "ByteDance-Seed/Seed-OSS-36B-Instruct": [ { "name": "Seed-OSS-36B", "huggingface_id": "ByteDance-Seed/Seed-OSS-36B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "seed_oss" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "google/diffusiongemma-26B-A4B-it": [ { "name": "DiffusionGemma 26B-A4B IT", "huggingface_id": "google/diffusiongemma-26B-A4B-it", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic": [ { "name": "DiffusionGemma 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4": [ { "name": "DiffusionGemma 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/diffusiongemma-26B-A4B-it-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4" }, "extra_envs": {} }, "balance": { "extra_args": { "--max-num-seqs": "4", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-12B-it": [ { "name": "Gemma 4 12B IT", "huggingface_id": "google/gemma-4-12B-it", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "google/gemma-4-12B-it-qat-w4a16-ct": [ { "name": "Gemma 4 12B IT (w4a16)", "huggingface_id": "google/gemma-4-12B-it-qat-w4a16-ct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-26B-A4B-it": [ { "name": "Gemma 4 26B-A4B IT", "huggingface_id": "google/gemma-4-26B-A4B-it", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic": [ { "name": "Gemma 4 26B-A4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "RedHatAI/gemma-4-26B-A4B-it-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (nvfp4)", "huggingface_id": "RedHatAI/gemma-4-26B-A4B-it-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it": [ { "name": "Gemma 4 31B IT", "huggingface_id": "google/gemma-4-31B-it", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-31B-it-FP8-dynamic": [ { "name": "Gemma 4 31B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-31B-it-FP8-dynamic", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/gemma-4-31B-it-NVFP4": [ { "name": "Gemma 4 31B IT (nvfp4)", "huggingface_id": "nvidia/gemma-4-31B-it-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "google/gemma-4-31B-it-qat-w4a16-ct": [ { "name": "Gemma 4 31B IT (w4a16)", "huggingface_id": "google/gemma-4-31B-it-qat-w4a16-ct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it": [ { "name": "Gemma 4 E2B IT", "huggingface_id": "google/gemma-4-E2B-it", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E2B-it-FP8-dynamic": [ { "name": "Gemma 4 E2B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E2B-it-FP8-dynamic", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E2B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E2B IT (w4a16)", "huggingface_id": "google/gemma-4-E2B-it-qat-w4a16-ct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E2B-it-qat-mobile-ct": [ { "name": "Gemma 4 E2B IT (mobile)", "huggingface_id": "google/gemma-4-E2B-it-qat-mobile-ct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E2B-it-assistant\",\"num_speculative_tokens\":2}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/gemma-4-E4B-it": [ { "name": "Gemma 4 E4B IT", "huggingface_id": "google/gemma-4-E4B-it", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/gemma-4-E4B-it-FP8-dynamic": [ { "name": "Gemma 4 E4B IT (fp8)", "huggingface_id": "RedHatAI/gemma-4-E4B-it-FP8-dynamic", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "google/gemma-4-E4B-it-qat-w4a16-ct": [ { "name": "Gemma 4 E4B IT (w4a16)", "huggingface_id": "google/gemma-4-E4B-it-qat-w4a16-ct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "w4a16" } ], "google/gemma-4-E4B-it-qat-mobile-ct": [ { "name": "Gemma 4 E4B IT (mobile)", "huggingface_id": "google/gemma-4-E4B-it-qat-mobile-ct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mobile" } ], "google/translategemma-27b-it": [ { "name": "TranslateGemma 27B IT", "huggingface_id": "google/translategemma-27b-it", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Infomaniak-AI/vllm-translategemma-27b-it": [ { "name": "TranslateGemma 27B IT (vllm_optimized)", "huggingface_id": "Infomaniak-AI/vllm-translategemma-27b-it", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "vllm_optimized" } ], "google/translategemma-4b-it": [ { "name": "TranslateGemma 27B IT (small_4b)", "huggingface_id": "google/translategemma-4b-it", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "small_4b" } ], "JetBrains/Mellum2-12B-A2.5B-Instruct": [ { "name": "Mellum2-12B-A2.5B-Instruct", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "JetBrains/Mellum2-12B-A2.5B-Thinking": [ { "name": "Mellum2-12B-A2.5B-Thinking", "huggingface_id": "JetBrains/Mellum2-12B-A2.5B-Thinking", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Base": [ { "name": "LFM2.5 1.2B Base", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Base", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Instruct": [ { "name": "LFM2.5 1.2B Instruct", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP-202606": [ { "name": "LFM2.5 1.2B JP (202606)", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP-202606", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-JP": [ { "name": "LFM2.5 1.2B JP", "huggingface_id": "LiquidAI/LFM2.5-1.2B-JP", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-1.2B-Thinking": [ { "name": "LFM2.5 1.2B Thinking", "huggingface_id": "LiquidAI/LFM2.5-1.2B-Thinking", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-230M": [ { "name": "LFM2.5 230M", "huggingface_id": "LiquidAI/LFM2.5-230M", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-350M": [ { "name": "LFM2.5 350M", "huggingface_id": "LiquidAI/LFM2.5-350M", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-8B-A1B": [ { "name": "LFM2.5 8B A1B", "huggingface_id": "LiquidAI/LFM2.5-8B-A1B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "lfm2", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-1.6B": [ { "name": "LFM2.5 VL 1.6B", "huggingface_id": "LiquidAI/LFM2.5-VL-1.6B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "LiquidAI/LFM2.5-VL-450M": [ { "name": "LFM2.5 VL 450M", "huggingface_id": "LiquidAI/LFM2.5-VL-450M", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.1": [ { "name": "MiniMax-M2.1", "huggingface_id": "MiniMaxAI/MiniMax-M2.1", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M2.5": [ { "name": "MiniMax-M2.5", "huggingface_id": "MiniMaxAI/MiniMax-M2.5", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.5-NVFP4": [ { "name": "MiniMax-M2.5 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.5-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--compilation-config": "{\"cudagraph_mode\":\"PIECEWISE\"}", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2.7": [ { "name": "MiniMax-M2.7", "huggingface_id": "MiniMaxAI/MiniMax-M2.7", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/MiniMax-M2.7-NVFP4": [ { "name": "MiniMax-M2.7 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M2.7-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M2": [ { "name": "MiniMax-M2", "huggingface_id": "MiniMaxAI/MiniMax-M2", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "RedHatAI/MiniMax-M2-NVFP4": [ { "name": "MiniMax-M2 (nvfp4)", "huggingface_id": "RedHatAI/MiniMax-M2-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "minimax_m2", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "MiniMaxAI/MiniMax-M3": [ { "name": "MiniMax-M3", "huggingface_id": "MiniMaxAI/MiniMax-M3", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "MiniMaxAI/MiniMax-M3-MXFP8": [ { "name": "MiniMax-M3 (mxfp8)", "huggingface_id": "MiniMaxAI/MiniMax-M3-MXFP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp8" } ], "nvidia/MiniMax-M3-NVFP4": [ { "name": "MiniMax-M3 (nvfp4)", "huggingface_id": "nvidia/MiniMax-M3-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "balance": { "extra_args": { "--block-size": "128", "--trust-remote-code": "", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "amd/MiniMax-M3-MXFP4": [ { "name": "MiniMax-M3 (mxfp4)", "huggingface_id": "amd/MiniMax-M3-MXFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--block-size": "128", "--tool-call-parser": "minimax_m3", "--enable-auto-tool-choice": "", "--reasoning-parser": "minimax_m3", "--speculative-config": "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mxfp4" } ], "OpenGVLab/InternVL3_5-8B": [ { "name": "InternVL3.5", "huggingface_id": "OpenGVLab/InternVL3_5-8B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-SoundEffect": [ { "name": "MOSS-SoundEffect", "huggingface_id": "OpenMOSS-Team/MOSS-SoundEffect", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_sound_effect.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTS-Realtime": [ { "name": "MOSS-TTS-Realtime", "huggingface_id": "OpenMOSS-Team/MOSS-TTS-Realtime", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTS": [ { "name": "MOSS-TTS", "huggingface_id": "OpenMOSS-Team/MOSS-TTS", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_tts.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-TTSD-v1.0": [ { "name": "MOSS-TTSD-v1.0", "huggingface_id": "OpenMOSS-Team/MOSS-TTSD-v1.0", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_ttsd.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-Transcribe-Diarize": [ { "name": "MOSS-Transcribe-Diarize", "huggingface_id": "OpenMOSS-Team/MOSS-Transcribe-Diarize", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "OpenMOSS-Team/MOSS-VoiceGenerator": [ { "name": "MOSS-VoiceGenerator", "huggingface_id": "OpenMOSS-Team/MOSS-VoiceGenerator", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--deploy-config": "vllm_omni/deploy/moss_voice_generator.yaml" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL-1.5": [ { "name": "PaddleOCR-VL-1.5", "huggingface_id": "PaddlePaddle/PaddleOCR-VL-1.5", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL-1.6": [ { "name": "PaddleOCR-VL-1.6", "huggingface_id": "PaddlePaddle/PaddleOCR-VL-1.6", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "PaddlePaddle/PaddleOCR-VL": [ { "name": "PaddleOCR-VL", "huggingface_id": "PaddlePaddle/PaddleOCR-VL", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-batched-tokens": "16384", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen-Image": [ { "name": "Qwen-Image", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" }, { "name": "Qwen-Image (fp8)", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "fp8", "--ignored-layers": "img_mlp" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" }, { "name": "Qwen-Image (int8)", "huggingface_id": "Qwen/Qwen-Image", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "int8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "Qwen/Qwen2.5-VL-72B-Instruct": [ { "name": "Qwen2.5-VL-72B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-72B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-72B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-72B-Instruct-AWQ", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen2.5-VL-7B-Instruct": [ { "name": "Qwen2.5-VL-7B-Instruct", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen2.5-VL-7B-Instruct-AWQ": [ { "name": "Qwen2.5-VL-7B-Instruct (awq)", "huggingface_id": "Qwen/Qwen2.5-VL-7B-Instruct-AWQ", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-235B-A22B-Instruct-2507": [ { "name": "Qwen3-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-235B-A22B-FP8": [ { "name": "Qwen3-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-235B-A22B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4": [ { "name": "Qwen3-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-32B": [ { "name": "Qwen3-32B", "huggingface_id": "Qwen/Qwen3-32B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-32B-FP8": [ { "name": "Qwen3-32B (fp8)", "huggingface_id": "Qwen/Qwen3-32B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-32B-AWQ": [ { "name": "Qwen3-32B (awq)", "huggingface_id": "Qwen/Qwen3-32B-AWQ", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--quantization": "awq", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "awq" } ], "Qwen/Qwen3-4B": [ { "name": "Qwen3-4B", "huggingface_id": "Qwen/Qwen3-4B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-4B-FP8": [ { "name": "Qwen3-4B (fp8)", "huggingface_id": "Qwen/Qwen3-4B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3-ASR-1.7B": [ { "name": "Qwen3-ASR-1.7B", "huggingface_id": "Qwen/Qwen3-ASR-1.7B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct": [ { "name": "Qwen3-Coder-480B-A35B-Instruct", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4": [ { "name": "Qwen3-Coder-480B-A35B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Coder-480B-A35B-Instruct-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct": [ { "name": "Qwen3-Next-80B-A3B-Instruct", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8": [ { "name": "Qwen3-Next-80B-A3B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4": [ { "name": "Qwen3-Next-80B-A3B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "hermes", "--speculative-config": "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill": "", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (voicedesign-1.7b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "voicedesign-1.7b" } ], "Qwen/Qwen3-TTS-12Hz-1.7B-Base": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (base-1.7b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base-1.7b" } ], "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (customvoice-0.6b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "customvoice-0.6b" } ], "Qwen/Qwen3-TTS-12Hz-0.6B-Base": [ { "name": "Qwen3-TTS-12Hz-1.7B-CustomVoice (base-0.6b)", "huggingface_id": "Qwen/Qwen3-TTS-12Hz-0.6B-Base", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base-0.6b" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct": [ { "name": "Qwen3-VL-235B-A22B-Instruct", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": [ { "name": "Qwen3-VL-235B-A22B-Instruct (fp8)", "huggingface_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4": [ { "name": "Qwen3-VL-235B-A22B-Instruct (nvfp4)", "huggingface_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-0.8B": [ { "name": "Qwen3.5-0.8B", "huggingface_id": "Qwen/Qwen3.5-0.8B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-122B-A10B": [ { "name": "Qwen3.5-122B-A10B", "huggingface_id": "Qwen/Qwen3.5-122B-A10B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-122B-A10B-FP8": [ { "name": "Qwen3.5-122B-A10B (fp8)", "huggingface_id": "Qwen/Qwen3.5-122B-A10B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-122B-A10B-GPTQ-Int4": [ { "name": "Qwen3.5-122B-A10B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-122B-A10B-GPTQ-Int4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-27B": [ { "name": "Qwen3.5-27B", "huggingface_id": "Qwen/Qwen3.5-27B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-27B-FP8": [ { "name": "Qwen3.5-27B (fp8)", "huggingface_id": "Qwen/Qwen3.5-27B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-27B-GPTQ-Int4": [ { "name": "Qwen3.5-27B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-27B-GPTQ-Int4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-2B": [ { "name": "Qwen3.5-2B", "huggingface_id": "Qwen/Qwen3.5-2B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-35B-A3B": [ { "name": "Qwen3.5-35B-A3B", "huggingface_id": "Qwen/Qwen3.5-35B-A3B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-35B-A3B-FP8": [ { "name": "Qwen3.5-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4": [ { "name": "Qwen3.5-35B-A3B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-397B-A17B": [ { "name": "Qwen3.5-397B", "huggingface_id": "Qwen/Qwen3.5-397B-A17B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Qwen3.5-397B-A17B-NVFP4": [ { "name": "Qwen3.5-397B (nvfp4)", "huggingface_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency", "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4": [ { "name": "Qwen3.5-397B (gptq_int4)", "huggingface_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_FLASHINFER_MOE_BACKEND": "latency" } } }, "default_strategy": "latency-focus", "variant_name": "gptq_int4" } ], "Qwen/Qwen3.5-4B": [ { "name": "Qwen3.5-4B", "huggingface_id": "Qwen/Qwen3.5-4B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.5-9B": [ { "name": "Qwen3.5-9B", "huggingface_id": "Qwen/Qwen3.5-9B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-27B": [ { "name": "Qwen3.6-27B", "huggingface_id": "Qwen/Qwen3.6-27B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-27B-FP8": [ { "name": "Qwen3.6-27B (fp8)", "huggingface_id": "Qwen/Qwen3.6-27B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "512", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-27B-NVFP4": [ { "name": "Qwen3.6-27B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-27B-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3.6-35B-A3B": [ { "name": "Qwen3.6-35B-A3B", "huggingface_id": "Qwen/Qwen3.6-35B-A3B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3.6-35B-A3B-FP8": [ { "name": "Qwen3.6-35B-A3B (fp8)", "huggingface_id": "Qwen/Qwen3.6-35B-A3B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Qwen3.6-35B-A3B-NVFP4": [ { "name": "Qwen3.6-35B-A3B (nvfp4)", "huggingface_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--attention-backend": "flashinfer", "--moe-backend": "marlin", "--enable-prefix-caching": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "Qwen/Qwen3Guard-Gen-8B": [ { "name": "Qwen3Guard-Gen-8B", "huggingface_id": "Qwen/Qwen3Guard-Gen-8B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "Qwen/Qwen3Guard-Gen-4B": [ { "name": "Qwen3Guard-Gen-8B (small_4b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-4B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "small_4b" } ], "Qwen/Qwen3Guard-Gen-0.6B": [ { "name": "Qwen3Guard-Gen-8B (tiny_0_6b)", "huggingface_id": "Qwen/Qwen3Guard-Gen-0.6B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "tiny_0_6b" } ], "Wan-AI/Wan2.2-T2V-A14B-Diffusers": [ { "name": "Wan2.2", "huggingface_id": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} }, "balance": { "extra_args": { "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2-Flash": [ { "name": "MiMo-V2-Flash", "huggingface_id": "XiaomiMiMo/MiMo-V2-Flash", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.9", "--tool-call-parser": "qwen3_xml", "--reasoning-parser": "qwen3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5-Pro": [ { "name": "MiMo-V2.5-Pro", "huggingface_id": "XiaomiMiMo/MiMo-V2.5-Pro", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "XiaomiMiMo/MiMo-V2.5": [ { "name": "MiMo-V2.5", "huggingface_id": "XiaomiMiMo/MiMo-V2.5", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--generation-config": "vllm", "--gpu-memory-utilization": "0.95", "--tool-call-parser": "mimo", "--enable-auto-tool-choice": "", "--reasoning-parser": "mimo", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "arcee-ai/Trinity-Large-Thinking": [ { "name": "Trinity-Large-Thinking", "huggingface_id": "arcee-ai/Trinity-Large-Thinking", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--dtype": "bfloat16", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--dtype": "bfloat16", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "arcee-ai/Trinity-Large-Thinking-NVFP4": [ { "name": "Trinity-Large-Thinking (nvfp4)", "huggingface_id": "arcee-ai/Trinity-Large-Thinking-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--dtype": "bfloat16", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--dtype": "bfloat16", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "deepseek_r1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "baidu/ERNIE-4.5-21B-A3B-PT": [ { "name": "ERNIE-4.5", "huggingface_id": "baidu/ERNIE-4.5-21B-A3B-PT", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-300B-A47B-PT": [ { "name": "ERNIE-4.5 (300b)", "huggingface_id": "baidu/ERNIE-4.5-300B-A47B-PT", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--gpu-memory-utilization": "0.95", "--quantization": "fp8", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--gpu-memory-utilization": "0.95", "--quantization": "fp8", "--speculative-config": "{\"method\":\"ernie_mtp\",\"model\":\"baidu/ERNIE-4.5-21B-A3B-PT\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "300b" } ], "baidu/ERNIE-4.5-VL-28B-A3B-PT": [ { "name": "ERNIE-4.5-VL", "huggingface_id": "baidu/ERNIE-4.5-VL-28B-A3B-PT", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "baidu/ERNIE-4.5-VL-424B-A47B-PT": [ { "name": "ERNIE-4.5-VL (424b)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b" }, { "name": "ERNIE-4.5-VL (424b_fp8)", "huggingface_id": "baidu/ERNIE-4.5-VL-424B-A47B-PT", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--quantization": "fp8", "--cpu-offload-gb": "50", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "424b_fp8" } ], "baidu/Unlimited-OCR": [ { "name": "Unlimited-OCR", "huggingface_id": "baidu/Unlimited-OCR", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.unlimited_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "bosonai/higgs-audio-v3-tts-4b": [ { "name": "Higgs Audio V3 TTS", "huggingface_id": "bosonai/higgs-audio-v3-tts-4b", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-OCR-2": [ { "name": "DeepSeek-OCR-2", "huggingface_id": "deepseek-ai/DeepSeek-OCR-2", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.deepseek_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-OCR": [ { "name": "DeepSeek-OCR", "huggingface_id": "deepseek-ai/DeepSeek-OCR", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--logits_processors": "vllm.model_executor.models.deepseek_ocr:NGramPerReqLogitsProcessor", "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1": [ { "name": "DeepSeek-R1", "huggingface_id": "deepseek-ai/DeepSeek-R1", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-R1-0528": [ { "name": "DeepSeek-R1 (r1_0528)", "huggingface_id": "deepseek-ai/DeepSeek-R1-0528", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "r1_0528" } ], "nvidia/DeepSeek-R1-0528-NVFP4-v2": [ { "name": "DeepSeek-R1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-R1-0528-NVFP4-v2", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekr1.jinja", "--reasoning-parser": "deepseek_r1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.1": [ { "name": "DeepSeek-V3.1", "huggingface_id": "deepseek-ai/DeepSeek-V3.1", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.1-NVFP4": [ { "name": "DeepSeek-V3.1 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.1-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v31", "--chat-template": "examples/tool_chat_template_deepseekv31.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3.2-Exp": [ { "name": "DeepSeek-V3.2-Exp", "huggingface_id": "deepseek-ai/DeepSeek-V3.2-Exp", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "deepseek-ai/DeepSeek-V3.2": [ { "name": "DeepSeek-V3.2", "huggingface_id": "deepseek-ai/DeepSeek-V3.2", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3.2-NVFP4": [ { "name": "DeepSeek-V3.2 (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V3.2-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kernel-config.enable_flashinfer_autotune=False": "", "--kv-cache-dtype": "fp8", "--tokenizer-mode": "deepseek_v32", "--tool-call-parser": "deepseek_v32", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V3": [ { "name": "DeepSeek-V3", "huggingface_id": "deepseek-ai/DeepSeek-V3", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V3-FP4": [ { "name": "DeepSeek-V3 (fp4)", "huggingface_id": "nvidia/DeepSeek-V3-FP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "deepseek_v3", "--chat-template": "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser": "deepseek_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "deepseek-ai/DeepSeek-V4-Flash": [ { "name": "DeepSeek-V4-Flash", "huggingface_id": "deepseek-ai/DeepSeek-V4-Flash", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Flash-NVFP4": [ { "name": "DeepSeek-V4-Flash (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro": [ { "name": "DeepSeek-V4-Pro", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/DeepSeek-V4-Pro-NVFP4": [ { "name": "DeepSeek-V4-Pro (nvfp4)", "huggingface_id": "nvidia/DeepSeek-V4-Pro-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "deepseek-ai/DeepSeek-V4-Pro-DSpark": [ { "name": "DeepSeek-V4-Pro (dspark)", "huggingface_id": "deepseek-ai/DeepSeek-V4-Pro-DSpark", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--block-size": "256", "--tokenizer-mode": "deepseek_v4", "--tool-call-parser": "deepseek_v4", "--enable-auto-tool-choice": "", "--reasoning-parser": "deepseek_v4", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "dspark" } ], "fishaudio/s2-pro": [ { "name": "Fish Speech S2 Pro", "huggingface_id": "fishaudio/s2-pro", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ling-2.6-1T": [ { "name": "Ling-2.6-1T", "huggingface_id": "inclusionAI/Ling-2.6-1T", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ling-2.6-flash": [ { "name": "Ling-2.6-flash", "huggingface_id": "inclusionAI/Ling-2.6-flash", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ming-omni-tts-0.5B": [ { "name": "Ming-omni-tts-0.5B", "huggingface_id": "inclusionAI/Ming-omni-tts-0.5B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--enforce-eager": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-1T-FP8": [ { "name": "Ring-1T-FP8", "huggingface_id": "inclusionAI/Ring-1T-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--max-num-seqs": "32", "--kv-cache-dtype": "fp8", "--served-model-name": "Ring-1T-FP8", "--gpu-memory-utilization": "0.97", "--compilation-config": "{\"use_inductor\": false}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "inclusionAI/Ring-2.6-1T": [ { "name": "Ring-2.6-1T", "huggingface_id": "inclusionAI/Ring-2.6-1T", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1": [ { "name": "Intern-S1", "huggingface_id": "internlm/Intern-S1", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S1-FP8": [ { "name": "Intern-S1 (fp8)", "huggingface_id": "internlm/Intern-S1-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "internlm", "--reasoning-parser": "deepseek_r1", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "internlm/Intern-S2-Preview": [ { "name": "Intern-S2-Preview", "huggingface_id": "internlm/Intern-S2-Preview", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "internlm/Intern-S2-Preview-FP8": [ { "name": "Intern-S2-Preview (fp8)", "huggingface_id": "internlm/Intern-S2-Preview-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "qwen3", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "jinaai/jina-embeddings-v5-text-small-retrieval": [ { "name": "Jina Embeddings v5 Text Small", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-retrieval", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "jinaai/jina-embeddings-v5-text-small-text-matching": [ { "name": "Jina Embeddings v5 Text Small (text_matching)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-text-matching", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "text_matching" } ], "jinaai/jina-embeddings-v5-text-small-classification": [ { "name": "Jina Embeddings v5 Text Small (classification)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-classification", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "classification" } ], "jinaai/jina-embeddings-v5-text-small-clustering": [ { "name": "Jina Embeddings v5 Text Small (clustering)", "huggingface_id": "jinaai/jina-embeddings-v5-text-small-clustering", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--runner": "pooling" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "clustering" } ], "jinaai/jina-reranker-m0": [ { "name": "Jina Reranker m0", "huggingface_id": "jinaai/jina-reranker-m0", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--gpu-memory-utilization": "0.75", "--max-num-seqs": "32" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "meituan-longcat/LongCat-Image-Edit": [ { "name": "LongCat-Image-Edit", "huggingface_id": "meituan-longcat/LongCat-Image-Edit", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "meta-llama/Llama-3.3-70B-Instruct": [ { "name": "Llama-3.3-70B", "huggingface_id": "meta-llama/Llama-3.3-70B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-3.3-70B-Instruct-FP8": [ { "name": "Llama-3.3-70B (fp8)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-3.3-70B-Instruct-NVFP4": [ { "name": "Llama-3.3-70B (nvfp4)", "huggingface_id": "nvidia/Llama-3.3-70B-Instruct-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "meta-llama/Llama-4-Scout-17B-16E-Instruct": [ { "name": "Llama-4-Scout", "huggingface_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} }, "balance": { "extra_args": { "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8": [ { "name": "Llama-4-Scout (fp8)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4": [ { "name": "Llama-4-Scout (nvfp4)", "huggingface_id": "nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "microsoft/Phi-4-mini-instruct": [ { "name": "Phi-4", "huggingface_id": "microsoft/Phi-4-mini-instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "microsoft/Phi-4-mini-reasoning": [ { "name": "Phi-4 (mini_reasoning)", "huggingface_id": "microsoft/Phi-4-mini-reasoning", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "mini_reasoning" } ], "microsoft/Phi-4-reasoning": [ { "name": "Phi-4 (reasoning)", "huggingface_id": "microsoft/Phi-4-reasoning", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "reasoning" } ], "microsoft/Phi-4-multimodal-instruct": [ { "name": "Phi-4 (multimodal)", "huggingface_id": "microsoft/Phi-4-multimodal-instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "multimodal" } ], "mistralai/Ministral-3-14B-Instruct-2512": [ { "name": "Ministral-3-Instruct", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-8B-Instruct-2512": [ { "name": "Ministral-3-Instruct (8b)", "huggingface_id": "mistralai/Ministral-3-8B-Instruct-2512", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "8b" } ], "mistralai/Ministral-3-3B-Instruct-2512": [ { "name": "Ministral-3-Instruct (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Instruct-2512", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Instruct-2512-FP8": [ { "name": "Ministral-3-Instruct (fp8)", "huggingface_id": "mistralai/Ministral-3-14B-Instruct-2512-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Ministral-3-8B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning", "huggingface_id": "mistralai/Ministral-3-8B-Reasoning-2512", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Ministral-3-3B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (3b)", "huggingface_id": "mistralai/Ministral-3-3B-Reasoning-2512", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "3b" } ], "mistralai/Ministral-3-14B-Reasoning-2512": [ { "name": "Ministral-3-Reasoning (14b)", "huggingface_id": "mistralai/Ministral-3-14B-Reasoning-2512", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "14b" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512": [ { "name": "Mistral-Large-3-675B-Instruct", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4": [ { "name": "Mistral-Large-3-675B-Instruct (nvfp4)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8": [ { "name": "Mistral-Large-3-675B-Instruct (fp8)", "huggingface_id": "mistralai/Mistral-Large-3-675B-Instruct-2512-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} }, "balance": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--mm-encoder-tp-mode": "data", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "mistralai/Mistral-Medium-3.5-128B": [ { "name": "Mistral-Medium-3.5", "huggingface_id": "mistralai/Mistral-Medium-3.5-128B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer_mode": "mistral", "--config_format": "mistral", "--load_format": "mistral", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"mistralai/Mistral-Medium-3.5-128B-EAGLE\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Small-4-119B-2603": [ { "name": "Mistral-Small-4-119B", "huggingface_id": "mistralai/Mistral-Small-4-119B-2603", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "FLASH_ATTN_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--attention-backend": "FLASH_ATTN_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Mistral-Small-4-119B-2603-NVFP4": [ { "name": "Mistral-Small-4-119B (nvfp4)", "huggingface_id": "mistralai/Mistral-Small-4-119B-2603-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--attention-backend": "TRITON_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--attention-backend": "TRITON_MLA", "--enable-auto-tool-choice": "", "--tool-call-parser": "mistral", "--reasoning-parser": "mistral", "--speculative-config": "{\"model\":\"mistralai/Mistral-Small-4-119B-2603-eagle\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "mistralai/Voxtral-4B-TTS-2603": [ { "name": "Voxtral-4B-TTS-2603", "huggingface_id": "mistralai/Voxtral-4B-TTS-2603", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "mistralai/Voxtral-Mini-4B-Realtime-2602": [ { "name": "Voxtral-Mini-4B-Realtime-2602", "huggingface_id": "mistralai/Voxtral-Mini-4B-Realtime-2602", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tokenizer-mode": "mistral", "--compilation_config": "{\"cudagraph_mode\": \"PIECEWISE\"}" }, "extra_envs": { "VLLM_DISABLE_COMPILE_CACHE": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Instruct": [ { "name": "Kimi-K2-Instruct", "huggingface_id": "moonshotai/Kimi-K2-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-K2-Thinking": [ { "name": "Kimi-K2-Thinking", "huggingface_id": "moonshotai/Kimi-K2-Thinking", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2-Thinking-NVFP4": [ { "name": "Kimi-K2-Thinking (nvfp4)", "huggingface_id": "nvidia/Kimi-K2-Thinking-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "kimi_k2", "--reasoning-parser": "kimi_k2", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.5": [ { "name": "Kimi-K2.5", "huggingface_id": "moonshotai/Kimi-K2.5", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.5-NVFP4": [ { "name": "Kimi-K2.5 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.5-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.6": [ { "name": "Kimi-K2.6", "huggingface_id": "moonshotai/Kimi-K2.6", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Kimi-K2.6-NVFP4": [ { "name": "Kimi-K2.6 (nvfp4)", "huggingface_id": "nvidia/Kimi-K2.6-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "moonshotai/Kimi-K2.7-Code": [ { "name": "Kimi-K2.7-Code", "huggingface_id": "moonshotai/Kimi-K2.7-Code", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "kimi_k2", "--enable-auto-tool-choice": "", "--reasoning-parser": "kimi_k2", "--mm-encoder-tp-mode": "data", "--speculative-config": "{\"model\":\"lightseekorg/kimi-k2.7-coder-eagle3.1-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "moonshotai/Kimi-Linear-48B-A3B-Instruct": [ { "name": "Kimi-Linear-48B-A3B-Instruct", "huggingface_id": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Nano": [ { "name": "Cosmos3-Nano", "huggingface_id": "nvidia/Cosmos3-Nano", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Image2Video": [ { "name": "Cosmos3-Super-Image2Video", "huggingface_id": "nvidia/Cosmos3-Super-Image2Video", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super-Text2Image": [ { "name": "Cosmos3-Super-Text2Image", "huggingface_id": "nvidia/Cosmos3-Super-Text2Image", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Cosmos3-Super": [ { "name": "Cosmos3-Super", "huggingface_id": "nvidia/Cosmos3-Super", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--host": "0.0.0.0", "--port": "8000", "--cfg-parallel-size": "2", "--ulysses-degree": "4", "--use-hsdp": "", "--hsdp-shard-size": "8", "--init-timeout": "1800" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-30B-A3B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP8": "1", "VLLM_FLASHINFER_MOE_BACKEND": "throughput" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16": [ { "name": "NVIDIA Nemotron-3-Nano-4B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "auto", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8": [ { "name": "NVIDIA Nemotron-3-Nano-4B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--async-scheduling": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser-plugin": "nano_v3_reasoning_parser.py", "--reasoning-parser": "nano_v3" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16": [ { "name": "NVIDIA Nemotron-3-Super-120B-A12B (base_bf16)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base_bf16" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4": [ { "name": "NVIDIA Nemotron-3-Ultra-550B-A55B (fp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } }, "balance": { "extra_args": { "--served-model-name": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--max-num-seqs": "16", "--gpu-memory-utilization": "0.90", "--max-num-batched-tokens": "32768", "--enable-flashinfer-autotune": "", "--async-scheduling": "", "--mamba-backend": "triton", "--mamba-ssm-cache-dtype": "float32", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":5}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": "1" } } }, "default_strategy": "latency-focus", "variant_name": "fp4" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD": [ { "name": "NVIDIA Nemotron-Nano-12B-v2-VL (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-NVFP4-QAD", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--mm-encoder-tp-mode": "data" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "opencv" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2": [ { "name": "NVIDIA Nemotron-Nano-9B-v2", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (fp8)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (nvfp4)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (base)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Base", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "base" } ], "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese": [ { "name": "NVIDIA Nemotron-Nano-9B-v2 (japanese)", "huggingface_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2-Japanese", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser-plugin": "nemotron_toolcall_parser_no_streaming.py", "--tool-call-parser": "nemotron_json" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "japanese" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (fp8)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": [ { "name": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning (nvfp4)", "huggingface_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--media-io-kwargs": "{\"video\": {\"num_frames\": 512, \"fps\": 1}}", "--video-pruning-rate": "0.5", "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder", "--reasoning-parser": "nemotron_v3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "openai/gpt-oss-120b": [ { "name": "GPT-OSS", "huggingface_id": "openai/gpt-oss-120b", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "amd/gpt-oss-120b-w-mxfp4-a-fp8": [ { "name": "GPT-OSS (amd_fp8)", "huggingface_id": "amd/gpt-oss-120b-w-mxfp4-a-fp8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--speculative-config": "{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"num_speculative_tokens\":7,\"method\":\"eagle3\",\"draft_tensor_parallel_size\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "amd_fp8" } ], "openai/gpt-oss-20b": [ { "name": "GPT-OSS 20B", "huggingface_id": "openai/gpt-oss-20b", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "openai", "--enable-auto-tool-choice": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/MiniCPM-V-4.6": [ { "name": "MiniCPM-V 4.6", "huggingface_id": "openbmb/MiniCPM-V-4.6", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "qwen3_coder" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/MiniCPM5-1B": [ { "name": "MiniCPM5-1B", "huggingface_id": "openbmb/MiniCPM5-1B", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "openbmb/VoxCPM2": [ { "name": "VoxCPM2", "huggingface_id": "openbmb/VoxCPM2", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "pfnet/plamo-2-translate": [ { "name": "PLaMo 2 Translate", "huggingface_id": "pfnet/plamo-2-translate", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "pfnet/plamo-3-nict-31b-base": [ { "name": "PLaMo 3 NICT 31B Base", "huggingface_id": "pfnet/plamo-3-nict-31b-base", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-M.1": [ { "name": "Laguna M.1", "huggingface_id": "poolside/Laguna-M.1", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-M.1-FP8": [ { "name": "Laguna M.1 (fp8)", "huggingface_id": "poolside/Laguna-M.1-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-M.1-NVFP4": [ { "name": "Laguna M.1 (nvfp4)", "huggingface_id": "poolside/Laguna-M.1-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}" }, "extra_envs": {} }, "balance": { "extra_args": { "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--default-chat-template-kwargs": "{\"enable_thinking\": true}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS-2.1": [ { "name": "Laguna XS-2.1", "huggingface_id": "poolside/Laguna-XS-2.1", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS-2.1-FP8": [ { "name": "Laguna XS-2.1 (fp8)", "huggingface_id": "poolside/Laguna-XS-2.1-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS-2.1-NVFP4": [ { "name": "Laguna XS-2.1 (nvfp4)", "huggingface_id": "poolside/Laguna-XS-2.1-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS-2.1-INT4": [ { "name": "Laguna XS-2.1 (int4)", "huggingface_id": "poolside/Laguna-XS-2.1-INT4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS-2.1-DFlash\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}", "--moe-backend": "triton", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "poolside/Laguna-XS.2": [ { "name": "Laguna XS.2", "huggingface_id": "poolside/Laguna-XS.2", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "poolside/Laguna-XS.2-FP8": [ { "name": "Laguna XS.2 (fp8)", "huggingface_id": "poolside/Laguna-XS.2-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": "0" } } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "poolside/Laguna-XS.2-NVFP4": [ { "name": "Laguna XS.2 (nvfp4)", "huggingface_id": "poolside/Laguna-XS.2-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "poolside/Laguna-XS.2-INT4": [ { "name": "Laguna XS.2 (int4)", "huggingface_id": "poolside/Laguna-XS.2-INT4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "poolside_v1", "--reasoning-parser": "poolside_v1", "--speculative-config": "{\"model\":\"poolside/Laguna-XS.2-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stabilityai/stable-audio-open-1.0": [ { "name": "Stable Audio Open", "huggingface_id": "stabilityai/stable-audio-open-1.0", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enforce-eager": "", "--gpu-memory-utilization": "0.9" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stabilityai/stable-diffusion-3.5-medium": [ { "name": "Stable Diffusion 3.5", "huggingface_id": "stabilityai/stable-diffusion-3.5-medium", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stabilityai/stable-diffusion-3.5-large": [ { "name": "Stable Diffusion 3.5 (large)", "huggingface_id": "stabilityai/stable-diffusion-3.5-large", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "large" } ], "stabilityai/stable-diffusion-3.5-large-turbo": [ { "name": "Stable Diffusion 3.5 (large_turbo)", "huggingface_id": "stabilityai/stable-diffusion-3.5-large-turbo", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "large_turbo" } ], "stepfun-ai/Step-3.5-Flash": [ { "name": "Step-3.5-Flash", "huggingface_id": "stepfun-ai/Step-3.5-Flash", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.5-Flash-FP8": [ { "name": "Step-3.5-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.5-Flash-INT4": [ { "name": "Step-3.5-Flash (int4)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int4" } ], "stepfun-ai/Step-3.5-Flash-INT8": [ { "name": "Step-3.5-Flash (int8)", "huggingface_id": "stepfun-ai/Step-3.5-Flash-INT8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--hf-overrides": "{\"num_nextn_predict_layers\": 1}", "--speculative-config": "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "int8" } ], "stepfun-ai/Step-3.7-Flash": [ { "name": "Step-3.7-Flash", "huggingface_id": "stepfun-ai/Step-3.7-Flash", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "stepfun-ai/Step-3.7-Flash-FP8": [ { "name": "Step-3.7-Flash (fp8)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "stepfun-ai/Step-3.7-Flash-NVFP4": [ { "name": "Step-3.7-Flash (nvfp4)", "huggingface_id": "stepfun-ai/Step-3.7-Flash-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--disable-cascade-attn": "", "--quantization": "modelopt", "--kv-cache-dtype": "fp8", "--gpu-memory-utilization": "0.9", "--async-scheduling": "", "--tool-call-parser": "step3p5", "--enable-auto-tool-choice": "", "--reasoning-parser": "step3p5", "--speculative-config": "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "tencent/Hunyuan-A13B-Instruct": [ { "name": "Hunyuan-A13B-Instruct", "huggingface_id": "tencent/Hunyuan-A13B-Instruct", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hunyuan-A13B-Instruct-FP8": [ { "name": "Hunyuan-A13B-Instruct (fp8)", "huggingface_id": "tencent/Hunyuan-A13B-Instruct-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "tencent/HunyuanOCR": [ { "name": "HunyuanOCR", "huggingface_id": "tencent/HunyuanOCR", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-preview": [ { "name": "Hy3-preview", "huggingface_id": "tencent/Hy3-preview", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":1}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3": [ { "name": "Hy3", "huggingface_id": "tencent/Hy3", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "tencent/Hy3-FP8": [ { "name": "Hy3 (fp8)", "huggingface_id": "tencent/Hy3-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" }, "extra_envs": {} }, "balance": { "extra_args": { "--tool-call-parser": "hy_v3", "--enable-auto-tool-choice": "", "--reasoning-parser": "hy_v3", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":2}", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "thinkingmachines/Inkling-NVFP4": [ { "name": "TML Inkling", "huggingface_id": "thinkingmachines/Inkling-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "thinkingmachines/Inkling": [ { "name": "TML Inkling (bf16)", "huggingface_id": "thinkingmachines/Inkling", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--tokenizer-mode": "inkling", "--kernel-config.enable_flashinfer_autotune=False": "", "--enable-auto-tool-choice": "", "--tool-call-parser": "inkling", "--reasoning-parser": "inkling", "--speculative-config": "{\"method\":\"mtp\",\"num_speculative_tokens\":8}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-4.5": [ { "name": "GLM-4.5", "huggingface_id": "zai-org/GLM-4.5", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5-FP8": [ { "name": "GLM-4.5 (fp8)", "huggingface_id": "zai-org/GLM-4.5-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.5V": [ { "name": "GLM-4.5V", "huggingface_id": "zai-org/GLM-4.5V", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.5V-FP8": [ { "name": "GLM-4.5V (fp8)", "huggingface_id": "zai-org/GLM-4.5V-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6": [ { "name": "GLM-4.6", "huggingface_id": "zai-org/GLM-4.6", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6-FP8": [ { "name": "GLM-4.6 (fp8)", "huggingface_id": "zai-org/GLM-4.6-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.6V": [ { "name": "GLM-4.6V", "huggingface_id": "zai-org/GLM-4.6V", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.6V-FP8": [ { "name": "GLM-4.6V (fp8)", "huggingface_id": "zai-org/GLM-4.6V-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--enable-expert-parallel": "", "--allowed-local-media-path": "/", "--mm-encoder-tp-mode": "data", "--mm-processor-cache-type": "shm", "--tool-call-parser": "glm45", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "zai-org/GLM-4.7": [ { "name": "GLM-4.7", "huggingface_id": "zai-org/GLM-4.7", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-4.7-FP8": [ { "name": "GLM-4.7 (fp8)", "huggingface_id": "zai-org/GLM-4.7-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-4.7-NVFP4": [ { "name": "GLM-4.7 (nvfp4)", "huggingface_id": "nvidia/GLM-4.7-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.1": [ { "name": "GLM-5.1", "huggingface_id": "zai-org/GLM-5.1", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5.1-FP8": [ { "name": "GLM-5.1 (fp8)", "huggingface_id": "zai-org/GLM-5.1-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5.1-NVFP4": [ { "name": "GLM-5.1 (nvfp4)", "huggingface_id": "nvidia/GLM-5.1-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2-FP8": [ { "name": "GLM-5.2", "huggingface_id": "zai-org/GLM-5.2-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/GLM-5.2-NVFP4": [ { "name": "GLM-5.2 (nvfp4)", "huggingface_id": "nvidia/GLM-5.2-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-5.2": [ { "name": "GLM-5.2 (bf16)", "huggingface_id": "zai-org/GLM-5.2", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5" }, "extra_envs": {} }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8_e4m3", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "5", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "bf16" } ], "zai-org/GLM-5": [ { "name": "GLM-5", "huggingface_id": "zai-org/GLM-5", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-5-FP8": [ { "name": "GLM-5 (fp8)", "huggingface_id": "zai-org/GLM-5-FP8", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": {} }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "fp8" } ], "nvidia/GLM-5-NVFP4": [ { "name": "GLM-5 (nvfp4)", "huggingface_id": "nvidia/GLM-5-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } }, "balance": { "extra_args": { "--trust-remote-code": "", "--chat-template-content-format=string": "", "--kv-cache-dtype": "fp8", "--tool-call-parser": "glm47", "--enable-auto-tool-choice": "", "--reasoning-parser": "glm45", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "3", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_FLASHINFER_MOE_FP4": "1" } } }, "default_strategy": "latency-focus", "variant_name": "nvfp4" } ], "zai-org/GLM-ASR-Nano-2512": [ { "name": "GLM-ASR-Nano-2512", "huggingface_id": "zai-org/GLM-ASR-Nano-2512", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": {}, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-GA": [ { "name": "GLM-GA", "huggingface_id": "zai-org/GLM-GA", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "", "--allowed-local-media-path": "/", "--mm-processor-cache-type": "shm", "--reasoning-parser": "glm45" }, "extra_envs": { "VLLM_VIDEO_LOADER_BACKEND": "glm4_6v" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-Image": [ { "name": "GLM-Image", "huggingface_id": "zai-org/GLM-Image", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-OCR": [ { "name": "GLM-OCR", "huggingface_id": "zai-org/GLM-OCR", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--mm-encoder-tp-mode": "data", "--speculative-config.method": "mtp", "--speculative-config.num_speculative_tokens": "1" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/GLM-TTS": [ { "name": "GLM-TTS", "huggingface_id": "zai-org/GLM-TTS", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--omni": "", "--trust-remote-code": "" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "zai-org/Glyph": [ { "name": "Glyph", "huggingface_id": "zai-org/Glyph", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--no-enable-prefix-caching": "", "--mm-processor-cache-gb": "0", "--limit-mm-per-prompt.video": "0", "--reasoning-parser": "glm45", "--mm-encoder-tp-mode": "data" }, "extra_envs": {} } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-26B-A4B-NVFP4": [ { "name": "Gemma 4 26B-A4B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-26B-A4B-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } }, "balance": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4}", "--enable-expert-parallel": "" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ], "nvidia/Gemma-4-31B-IT-NVFP4": [ { "name": "Gemma 4 31B IT (NVFP4)", "huggingface_id": "nvidia/Gemma-4-31B-IT-NVFP4", "accelerators": "nvidia", "provider": "vllm", "custom_image": "vllm/vllm-openai:v0.25.1", "strategies": { "latency-focus": { "extra_args": { "--kv-cache-dtype": "fp8", "--enable-auto-tool-choice": "", "--tool-call-parser": "gemma4", "--chat-template": "/vllm-workspace/examples/tool_chat_template_gemma4.jinja", "--reasoning-parser": "gemma4", "--speculative-config": "{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":4}" }, "extra_envs": { "VLLM_USE_V2_MODEL_RUNNER": "1" } } }, "default_strategy": "latency-focus", "variant_name": "default" } ] } } }