RBLN NPU에서 Qwen3-VL-4B-Instruct 서빙¶

{"attn": "flash_attn", "axes": [{"axis": "lmcache", "options": [{"label": "Off", "value": "off"}, {"label": "LMCache (CPU)", "value": "cpu"}, {"label": "LMCache (RDS)", "value": "rds"}], "title": "KV Offload"}, {"axis": "kv_connector", "options": [{"label": "Off", "value": "off"}, {"label": "LMCache (CPU)", "value": "lmcache_cpu"}, {"label": "LMCache (RDS)", "value": "lmcache_rds"}], "title": "KV Offload"}, {"axis": "prefix_caching", "options": [{"label": "On", "value": "on"}, {"label": "On (sub-block)", "value": "on_sub_block"}, {"label": "Off", "value": "off"}], "title": "Prefix Caching"}, {"axis": "spec_decode", "options": [{"label": "Off", "value": "off"}, {"label": "N-gram", "value": "ngram"}, {"label": "EAGLE-3", "value": "eagle3"}, {"label": "Suffix", "value": "suffix"}, {"label": "MTP", "value": "mtp"}], "title": "Speculative Decoding"}, {"axis": "on_device_sampler", "options": [{"label": "On", "value": "on"}, {"label": "Off", "value": "off"}], "title": "On-device Sampler"}, {"axis": "structured_output", "options": [{"label": "On", "value": "on"}, {"label": "Off", "value": "off"}], "title": "Structured Output"}], "badges": ["Dense", "4B", "262,144 ctx", {"hl": true, "text": "vllm-rbln"}, "flash_attn", "Multimodal"], "client": "curl http://localhost:8000/v1/chat/completions \\\n    -H \"Content-Type: application/json\" \\\n    -d \u0027{\n      \"model\": \"Qwen3-VL-4B-Instruct\",\n      \"messages\": [{\"role\": \"user\", \"content\": [\n        {\"type\": \"text\", \"text\": \"What is in this image?\"},\n        {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://rbln-public.s3.ap-northeast-2.amazonaws.com/images/tabby.jpg\"}}\n      ]}],\n      \"max_tokens\": 64\n    }\u0027", "features": {"api": true, "mm": true, "text": true}, "hf": "https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct", "installLine": "pip install rebel-compiler vllm-rbln", "model": "Qwen/Qwen3-VL-4B-Instruct", "name": "Qwen3-VL-4B-Instruct", "provider": "Qwen", "strategies": [{"label": "RSD", "nodes": null, "options": [{"attn": "flash_attn", "features": {}, "label": null, "modelImpl": "transformers", "note": null, "npus": 1, "roles": [{"cmd": "vllm serve Qwen/Qwen3-VL-4B-Instruct \\\n    --additional-config \u0027{\n      \"optimum_overrides\": {\n        \"visual\": {\n          \"num_devices\": 1,\n          \"max_seq_len\": 16384\n        }\n      }\n    }\u0027 \\\n    --block-size 8192 \\\n    --max-model-len 262144 \\\n    --max-num-seqs 1 \\\n    --mm-processor-kwargs \u0027{\n      \"min_pixels\": 65536,\n      \"max_pixels\": 4194304\n    }\u0027 \\\n    --model-impl transformers \\\n    --rbln-num-devices-per-local-rank 1 \\\n    --served-model-name Qwen3-VL-4B-Instruct", "role": "main"}], "skus": ["RBLN-CR13"], "target": null, "verified": "2026-10-16"}, {"attn": "flash_attn", "features": {}, "label": null, "modelImpl": "transformers", "note": null, "npus": 8, "roles": [{"cmd": "vllm serve Qwen/Qwen3-VL-4B-Instruct \\\n    --additional-config \u0027{\n      \"optimum_overrides\": {\n        \"visual\": {\n          \"num_devices\": 8,\n          \"max_seq_len\": 16384\n        }\n      }\n    }\u0027 \\\n    --block-size 16384 \\\n    --max-model-len 262144 \\\n    --max-num-seqs 1 \\\n    --mm-processor-kwargs \u0027{\n      \"min_pixels\": 65536,\n      \"max_pixels\": 4194304\n    }\u0027 \\\n    --model-impl transformers \\\n    --rbln-num-devices-per-local-rank 8 \\\n    --served-model-name Qwen3-VL-4B-Instruct", "role": "main"}], "skus": ["RBLN-CA22", "RBLN-CA25"], "target": null, "verified": "2026-10-16"}], "slug": "rsd", "tag": "Aggregated", "variant": "BF16"}], "text": {"en": {"strategyNote": "Rebellions Scalable Design (RSD): the RBLN Compiler partitions the model across NPU devices, or across the chiplets within one device. See \u003ca href=\"../features/model-parallelism.html#rebellions-scalable-design-rsd\"\u003eModel Parallelism\u003c/a\u003e for how RSD applies on each NPU, and \u003cstrong\u003eConfiguration\u003c/strong\u003e in the guide for the engine arguments.", "sub": "Qwen3-VL-4B-Instruct \u2014 vision-language model served on RBLN NPUs with RBLN vLLM (262,144-token context); accepts images and text through an OpenAI-compatible chat API.", "tag": "The vllm serve command compiles the model on first run and starts an OpenAI-compatible server."}, "ko": {"strategyNote": "Rebellions Scalable Design(RSD): RBLN \ucef4\ud30c\uc77c\ub7ec\uac00 \ubaa8\ub378\uc744 NPU \ub514\ubc14\uc774\uc2a4 \uac04 \ub610\ub294 \ub514\ubc14\uc774\uc2a4 \ud558\ub098\uc758 \uce69\ub81b \uac04\uc5d0 \ubd84\ud560\ud569\ub2c8\ub2e4. NPU\ubcc4 RSD \uc801\uc6a9 \ubc29\uc2dd\uc740 \u003ca href=\"../features/model-parallelism.html#rebellions-scalable-design-rsd\"\u003e\ubaa8\ub378 \ubcd1\ub82c\ud654\u003c/a\u003e\uc5d0\uc11c, \uc5d4\uc9c4 \uc778\uc790\ub294 \uac00\uc774\ub4dc\uc758 \u003cstrong\u003eConfiguration\u003c/strong\u003e\uc5d0\uc11c \ud655\uc778\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.", "sub": "Qwen3-VL-4B-Instruct \u2014 \ube44\uc804-\uc5b8\uc5b4 \ubaa8\ub378, RBLN vLLM\uc73c\ub85c RBLN NPU\uc5d0\uc11c \uc11c\ube59(262,144 \ud1a0\ud070 \ucee8\ud14d\uc2a4\ud2b8). OpenAI \ud638\ud658 chat API\ub85c \uc774\ubbf8\uc9c0\uc640 \ud14d\uc2a4\ud2b8\ub97c \ud568\uaed8 \uc785\ub825\ud569\ub2c8\ub2e4.", "tag": "vllm serve \uba85\ub839\uc740 \ucd5c\ucd08 \uc2e4\ud589 \uc2dc \ubaa8\ub378\uc744 \ucef4\ud30c\uc77c\ud558\uace0 OpenAI \ud638\ud658 \uc11c\ubc84\ub97c \uc2dc\uc791\ud569\ub2c8\ub2e4."}}, "variant": "BF16"}

1
2
3

$ pip install \
  --extra-index-url https://pypi.rbln.ai/simple \
  rebel-compiler==0.12.0

1
2
3
4

$ pip install \
  --extra-index-url https://wheels.vllm.ai/0.30.0/cpu \
  --extra-index-url https://download.pytorch.org/whl/cpu \
  vllm-rbln==0.12.0