Qwen2.5-VL 7B¶

from optimum.rbln import RBLNQwen2_5_VLForConditionalGeneration

model_id = "Qwen/Qwen2.5-VL-7B-Instruct"
model = RBLNQwen2_5_VLForConditionalGeneration.from_pretrained(
    model_id,
    export=True,
    rbln_config={
        "visual": {
            "max_seq_len": 6400,
            "device": 0,
        },
        "num_devices": 8,
        "kvcache_partition_len": 16_384,
        "max_seq_len": 114_688,
        "device": [0, 1, 2, 3, 4, 5, 6, 7],
        "batch_size": 2,
        "decoder_batch_sizes": [2, 1],
    },
)
model.save_pretrained("rbln-Qwen2-5-7B-Instruct")

from qwen_vl_utils import process_vision_info
from transformers import AutoProcessor, AutoTokenizer
from vllm import LLM, SamplingParams

# If the video is too long
# set `VLLM_ENGINE_ITERATION_TIMEOUT_S` to a higher timeout value.
VIDEO_URLS = [
    "https://duguang-labelling.oss-cn-shanghai.aliyuncs.com/qiansun/video_ocr/videos/50221078283.mp4",
]

def generate_prompts_video(model_id: str):
    processor = AutoProcessor.from_pretrained(model_id, padding_side="left")
    video_nums = len(VIDEO_URLS)
    messages = [[
        {
            "role":
            "user",
            "content": [
                {
                    "type": "video",
                    "video": VIDEO_URLS[i],
                },
                {
                    "type": "text",
                    "text": "Describe this video."
                },
            ],
        },
    ] for i in range(video_nums)]

    texts = processor.apply_chat_template(
        messages,
        add_generation_prompt=True,
        tokenize=False,
    )

    arr_video_inputs = []
    arr_video_kwargs = []
    for i in range(video_nums):
        image_inputs, video_inputs, video_kwargs = process_vision_info(
            messages[i], return_video_kwargs=True)
        arr_video_inputs.append(video_inputs)
        arr_video_kwargs.append(video_kwargs)

    return [{
        "prompt": text,
        "multi_modal_data": {
            "video": video_inputs,
        },
        "mm_processor_kwargs": {
            "min_pixels": 1024 * 14 * 14,
            "max_pixels": 5120 * 14 * 14,
            **video_kwargs,
        },
    } for text, video_inputs, video_kwargs in zip(
        texts, arr_video_inputs, arr_video_kwargs)]

def main():
    model_id = "rbln-Qwen2-5-7B-Instruct"
    llm = LLM(model=model_id)

    tokenizer = AutoTokenizer.from_pretrained(model_id)
    sampling_params = SamplingParams(
        temperature=0,
        ignore_eos=False,
        skip_special_tokens=True,
        stop_token_ids=[tokenizer.eos_token_id],
        max_tokens=200
    )

    inputs = generate_prompts_video(model_id)
    outputs = llm.generate(inputs, sampling_params)

    for output in outputs:
        prompt = output.prompt
        generated_text = output.outputs[0].text
        print(generated_text)

if __name__ == "__main__":
    main()

The video showcases a clear plastic food container being used to hold various fruits, demonstrating its features and benefits.The container is placed on a wooden table with a decorative background that includes a bouquet of artificial flowers and a plate with a mango and some berries.


The video begins with a close-up of the container filled with peaches, accompanied by text highlighting its versatility for different types of fruits such as longan, sliced watermelon, strawberries, and cherries. The container is then shown being opened and closed, emphasizing its easy-to-use design and secure locking mechanism.


Next, the container is filled with cherries, and the text explains that it is made of PET material, ensuring durability and quality. The container is then subjected to a durability test by placing two bricks on top of it, demonstrating its strength and ability to withstand pressure.


The video concludes with a final shot of the container filled with peaches, showcasing its transparency and the clear view of the contents inside. The text reiterates the container

from qwen_vl_utils import process_vision_info
from transformers import AutoProcessor, AutoTokenizer
from vllm import LLM, SamplingParams

# VIDEO_URLS와 generate_prompts_video()는 동일합니다.

def main():
    model_id = "Qwen/Qwen2.5-VL-7B-Instruct"
    llm = LLM(
        model=model_id,
        block_size=16_384,
        max_model_len=114_688,
        max_num_seqs=2,
        additional_config={
            "num_devices_per_local_rank": 8,
            "optimum_overrides": {
                "visual": {
                    "max_seq_len": 6400,
                    "device": 0,
                },
                "decoder_batch_sizes": [2, 1],
                "device": [0, 1, 2, 3, 4, 5, 6, 7],
            }
        }
    )

    tokenizer = AutoTokenizer.from_pretrained(model_id)
    # ... (이하 main()의 나머지는 동일)

if __name__ == "__main__":
    main()