From 782ba070fe4ee73bce632c39303d319de0ba56d7 Mon Sep 17 00:00:00 2001 From: Artem Vasenin Date: Wed, 8 Jul 2026 12:04:41 +0200 Subject: [PATCH 1/6] Add class for Qwen3-Omni --- tools/llm_bench/llm_bench_utils/config_class.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/llm_bench/llm_bench_utils/config_class.py b/tools/llm_bench/llm_bench_utils/config_class.py index 8dbfb51ece..be8d9e2eb6 100644 --- a/tools/llm_bench/llm_bench_utils/config_class.py +++ b/tools/llm_bench/llm_bench_utils/config_class.py @@ -19,6 +19,7 @@ OVDiffusionPipeline, OVModelForSpeechSeq2Seq, OVModelForVisualCausalLM, + OVModelForMultimodalLM, OVPipelineForInpainting, OVPipelineForImage2Image, OVModelForFeatureExtraction, @@ -171,7 +172,8 @@ class UseCaseTextToSpeech(UseCase): "videochat-flash-qwen", "gemma4", ] - ) + ), + UseCaseVLM(["qwen3-omni"], ov_cls=OVModelForMultimodalLM), ], "speech_to_text": [UseCaseSpeech2Text(["whisper"])], "image_cls": [UseCaseImageCls(["vit"])], From 86026645fda646834fcf6d298b5eb31df69eae1c Mon Sep 17 00:00:00 2001 From: Artem Vasenin Date: Wed, 8 Jul 2026 13:52:51 +0200 Subject: [PATCH 2/6] Add tiny random tests --- tests/python_tests/samples/conftest.py | 4 +++ .../samples/test_tools_llm_benchmark.py | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/tests/python_tests/samples/conftest.py b/tests/python_tests/samples/conftest.py index 045f97a9bd..23cd070768 100644 --- a/tests/python_tests/samples/conftest.py +++ b/tests/python_tests/samples/conftest.py @@ -197,6 +197,10 @@ "name": "optimum-intel-internal-testing/tiny-random-qwen3-vl", "convert_args": ["--trust-remote-code", "--task", "image-text-to-text"], }, + "tiny-random-qwen3-omni": { + "name": "optimum-intel-internal-testing/tiny-random-qwen3-omni", + "convert_args": ["--trust-remote-code", "--task", "image-text-to-text"], + }, } TEST_FILES = { diff --git a/tests/python_tests/samples/test_tools_llm_benchmark.py b/tests/python_tests/samples/test_tools_llm_benchmark.py index 9e4623362b..3d5f7e0045 100644 --- a/tests/python_tests/samples/test_tools_llm_benchmark.py +++ b/tests/python_tests/samples/test_tools_llm_benchmark.py @@ -4,6 +4,7 @@ import os import pytest import sys +from importlib import metadata from pathlib import Path from test_utils import run_sample @@ -11,6 +12,9 @@ from utils.hugging_face import download_gguf_model from conftest import SAMPLES_PY_DIR, convert_model, download_test_content + +_TRANSFORMERS_VERSION = tuple(int(p) for p in metadata.version("transformers").split(".")[:2]) + convert_draft_model = convert_model download_mask_image = download_test_content @@ -615,3 +619,31 @@ def test_python_tool_llm_benchmark_video_gen_json( generate_llm_bench_input_generation_jsonl, ] + sample_args run_sample(benchmark_py_command) + + @pytest.mark.samples + @pytest.mark.skipif( + _TRANSFORMERS_VERSION < (5, 1), + reason="Requires transformers >= 5.1", + ) + @pytest.mark.parametrize("download_test_content", ["cat.png"], indirect=True) + @pytest.mark.parametrize("convert_model", ["tiny-random-qwen3-omni"], indirect=True) + @pytest.mark.parametrize( + "sample_args", + [ + ["-d", "cpu", "-n", "1", "--num_steps", "4", "--task", "visual_text_gen", "--optimum"], + ["-d", "cpu", "-n", "1", "--num_steps", "4", "--task", "visual_text_gen", "--genai"], + ], + ) + def test_python_tool_llm_benchmark_visual_text_gen(self, download_test_content, convert_model, sample_args): + benchmark_script = SAMPLES_PY_DIR / "llm_bench/benchmark.py" + benchmark_py_command = [ + sys.executable, + benchmark_script, + "-m", + convert_model, + "--media", + download_test_content, + "--prompt", + "What animal is in this image?", + ] + sample_args + run_sample(benchmark_py_command) From 50be29f8adb7f82ae1f4c7c5c2d6ebf5df6c3c79 Mon Sep 17 00:00:00 2001 From: Artem Vasenin Date: Wed, 8 Jul 2026 14:02:04 +0200 Subject: [PATCH 3/6] Update transformers version check --- tests/python_tests/samples/conftest.py | 16 ++++++++++++++++ .../samples/test_tools_llm_benchmark.py | 7 ++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/python_tests/samples/conftest.py b/tests/python_tests/samples/conftest.py index 23cd070768..d7deeda8b3 100644 --- a/tests/python_tests/samples/conftest.py +++ b/tests/python_tests/samples/conftest.py @@ -25,6 +25,22 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + +def get_transformers_version(): + """Return (major, minor) of the installed transformers, or None if unavailable.""" + try: + version = metadata.version("transformers") + except metadata.PackageNotFoundError: + return None + try: + major, minor = version.split(".")[:2] + return int(major), int(minor) + except ValueError: + return None + + +TRANSFORMERS_VERSION = get_transformers_version() + # Dictionary containing model configurations. # Each key is a model identifier, and the value is a dictionary with: # - "name": the model's name or path diff --git a/tests/python_tests/samples/test_tools_llm_benchmark.py b/tests/python_tests/samples/test_tools_llm_benchmark.py index 3d5f7e0045..77cebc0c68 100644 --- a/tests/python_tests/samples/test_tools_llm_benchmark.py +++ b/tests/python_tests/samples/test_tools_llm_benchmark.py @@ -4,17 +4,14 @@ import os import pytest import sys -from importlib import metadata from pathlib import Path from test_utils import run_sample from data.models import GGUF_MODEL_LIST from utils.hugging_face import download_gguf_model -from conftest import SAMPLES_PY_DIR, convert_model, download_test_content +from conftest import SAMPLES_PY_DIR, convert_model, download_test_content, TRANSFORMERS_VERSION -_TRANSFORMERS_VERSION = tuple(int(p) for p in metadata.version("transformers").split(".")[:2]) - convert_draft_model = convert_model download_mask_image = download_test_content @@ -622,7 +619,7 @@ def test_python_tool_llm_benchmark_video_gen_json( @pytest.mark.samples @pytest.mark.skipif( - _TRANSFORMERS_VERSION < (5, 1), + TRANSFORMERS_VERSION is None or TRANSFORMERS_VERSION < (5, 1), reason="Requires transformers >= 5.1", ) @pytest.mark.parametrize("download_test_content", ["cat.png"], indirect=True) From 7874f92654e0875c644ebd540f2c67a5101e0984 Mon Sep 17 00:00:00 2001 From: Artem Vasenin Date: Wed, 8 Jul 2026 16:59:29 +0200 Subject: [PATCH 4/6] Add dedicated test in CI --- .github/workflows/linux.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4089ee0fe9..7ad05efaf4 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -756,6 +756,12 @@ jobs: python -m pytest -v ./tests/python_tests/test_continuous_batching.py -m "transformers_lower_v5 or transformers_dependent" ./tests/python_tests/test_generation_config.py ./tests/python_tests/test_sampling.py ./tests/python_tests/test_text_streamer.py run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching.test || fromJSON(needs.smart_ci.outputs.affected_components).sampling.test || fromJSON(needs.smart_ci.outputs.affected_components).text_streamer.test }} timeout: 60 + - name: 'Qwen3-Omni llm_bench tests' + cmd: | + python -m pip install transformers==5.1 + python -m pytest -s -v tests/python_tests/samples/test_tools_llm_benchmark.py --override-ini cache_dir=/mount/caches/pytest/ -k "qwen3-omni" + run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test || fromJSON(needs.smart_ci.outputs.affected_components).llm_bench.test }} + timeout: 60 defaults: run: shell: bash From 375f9726568d77d9f4974e15f99a4abf78d54584 Mon Sep 17 00:00:00 2001 From: Artem Vasenin Date: Wed, 8 Jul 2026 22:03:22 +0200 Subject: [PATCH 5/6] Move the test to tools run --- .github/workflows/linux.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7ad05efaf4..e13b9468a4 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -756,12 +756,6 @@ jobs: python -m pytest -v ./tests/python_tests/test_continuous_batching.py -m "transformers_lower_v5 or transformers_dependent" ./tests/python_tests/test_generation_config.py ./tests/python_tests/test_sampling.py ./tests/python_tests/test_text_streamer.py run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching.test || fromJSON(needs.smart_ci.outputs.affected_components).sampling.test || fromJSON(needs.smart_ci.outputs.affected_components).text_streamer.test }} timeout: 60 - - name: 'Qwen3-Omni llm_bench tests' - cmd: | - python -m pip install transformers==5.1 - python -m pytest -s -v tests/python_tests/samples/test_tools_llm_benchmark.py --override-ini cache_dir=/mount/caches/pytest/ -k "qwen3-omni" - run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test || fromJSON(needs.smart_ci.outputs.affected_components).llm_bench.test }} - timeout: 60 defaults: run: shell: bash @@ -1080,6 +1074,14 @@ jobs: python -m pytest -vs ${{ env.SRC_DIR }}/tests/python_tests/samples/test_tools_llm_benchmark.py -m "samples and transformers_lower_v5" env: SAMPLES_PY_DIR: "${{ env.SRC_DIR }}/tools" + - name: Test LLM Benchmark Tools with Qwen3-Omni model + if: ${{ fromJSON(needs.smart_ci.outputs.affected_components).llm_bench }} + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + python -m pip install transformers==5.1 + python -m pytest -vs ${{ env.SRC_DIR }}/tests/python_tests/samples/test_tools_llm_benchmark.py -k "qwen3-omni" + env: + SAMPLES_PY_DIR: "${{ env.SRC_DIR }}/tools" genai_nodejs_tests: name: Node.js bindings tests From 712b54f90bb3ccbb4a89829132300eae77ab3a7b Mon Sep 17 00:00:00 2001 From: Artem Vasenin Date: Thu, 9 Jul 2026 12:10:00 +0200 Subject: [PATCH 6/6] Add check for other optimum-intel versions --- tools/llm_bench/llm_bench_utils/config_class.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/llm_bench/llm_bench_utils/config_class.py b/tools/llm_bench/llm_bench_utils/config_class.py index 56b822fd03..2289d3a799 100644 --- a/tools/llm_bench/llm_bench_utils/config_class.py +++ b/tools/llm_bench/llm_bench_utils/config_class.py @@ -19,7 +19,6 @@ OVDiffusionPipeline, OVModelForSpeechSeq2Seq, OVModelForVisualCausalLM, - OVModelForMultimodalLM, OVPipelineForInpainting, OVPipelineForImage2Image, OVModelForFeatureExtraction, @@ -173,7 +172,6 @@ class UseCaseTextToSpeech(UseCase): "gemma4", ] ), - UseCaseVLM(["qwen3-omni"], ov_cls=OVModelForMultimodalLM), ], "speech_to_text": [UseCaseSpeech2Text(["whisper"])], "image_cls": [UseCaseImageCls(["vit"])], @@ -253,5 +251,12 @@ class UseCaseTextToSpeech(UseCase): "text_to_speech": [UseCaseTextToSpeech(["speecht5", "kokoro"])], } +try: + from optimum.intel.openvino import OVModelForMultimodalLM +except ImportError: + pass +else: + USE_CASES["visual_text_gen"].append(UseCaseVLM(["qwen3-omni"], ov_cls=OVModelForMultimodalLM)) + PA_ATTENTION_BACKEND = "PA" SDPA_ATTENTION_BACKEND = "SDPA"