diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4089ee0fe9..e13b9468a4 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1074,6 +1074,14 @@ jobs: python -m pytest -vs ${{ env.SRC_DIR }}/tests/python_tests/samples/test_tools_llm_benchmark.py -m "samples and transformers_lower_v5" env: SAMPLES_PY_DIR: "${{ env.SRC_DIR }}/tools" + - name: Test LLM Benchmark Tools with Qwen3-Omni model + if: ${{ fromJSON(needs.smart_ci.outputs.affected_components).llm_bench }} + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + python -m pip install transformers==5.1 + python -m pytest -vs ${{ env.SRC_DIR }}/tests/python_tests/samples/test_tools_llm_benchmark.py -k "qwen3-omni" + env: + SAMPLES_PY_DIR: "${{ env.SRC_DIR }}/tools" genai_nodejs_tests: name: Node.js bindings tests diff --git a/tests/python_tests/samples/conftest.py b/tests/python_tests/samples/conftest.py index 045f97a9bd..d7deeda8b3 100644 --- a/tests/python_tests/samples/conftest.py +++ b/tests/python_tests/samples/conftest.py @@ -25,6 +25,22 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + +def get_transformers_version(): + """Return (major, minor) of the installed transformers, or None if unavailable.""" + try: + version = metadata.version("transformers") + except metadata.PackageNotFoundError: + return None + try: + major, minor = version.split(".")[:2] + return int(major), int(minor) + except ValueError: + return None + + +TRANSFORMERS_VERSION = get_transformers_version() + # Dictionary containing model configurations. # Each key is a model identifier, and the value is a dictionary with: # - "name": the model's name or path @@ -197,6 +213,10 @@ "name": "optimum-intel-internal-testing/tiny-random-qwen3-vl", "convert_args": ["--trust-remote-code", "--task", "image-text-to-text"], }, + "tiny-random-qwen3-omni": { + "name": "optimum-intel-internal-testing/tiny-random-qwen3-omni", + "convert_args": ["--trust-remote-code", "--task", "image-text-to-text"], + }, } TEST_FILES = { diff --git a/tests/python_tests/samples/test_tools_llm_benchmark.py b/tests/python_tests/samples/test_tools_llm_benchmark.py index 61a8f47f1f..ec16739f9e 100644 --- a/tests/python_tests/samples/test_tools_llm_benchmark.py +++ b/tests/python_tests/samples/test_tools_llm_benchmark.py @@ -12,7 +12,8 @@ from utils.constants import get_ov_cache_converted_models_dir from utils.kokoro_test_assets import prepare_tiny_kokoro_model_path from utils.kokoro_test_assets import prepare_tiny_kokoro_ov_path -from conftest import SAMPLES_PY_DIR, convert_model, download_test_content +from conftest import SAMPLES_PY_DIR, convert_model, download_test_content, TRANSFORMERS_VERSION + convert_draft_model = convert_model download_mask_image = download_test_content @@ -714,3 +715,31 @@ def test_python_tool_llm_benchmark_video_gen_json( generate_llm_bench_input_generation_jsonl, ] + sample_args run_sample(benchmark_py_command) + + @pytest.mark.samples + @pytest.mark.skipif( + TRANSFORMERS_VERSION is None or TRANSFORMERS_VERSION < (5, 1), + reason="Requires transformers >= 5.1", + ) + @pytest.mark.parametrize("download_test_content", ["cat.png"], indirect=True) + @pytest.mark.parametrize("convert_model", ["tiny-random-qwen3-omni"], indirect=True) + @pytest.mark.parametrize( + "sample_args", + [ + ["-d", "cpu", "-n", "1", "--num_steps", "4", "--task", "visual_text_gen", "--optimum"], + ["-d", "cpu", "-n", "1", "--num_steps", "4", "--task", "visual_text_gen", "--genai"], + ], + ) + def test_python_tool_llm_benchmark_visual_text_gen(self, download_test_content, convert_model, sample_args): + benchmark_script = SAMPLES_PY_DIR / "llm_bench/benchmark.py" + benchmark_py_command = [ + sys.executable, + benchmark_script, + "-m", + convert_model, + "--media", + download_test_content, + "--prompt", + "What animal is in this image?", + ] + sample_args + run_sample(benchmark_py_command) diff --git a/tools/llm_bench/llm_bench_utils/config_class.py b/tools/llm_bench/llm_bench_utils/config_class.py index 2478e522c1..2289d3a799 100644 --- a/tools/llm_bench/llm_bench_utils/config_class.py +++ b/tools/llm_bench/llm_bench_utils/config_class.py @@ -171,7 +171,7 @@ class UseCaseTextToSpeech(UseCase): "videochat-flash-qwen", "gemma4", ] - ) + ), ], "speech_to_text": [UseCaseSpeech2Text(["whisper"])], "image_cls": [UseCaseImageCls(["vit"])], @@ -251,5 +251,12 @@ class UseCaseTextToSpeech(UseCase): "text_to_speech": [UseCaseTextToSpeech(["speecht5", "kokoro"])], } +try: + from optimum.intel.openvino import OVModelForMultimodalLM +except ImportError: + pass +else: + USE_CASES["visual_text_gen"].append(UseCaseVLM(["qwen3-omni"], ov_cls=OVModelForMultimodalLM)) + PA_ATTENTION_BACKEND = "PA" SDPA_ATTENTION_BACKEND = "SDPA"