diff --git a/angelslim/data/multimodal_dataset.py b/angelslim/data/multimodal_dataset.py index a9d3e0fd..44ab3c24 100644 --- a/angelslim/data/multimodal_dataset.py +++ b/angelslim/data/multimodal_dataset.py @@ -174,7 +174,7 @@ def _process_and_append(self, messages: List[Dict], tools=None): """Process messages and append to dataset""" # max_length padding for int4 gptq, gptaq and awq - if "int4_" in self.quant_algo: + if self.quant_algo and "int4_" in self.quant_algo: padding = "max_length" else: padding = True @@ -248,7 +248,7 @@ def _extract_vision_info(self, messages: List[Dict]) -> tuple: try: img = Image.open(item["image"]) image_paths.append(img) - except ValueError as e: + except (ValueError, OSError) as e: raise ValueError(f"Could not open image file: {item['image']}, {e}") elif isinstance(item["image"], Image.Image): image_paths.append(item["image"]) diff --git a/angelslim/data/omni_dataset.py b/angelslim/data/omni_dataset.py index 8d2c2019..aecec11f 100644 --- a/angelslim/data/omni_dataset.py +++ b/angelslim/data/omni_dataset.py @@ -98,6 +98,13 @@ def _load_file_based_dataset(self, data_path: str, num_samples: int): "content": content, } ) + else: + conversation.append( + { + "role": m["role"], + "content": [{"type": "text", "text": m["content"]}], + } + ) self._process_and_append(conversation) line_count += 1 diff --git a/tests/test_multimodal_dataset.py b/tests/test_multimodal_dataset.py new file mode 100644 index 00000000..e8fa2d89 --- /dev/null +++ b/tests/test_multimodal_dataset.py @@ -0,0 +1,174 @@ +# Copyright 2025 Tencent Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for ``MultiModalDataset`` helpers. + +These tests are CPU-only and require neither a GPU nor model weights. +Heavy dependencies are stubbed so the data-preparation logic can be +exercised in isolation. + +Regression: ``_process_and_append`` crashed with ``TypeError`` when +``quant_algo`` was ``None`` (sparsity-only or distill-only pipelines). +Regression: ``_extract_vision_info`` only caught ``ValueError`` from +``Image.open()``, but missing files raise ``FileNotFoundError``. +""" + +import importlib.util +import os +import sys +import types + +import pytest + +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_MM_DATASET_PATH = os.path.join(_REPO_ROOT, "angelslim", "data", "multimodal_dataset.py") + + +def _install_stubs(): + """Register lightweight stand-ins so ``multimodal_dataset.py`` imports cleanly.""" + + def _module(name): + mod = types.ModuleType(name) + sys.modules[name] = mod + return mod + + if "torch" not in sys.modules: + torch = _module("torch") + torch.Tensor = type("Tensor", (), {}) + _module("torch.utils") + torch_utils_data = _module("torch.utils.data") + torch_utils_data.Dataset = type("Dataset", (), {}) + if "datasets" not in sys.modules: + datasets = _module("datasets") + datasets.load_dataset = lambda *a, **k: None + if "tqdm" not in sys.modules: + tqdm_mod = _module("tqdm") + tqdm_mod.tqdm = lambda x, **kw: x + if "transformers" not in sys.modules: + transformers = _module("transformers") + transformers.ProcessorMixin = type("ProcessorMixin", (), {}) + + for pkg in ("angelslim", "angelslim.utils", "angelslim.data"): + if pkg not in sys.modules: + mod = _module(pkg) + mod.__path__ = [] + + if "angelslim.utils.lazy_imports" not in sys.modules: + lazy = _module("angelslim.utils.lazy_imports") + lazy.qwen_vl_utils = types.ModuleType("qwen_vl_utils") + + if "angelslim.data.base_dataset" not in sys.modules: + base_mod = _module("angelslim.data.base_dataset") + base_mod.BaseDataset = type("BaseDataset", (), {"__init__": lambda *a, **k: None}) + + +def _load_mm_dataset_cls(): + _install_stubs() + spec = importlib.util.spec_from_file_location( + "angelslim.data.multimodal_dataset", _MM_DATASET_PATH + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.MultiModalDataset + + +@pytest.fixture(scope="module") +def mm_cls(): + return _load_mm_dataset_cls() + + +def _make_instance(cls, quant_algo=None, model_name="Qwen3VL"): + """Create an uninitialized MultiModalDataset with specific attributes.""" + inst = cls.__new__(cls) + inst.quant_algo = quant_algo + inst.model_name = model_name + inst.data = [] + inst.processor = None + inst.max_length = 512 + return inst + + +def test_process_and_append_quant_algo_none_does_not_crash(mm_cls): + """When quant_algo is None (sparsity/distill pipeline), padding defaults to True. + + Regression: ``"int4_" in None`` raised ``TypeError``. + """ + inst = _make_instance(mm_cls, quant_algo=None, model_name="Qwen3VL") + + call_args = {} + + def fake_apply_chat_template(*args, **kwargs): + call_args["padding"] = kwargs.get("padding") + # Return a dict mimicking processor output with input_ids + dummy_tensor = type("T", (), {"roll": lambda s, **k: s, "__setitem__": lambda *a: None})() + return {"input_ids": dummy_tensor} + + inst.processor = type("P", (), {"apply_chat_template": fake_apply_chat_template})() + + messages = [ + {"role": "user", "content": [{"type": "text", "text": "hello"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "hi"}]}, + ] + inst._process_and_append(messages) + assert call_args["padding"] is True + + +def test_process_and_append_int4_uses_max_length_padding(mm_cls): + """When quant_algo contains 'int4_', padding should be 'max_length'.""" + inst = _make_instance(mm_cls, quant_algo="int4_gptq", model_name="Qwen3VL") + + call_args = {} + + def fake_apply_chat_template(*args, **kwargs): + call_args["padding"] = kwargs.get("padding") + dummy_tensor = type("T", (), {"roll": lambda s, **k: s, "__setitem__": lambda *a: None})() + return {"input_ids": dummy_tensor} + + inst.processor = type("P", (), {"apply_chat_template": fake_apply_chat_template})() + + messages = [ + {"role": "user", "content": [{"type": "text", "text": "hello"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "hi"}]}, + ] + inst._process_and_append(messages) + assert call_args["padding"] == "max_length" + + +def test_extract_vision_info_missing_file_raises_value_error(mm_cls): + """Missing image path should raise ValueError with a helpful message. + + Regression: ``Image.open()`` raised ``FileNotFoundError`` which was + not caught, producing an unhelpful traceback. + """ + inst = _make_instance(mm_cls) + messages = [ + { + "role": "user", + "content": [{"type": "image", "image": "/nonexistent/path/img.jpg"}], + } + ] + with pytest.raises(ValueError, match="Could not open image file"): + inst._extract_vision_info(messages) + + +def test_extract_vision_info_skips_non_list_content(mm_cls): + """Messages with string content (e.g. HunyuanVL adapted) are skipped.""" + inst = _make_instance(mm_cls) + messages = [ + {"role": "assistant", "content": "plain text response"}, + {"role": "user", "content": [{"type": "text", "text": "hello"}]}, + ] + images, videos = inst._extract_vision_info(messages) + assert images == [] + assert videos == [] diff --git a/tests/test_omni_dataset.py b/tests/test_omni_dataset.py new file mode 100644 index 00000000..dfd02ee9 --- /dev/null +++ b/tests/test_omni_dataset.py @@ -0,0 +1,195 @@ +# Copyright 2025 Tencent Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for ``OmniDataset._load_file_based_dataset``. + +These tests are CPU-only and require neither a GPU nor model weights. +Heavy dependencies are stubbed so the conversation-construction logic +can be exercised in isolation. + +Regression: assistant messages were silently dropped from the +conversation list, producing incomplete calibration data. +""" + +import importlib.util +import json +import os +import sys +import types + +import pytest + +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_OMNI_DATASET_PATH = os.path.join(_REPO_ROOT, "angelslim", "data", "omni_dataset.py") + + +def _install_stubs(): + """Register lightweight stand-ins so ``omni_dataset.py`` imports cleanly.""" + + def _module(name): + mod = types.ModuleType(name) + sys.modules[name] = mod + return mod + + if "torch" not in sys.modules: + _module("torch") + _module("torch.utils") + torch_utils_data = _module("torch.utils.data") + torch_utils_data.Dataset = type("Dataset", (), {}) + if "transformers" not in sys.modules: + transformers = _module("transformers") + transformers.ProcessorMixin = type("ProcessorMixin", (), {}) + + for pkg in ("angelslim", "angelslim.utils", "angelslim.data"): + if pkg not in sys.modules: + mod = _module(pkg) + mod.__path__ = [] + + if "angelslim.utils.lazy_imports" not in sys.modules: + lazy = _module("angelslim.utils.lazy_imports") + lazy.qwen_omni_utils = types.ModuleType("qwen_omni_utils") + + if "angelslim.data.base_dataset" not in sys.modules: + base_mod = _module("angelslim.data.base_dataset") + base_mod.BaseDataset = type("BaseDataset", (), {"__init__": lambda *a, **k: None}) + + +def _load_omni_dataset_cls(): + _install_stubs() + spec = importlib.util.spec_from_file_location( + "angelslim.data.omni_dataset", _OMNI_DATASET_PATH + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.OmniDataset + + +@pytest.fixture(scope="module") +def omni_cls(): + return _load_omni_dataset_cls() + + +def _make_instance(cls): + """Create an uninitialized OmniDataset for testing.""" + inst = cls.__new__(cls) + inst.data = [] + inst.processor = None + inst.use_audio_in_video = False + return inst + + +def _write_jsonl(path, records): + with open(path, "w") as f: + for r in records: + f.write(json.dumps(r) + "\n") + + +def test_assistant_messages_are_preserved(tmp_path, omni_cls): + """Assistant turns must appear in the conversation passed downstream. + + Regression: the conversation loop only handled ``system`` and ``user`` + roles, silently dropping ``assistant`` turns from calibration data. + """ + data_file = tmp_path / "data.jsonl" + _write_jsonl( + data_file, + [ + { + "messages": [ + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": "4"}, + {"role": "user", "content": "And 3+3?"}, + {"role": "assistant", "content": "6"}, + ] + } + ], + ) + + inst = _make_instance(omni_cls) + captured = [] + inst._process_and_append = lambda msgs: captured.append(msgs) + + inst._load_file_based_dataset(str(data_file), num_samples=-1) + + assert len(captured) == 1 + roles = [m["role"] for m in captured[0]] + assert roles == ["user", "assistant", "user", "assistant"] + + +def test_system_message_is_preserved(tmp_path, omni_cls): + """System messages must still appear in the conversation.""" + data_file = tmp_path / "data.jsonl" + _write_jsonl( + data_file, + [ + { + "messages": [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + } + ], + ) + + inst = _make_instance(omni_cls) + captured = [] + inst._process_and_append = lambda msgs: captured.append(msgs) + + inst._load_file_based_dataset(str(data_file), num_samples=-1) + + roles = [m["role"] for m in captured[0]] + assert roles == ["system", "user", "assistant"] + + +def test_assistant_content_is_wrapped_as_text(tmp_path, omni_cls): + """Assistant content should be wrapped in the ``[{type: text, text: ...}]`` format.""" + data_file = tmp_path / "data.jsonl" + _write_jsonl( + data_file, + [ + { + "messages": [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + } + ], + ) + + inst = _make_instance(omni_cls) + captured = [] + inst._process_and_append = lambda msgs: captured.append(msgs) + + inst._load_file_based_dataset(str(data_file), num_samples=-1) + + assistant_msg = captured[0][1] + assert assistant_msg["role"] == "assistant" + assert assistant_msg["content"] == [{"type": "text", "text": "Hello!"}] + + +def test_num_samples_limits_loaded_records(tmp_path, omni_cls): + """When num_samples > 0, only that many records are processed.""" + data_file = tmp_path / "data.jsonl" + _write_jsonl( + data_file, + [{"messages": [{"role": "user", "content": f"q{i}"}]} for i in range(5)], + ) + + inst = _make_instance(omni_cls) + captured = [] + inst._process_and_append = lambda msgs: captured.append(msgs) + + inst._load_file_based_dataset(str(data_file), num_samples=2) + assert len(captured) == 2