diff --git a/.github/workflows/basic-ci.yml b/.github/workflows/basic-ci.yml index 0032d3b..11eea61 100644 --- a/.github/workflows/basic-ci.yml +++ b/.github/workflows/basic-ci.yml @@ -2,9 +2,9 @@ name: Basic CI on: push: - branches: [ mainline ] + branches: [ main ] pull_request: - branches: [ mainline ] + branches: [ main ] permissions: contents: read @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d1fb096..e35a710 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,9 +2,9 @@ name: "CodeQL" on: push: - branches: [ mainline ] + branches: [ main ] pull_request: - branches: [ mainline ] + branches: [ main ] jobs: analyze: diff --git a/.github/workflows/coverage-check.yml b/.github/workflows/coverage-check.yml index a84ea13..87847c5 100644 --- a/.github/workflows/coverage-check.yml +++ b/.github/workflows/coverage-check.yml @@ -29,7 +29,7 @@ jobs: - name: Get base branch coverage run: | - git checkout origin/mainline + git checkout origin/main pip install -e ".[dev]" python -m pytest tests/ -m "not integration" --cov=src --cov-report=json:coverage-base.json --tb=no -q || true BASE_COVERAGE=$(python -c "import json; print(json.load(open('coverage-base.json'))['totals']['percent_covered'])") diff --git a/.gitignore b/.gitignore index 7ea422a..d8be258 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,12 @@ analysis/data/ # Distribution archives share/ +test-nested.json + +# kvm-unit-tests run artifacts (SSM logs + fetched sources; not source) +vm-tests/*/ssm-output +vm-tests/*/ssm-output-*/ +vm-tests/kvm-unit-tests/kvm-unit-tests/ +vm-tests/kvm-unit-tests/kernel_version_before.txt +vm-tests/kvm-unit-tests/kvm-unit-tests.log +vm-tests/kvm-unit-tests/results_kvm-unit-tests.* diff --git a/examples/aws/alltests-config.json b/examples/aws/alltests-config.json index 374f9f0..055e70c 100644 --- a/examples/aws/alltests-config.json +++ b/examples/aws/alltests-config.json @@ -205,7 +205,7 @@ "vms": [ { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -215,7 +215,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -225,7 +225,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -235,7 +235,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -245,7 +245,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -255,7 +255,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -265,7 +265,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -275,7 +275,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ diff --git a/examples/aws/config.json b/examples/aws/config.json index 7348f26..e23aac7 100644 --- a/examples/aws/config.json +++ b/examples/aws/config.json @@ -219,7 +219,7 @@ "vms": [ { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -229,13 +229,23 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ "unixbench-kernel-regression" ], "min_count": 1 + }, + { + "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", + "instance_type": "c8i.4xlarge", + "root_volume_size": 40, + "max_runtime": 5400, + "test": [ + "kvm-unit-tests" + ], + "min_count": 1 } ] } diff --git a/setup.py b/setup.py index 8e39de9..520742b 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ version="1.0.0", description="Kernel-ci-cloud-labs", long_description=long_description, + long_description_content_type="text/markdown", package_dir={"": "src"}, packages=find_packages(where="src"), python_requires=">=3.11", @@ -45,6 +46,9 @@ "isort>=5.0", "pre-commit>=2.0", "pytest-cov>=2.0", + "pandas>=1.3.0", + "matplotlib>=3.4.0", + "seaborn>=0.11.0", ], "analysis": [ "pandas>=1.3.0", diff --git a/src/kernel_ci_cloud_labs/cli.py b/src/kernel_ci_cloud_labs/cli.py index 2187bce..76df4ac 100644 --- a/src/kernel_ci_cloud_labs/cli.py +++ b/src/kernel_ci_cloud_labs/cli.py @@ -292,8 +292,11 @@ def main(): val_parser.add_argument("--bucket", help="S3 bucket to verify (and create with --fix)") val_parser.add_argument("--role", help="IAM role name used by VM instance profiles") val_parser.add_argument("--region", default="us-west-2", help="AWS region (default: us-west-2)") - val_parser.add_argument("--api-url", help=f"KernelCI API base URI (overrides $KERNELCI_API_BASE_URI)") - val_parser.add_argument("--fix", action="store_true", help="Create missing resources (S3 bucket) instead of just reporting them") + val_parser.add_argument("--api-url", help="KernelCI API base URI (overrides $KERNELCI_API_BASE_URI)") + val_parser.add_argument( + "--fix", action="store_true", + help="Create missing resources (S3 bucket) instead of just reporting them", + ) val_parser.set_defaults(func=cmd_setup_validate) args = parser.parse_args() diff --git a/src/kernel_ci_cloud_labs/core/artifacts.py b/src/kernel_ci_cloud_labs/core/artifacts.py index ebe330d..4b4de9b 100644 --- a/src/kernel_ci_cloud_labs/core/artifacts.py +++ b/src/kernel_ci_cloud_labs/core/artifacts.py @@ -262,5 +262,3 @@ def collect_run_artifacts( manifest_path, ) return manifest - - diff --git a/src/kernel_ci_cloud_labs/launch_vm.py b/src/kernel_ci_cloud_labs/launch_vm.py index ac4add2..8cad9aa 100644 --- a/src/kernel_ci_cloud_labs/launch_vm.py +++ b/src/kernel_ci_cloud_labs/launch_vm.py @@ -175,6 +175,25 @@ def _resolve_ssm_parameter(self, parameter_name): log_error(f"Failed to resolve SSM parameter {parameter_name}: {e}") raise + def _supports_nested_virtualization(self): + """Check if the instance type supports nested virtualization. + + AWS supports nested virtualization on C8i, M8i, R8i, C7i, M7i, R7i + and related instance families. + Returns True if the current instance_type belongs to a supported family. + + See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/amazon-ec2-nested-virtualization.html + """ + nested_families = ( + "c8i", "m8i", "r8i", "c8id", "m8id", "r8id", + "c8i-flex", "m8i-flex", "r8i-flex", "x8i", + "c7i", "m7i", "r7i", "c7id", "m7id", "r7id", + "c7i-flex", "m7i-flex", "i7i", + ) + # Extract family: e.g. "c8i.4xlarge" -> "c8i" + instance_family = self.instance_type.rsplit(".", 1)[0].lower() + return instance_family in nested_families + def prepare_test_artifacts(self): """Verify test payload zip exists in S3.""" log_info(f"\n=== Verifying test artifacts for {self.test} ===") @@ -254,6 +273,12 @@ def spawn_vm(self): params["IamInstanceProfile"] = {"Name": self.role_name} + # Enable nested virtualization for instance families that support it. + # Required for KVM/Hyper-V workloads inside the EC2 instance. + if self._supports_nested_virtualization(): + params["CpuOptions"] = {"NestedVirtualization": "enabled"} + log_info(f"Nested virtualization enabled for {self.instance_type}") + log_not("Calling run_instances...") response = self.ec2.run_instances(**params) diff --git a/src/kernel_ci_cloud_labs/pull_labs_poller.py b/src/kernel_ci_cloud_labs/pull_labs_poller.py index 124e596..5057fe6 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_poller.py +++ b/src/kernel_ci_cloud_labs/pull_labs_poller.py @@ -45,7 +45,6 @@ from kernel_ci_cloud_labs.kcidb_submit import ( build_test_row, - submit_tests, to_kcidb_status, ) from kernel_ci_cloud_labs.pull_labs_translate import translate_job @@ -314,7 +313,7 @@ def _validate_default_executor_deps() -> None: problems: List[str] = [] try: - import boto3 # noqa: F401,PLC0415 + import boto3 # noqa: F401,PLC0415 # pylint: disable=unused-import except ImportError as e: problems.append( f"boto3 import failed ({e}) — run: python3.11 -m pip install -e ." diff --git a/src/kernel_ci_cloud_labs/pull_labs_translate.py b/src/kernel_ci_cloud_labs/pull_labs_translate.py index 2695d21..fe78a70 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_translate.py +++ b/src/kernel_ci_cloud_labs/pull_labs_translate.py @@ -24,7 +24,7 @@ # argument of translate_job() to customise per-deployment. DEFAULT_PLATFORM_MAP: Dict[str, Dict[str, str]] = { "x86_64": { - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", }, "arm64": { diff --git a/src/kernel_ci_cloud_labs/setup_validate.py b/src/kernel_ci_cloud_labs/setup_validate.py index bb51f5b..da0419b 100644 --- a/src/kernel_ci_cloud_labs/setup_validate.py +++ b/src/kernel_ci_cloud_labs/setup_validate.py @@ -200,7 +200,7 @@ def _check_bucket_policy_statement(s3, bucket_name: str, fix: bool) -> bool: print(f"✗ Could not read bucket policy ({code}): {e}") return False existing = None - except (ValueError, json.JSONDecodeError) as e: + except ValueError as e: print(f"✗ Bucket policy is not valid JSON: {e}") return False @@ -365,7 +365,7 @@ def check_kcidb_jwt() -> bool: # JWT payload is base64url; pad to a multiple of 4 before decoding. payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4) payload = json.loads(base64.urlsafe_b64decode(payload_b64)) - except (ValueError, json.JSONDecodeError) as e: + except ValueError as e: print(f"✗ JWT payload not decodable: {e}") return False diff --git a/tests/test_kcidb_submit.py b/tests/test_kcidb_submit.py index 5c2a8d2..fd1dfeb 100644 --- a/tests/test_kcidb_submit.py +++ b/tests/test_kcidb_submit.py @@ -187,8 +187,10 @@ def test_submit_revision_posts_with_bearer_auth(self): class FakeResp: def __enter__(self): return self + def __exit__(self, *a): return False + def read(self): return b'{"status":"ok","id":"sub-1"}' @@ -237,7 +239,7 @@ def __enter__(self): return self def __exit__(self, *a): return False def read(self): return b'{}' - def fake_urlopen(req, timeout=None): + def fake_urlopen(req, timeout=None): # pylint: disable=unused-argument captured["body"] = json.loads(req.data.decode("utf-8")) return FakeResp() @@ -264,7 +266,7 @@ def __enter__(self): return self def __exit__(self, *a): return False def read(self): return b'{}' - def fake_urlopen(req, timeout=None): + def fake_urlopen(req, timeout=None): # pylint: disable=unused-argument captured["body"] = json.loads(req.data.decode("utf-8")) return FakeResp() diff --git a/tests/test_nested_virtualization.py b/tests/test_nested_virtualization.py new file mode 100644 index 0000000..c664b27 --- /dev/null +++ b/tests/test_nested_virtualization.py @@ -0,0 +1,83 @@ +"""Unit tests for nested virtualization support in launch_vm.""" +# pylint: disable=protected-access + +__authors__ = ["Norbert Manthey "] +__copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved." +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import patch, MagicMock + +import pytest + +from kernel_ci_cloud_labs.launch_vm import VMLauncher + + +@pytest.fixture +def _mock_boto3(): + """Patch boto3 so VMLauncher.__init__ doesn't need real AWS credentials.""" + with patch("kernel_ci_cloud_labs.launch_vm.boto3") as mock_boto: + mock_boto.client.return_value = MagicMock() + yield mock_boto + + +class TestNestedVirtualization: + """Tests for _supports_nested_virtualization and CpuOptions.""" + + def _make_launcher(self, instance_type, _mock_boto3): + """Create a VMLauncher with a given instance_type, mocking AWS.""" + vm_config = { + "instance_type": instance_type, + "ami_id": "ami-test123", + "role_name": "test-role", + "s3_bucket": "test-bucket", + "run_prefix": "run_test", + } + return VMLauncher(vm_config) + + def test_c8i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("c8i.4xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_c8i_flex_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("c8i-flex.2xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_m8i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("m8i.large", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_r8i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("r8i.xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_c7i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("c7i.2xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_m7i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("m7i.xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_x8i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("x8i.large", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_i7i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("i7i.xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_c5a_does_not_support_nested(self, _mock_boto3): + launcher = self._make_launcher("c5a.4xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is False + + def test_c6g_does_not_support_nested(self, _mock_boto3): + launcher = self._make_launcher("c6g.4xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is False + + def test_t3_does_not_support_nested(self, _mock_boto3): + launcher = self._make_launcher("t3.micro", _mock_boto3) + assert launcher._supports_nested_virtualization() is False + + def test_m5_does_not_support_nested(self, _mock_boto3): + launcher = self._make_launcher("m5.large", _mock_boto3) + assert launcher._supports_nested_virtualization() is False diff --git a/tests/test_pull_labs_poller.py b/tests/test_pull_labs_poller.py index 787c341..8b5e980 100644 --- a/tests/test_pull_labs_poller.py +++ b/tests/test_pull_labs_poller.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # # Copyright (C) 2026 Collabora Limited +# pylint: disable=protected-access # Author: Denys Fedoryshchenko """Unit tests for pull_labs_poller (no network, no AWS).""" @@ -8,7 +9,6 @@ import json import logging import os -import tempfile import urllib.error from unittest.mock import patch @@ -626,14 +626,16 @@ class TestProcessEventNodeResult: def _run(self, poller, event, translate=None): translate = translate or {"return_value": {}} captured = {} - with patch.object(poller, "_claim_node", return_value=True), \ - patch.object( - poller, "_finish_node", - side_effect=lambda nid, outcome: captured.update(outcome=outcome), - ), \ - patch(_GET, return_value={"artifacts": {}}), \ - patch("kernel_ci_cloud_labs.pull_labs_poller.translate_job", **translate), \ - patch("kernel_ci_cloud_labs.pull_labs_poller.submit_tests", return_value={}): + with ( + patch.object(poller, "_claim_node", return_value=True), + patch.object( + poller, "_finish_node", + side_effect=lambda nid, outcome: captured.update(outcome=outcome), + ), + patch(_GET, return_value={"artifacts": {}}), + patch("kernel_ci_cloud_labs.pull_labs_poller.translate_job", + **translate), + ): poller.process_event(event) return captured["outcome"] @@ -682,9 +684,10 @@ def test_translate_failure_finishes_invalid_job_params(self): assert "missing artifacts.kernel" in outcome.error_msg def test_per_instance_rows_carry_log_url_and_stable_test_id(self): - """When executor returns per-instance rows with log_url, the submitted - KCIDB rows must each carry that URL and a test_id derived from the - instance_id (not the positional index).""" + """When executor returns per-instance rows with log_url, the node + outcome must carry those URLs in artifacts.test_log (first URL) and + test_log_N (subsequent URLs), and the node result must reflect the + aggregated per-instance statuses.""" per_test = [ {"name": "boot", "status": "PASS", "instance_id": "i-aaaa1111", "log_url": "https://b.s3.eu-west-1.amazonaws.com/a.log"}, @@ -695,31 +698,15 @@ def test_per_instance_rows_carry_log_url_and_stable_test_id(self): _minimal_kc(), job_executor=lambda cfg: (per_test, None), ) - seen = {} - with patch.object(p, "_claim_node", return_value=True), \ - patch.object(p, "_finish_node"), \ - patch(_GET, return_value={"artifacts": {}}), \ - patch("kernel_ci_cloud_labs.pull_labs_poller.translate_job", - return_value={}), \ - patch( - "kernel_ci_cloud_labs.pull_labs_poller.submit_tests", - side_effect=lambda url, jwt, origin, build_id, rows: seen.update(rows=rows), - ): - p.process_event(_job_event(node_id="ndX")) - - rows = seen["rows"] - assert len(rows) == 2 - by_id = {r["id"]: r for r in rows} - # test_id derived from instance_id => stable across retries. - assert set(by_id) == {"pullab_cloud_aws:ndX.i-aaaa1111", "pullab_cloud_aws:ndX.i-bbbb2222"} - # Per-row log_url survives the build_test_row pass-through. - assert by_id["pullab_cloud_aws:ndX.i-aaaa1111"]["log_url"] == \ + outcome = self._run(p, _job_event(node_id="ndX")) + + # One FAIL among results -> overall node result is "fail". + assert outcome.result == "fail" + # Log URLs are attached to the outcome artifacts for send_kcidb. + assert outcome.artifacts["test_log"] == \ "https://b.s3.eu-west-1.amazonaws.com/a.log" - assert by_id["pullab_cloud_aws:ndX.i-bbbb2222"]["log_url"] == \ + assert outcome.artifacts["test_log_1"] == \ "https://b.s3.eu-west-1.amazonaws.com/b.log" - # instance_id surfaces in misc for traceability. - assert by_id["pullab_cloud_aws:ndX.i-aaaa1111"]["misc"]["instance_id"] == "i-aaaa1111" - # Aggregated node outcome from per-instance statuses. # (one fail among two -> fail; verified indirectly via existing tests). diff --git a/tests/test_pull_labs_translate.py b/tests/test_pull_labs_translate.py index 68a33a2..b1a6298 100644 --- a/tests/test_pull_labs_translate.py +++ b/tests/test_pull_labs_translate.py @@ -9,7 +9,6 @@ from kernel_ci_cloud_labs.pull_labs_translate import ( DEFAULT_PLATFORM_MAP, - DEFAULT_TEST_TYPE_MAP, translate_job, ) diff --git a/tests/test_role_manager.py b/tests/test_role_manager.py index c3afbfb..dede843 100644 --- a/tests/test_role_manager.py +++ b/tests/test_role_manager.py @@ -1,4 +1,5 @@ """Unit tests for AWS Role Manager""" +# pylint: disable=protected-access __authors__ = ["Max Hubmann ", "Norbert Manthey "] __copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved." diff --git a/vm-tests/kvm-unit-tests/README.md b/vm-tests/kvm-unit-tests/README.md new file mode 100644 index 0000000..b2fa048 --- /dev/null +++ b/vm-tests/kvm-unit-tests/README.md @@ -0,0 +1,89 @@ +# kvm-unit-tests + +Runs the upstream [kvm-unit-tests](https://gitlab.com/kvm-unit-tests/kvm-unit-tests) +suite against the guest's KVM hypervisor. Intended to run on an EC2 instance +with **nested virtualization enabled** (c8i/c7i and related Intel families, +handled automatically by `_supports_nested_virtualization()` in `launch_vm.py`). + +## What it does + +The test runs in two stages (two `run-*.sh` scripts, with a reboot between them +handled by the pipeline client): + +**run-01 -- install the target kernel.** Records the running kernel, then +installs the kernel under test from the pipeline's shared `kernel-rpms` area +(reusing the flow from `unixbench-kernel-regression`) and makes it the default +boot target; the client then reboots into it. If no kernel RPMs are provided, +the current kernel is kept. + +**run-02 -- run kvm-unit-tests.** After the reboot: + +1. Confirms the running kernel is the one run-01 installed (when applicable). +2. Installs the build toolchain (`git`, `gcc`, `make`, `binutils`) and builds + **QEMU from source** into `/opt/qemu` (`x86_64-softmmu`, `--enable-kvm`) via + `build-qemu.sh`. Amazon Linux 2023 ships no qemu system emulator in its core + repos (only `qemu-img`), and its EPEL9-derived SPAL repo does not carry it + either, so a source build is required. `install_qemu()` exports `QEMU` so + `run_tests.sh` uses the built binary. +3. Verifies `/dev/kvm` is present in the guest -- a built-in regression check on + the EC2 nested-virtualization enablement itself. +4. Clones and builds kvm-unit-tests (pinned to a fixed revision by default via + `KVMUT_REF`; set it empty to track upstream tip). +5. Runs a **curated stable subset** by default (see `KVMUT_TESTS`) with + `ACCEL=kvm ./run_tests.sh -v`. Set `KVMUT_TESTS=""` for the full suite. +6. Writes `results_kvm-unit-tests.txt` (human summary) and + `results_kvm-unit-tests.csv` (`test_name,status,kernel_version,arch`) for + KCIDB ingestion. The parser strips ANSI colour codes from run_tests.sh output + before counting. Both use the `results_` prefix so `test-vm-client.sh` + uploads them; the pass/fail verdict itself is conveyed by the script's exit + code, which the client records as `SUCCESS`/`FAILED` in `result.txt`. + +This is a **functional** test (pass/fail per sub-test), not a benchmark, so it +produces no `benchmark-*.csv` and is not part of the performance-regression +analysis. + +## Why + +kvm-unit-tests is the standard upstream suite for exercising a running kernel's +KVM implementation. Running it continuously in nested-virtualization cloud +guests gives a per-kernel regression signal for the virtualization stack, and +additional test groups can be layered in later via `KVMUT_GROUP`. + +## Validated on c8i (2026-07-10) + +Run on a live `c8i.4xlarge` with nested virt + source-built QEMU 9.2.0: + +- **Full suite:** 54 PASS / 10 FAIL / 23 SKIP. The 10 failures are a coherent + nested-virt-sensitive class: timeouts (`xapic`, `access`, `vmx_apicv_test`, + `vmx_posted_intr_test`, `vmx_pf_exception_test`), MSR/PMU emulation gaps + (`msr`, `msr64`, `pmu`), and nested-VMX instability (`vmx` SIGSEGV, `la57`). +- **Curated subset (default):** 6 PASS / 0 FAIL / 0 SKIP -- clean and stable, + suitable as a hard gate. + +## Two-tier model + +- **Default = stable gate.** `KVMUT_TESTS` runs the proven-stable subset and + `KVMUT_OBSERVE_ONLY=false` gates on it. A regression that breaks one of these + tests fails the pipeline. +- **Full suite = tracking.** Set `KVMUT_TESTS=""`. `KVMUT_IGNORE_FAILURES` is + pre-seeded with the 10 nested-virt-sensitive tests above so a full run still + gates on the stable core; set `KVMUT_OBSERVE_ONLY=true` to only report. + +## Configuration (environment overrides) + +| Variable | Default | Purpose | +|---|---|---| +| `KVMUT_TESTS` | `debug intel_iommu lam vmx_init_signal_test vmx_sipi_signal_test hyperv_clock` | Space-separated test names to run. Empty = full suite. | +| `KVMUT_OBSERVE_ONLY` | `false` | Report counts but never fail. Set `true` for full-suite tracking runs. | +| `KVMUT_IGNORE_FAILURES` | *(10 nested-virt-sensitive tests)* | Test names excluded from the pass/fail decision. | +| `KVMUT_REF` | *(pinned commit `1da1819e`)* | kvm-unit-tests revision for reproducibility. Empty = upstream tip. | +| `KVMUT_GROUP` | *(empty)* | Run only a specific `run_tests.sh -g` group. | +| `QEMU_VERSION` | `9.2.0` | QEMU source version built by `build-qemu.sh`. | + +## Requirements + +- x86_64 instance from a nested-virt-capable family (e.g. `c8i.4xlarge`), + 32 GB+ RAM and ~40 GB disk (QEMU builds in `/tmp`, needs ~3 GB free). +- Kernel RPMs for the kernel under test, supplied by the pipeline + (`external_requirements.json` sets `kernel-rpms/binary: true`). Without them + run-01 keeps the current kernel. diff --git a/vm-tests/kvm-unit-tests/build-qemu.sh b/vm-tests/kvm-unit-tests/build-qemu.sh new file mode 100755 index 0000000..a1cdd4e --- /dev/null +++ b/vm-tests/kvm-unit-tests/build-qemu.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Build QEMU from source on AL2023 (no QEMU in default repos). +# Builds a minimal x86_64-softmmu target — enough for nested KVM testing. +set -e +exec 2>&1 + +QEMU_VERSION="${QEMU_VERSION:-9.2.0}" +# Expected sha256 of qemu-${QEMU_VERSION}.tar.xz. download.qemu.org publishes +# only GPG signatures, so the digest is pinned here. This value matches the +# default 9.2.0 tarball; override QEMU_SHA256 when bumping QEMU_VERSION, or set +# it to empty to skip the integrity check. +QEMU_SHA256="${QEMU_SHA256:-f859f0bc65e1f533d040bbe8c92bcfecee5af2c921a6687c652fb44d089bd894}" +QEMU_DIR="/opt/qemu" +NPROC=$(nproc) + +# Pre-flight: disk space check +AVAIL_MB=$(df --output=avail -m /tmp | tail -1 | tr -d ' ') +if [ "$AVAIL_MB" -lt 3072 ]; then + echo "ERROR: Need ≥3GB free in /tmp for QEMU build (have ${AVAIL_MB}MB)" + exit 1 +fi + +if [ -x "$QEMU_DIR/bin/qemu-system-x86_64" ]; then + echo "QEMU already built: $($QEMU_DIR/bin/qemu-system-x86_64 --version | head -1)" + exit 0 +fi + +echo "=== Installing build dependencies ===" +dnf install -y gcc gcc-c++ make ninja-build python3 python3-pip \ + glib2-devel pixman-devel zlib-devel libfdt-devel \ + flex bison diffutils findutils tar gzip xz wget bzip2 +# meson needs a TOML parser on Python <3.11 (AL2023 ships 3.9); tomllib is only +# in the 3.11+ stdlib. Prefer the distro package python3-tomli; fall back to pip +# only if it is not available. +dnf install -y python3-tomli 2>/dev/null || pip3 install tomli 2>/dev/null || true + +# libslirp-devel may not be available — build from source if needed +if ! dnf install -y libslirp-devel 2>/dev/null; then + echo "libslirp-devel not available, building from source..." + dnf install -y meson git 2>/dev/null || pip3 install meson + cd /tmp + wget -q "https://gitlab.freedesktop.org/slirp/libslirp/-/archive/v4.7.0/libslirp-v4.7.0.tar.gz" + tar xf libslirp-v4.7.0.tar.gz + cd libslirp-v4.7.0 + meson setup build --prefix=/usr --default-library=both + ninja -C build install + ldconfig + cd /tmp + rm -rf libslirp-v4.7.0* +fi + +echo "=== Downloading QEMU $QEMU_VERSION ===" +cd /tmp +wget -q "https://download.qemu.org/qemu-${QEMU_VERSION}.tar.xz" +if [ -n "$QEMU_SHA256" ]; then + echo "Verifying sha256 of qemu-${QEMU_VERSION}.tar.xz ..." + echo "${QEMU_SHA256} qemu-${QEMU_VERSION}.tar.xz" | sha256sum -c - || { + echo "ERROR: qemu-${QEMU_VERSION}.tar.xz sha256 mismatch (expected ${QEMU_SHA256})" + exit 1 + } +else + echo "WARNING: QEMU_SHA256 is empty -- skipping archive integrity check" +fi +tar xf "qemu-${QEMU_VERSION}.tar.xz" +cd "qemu-${QEMU_VERSION}" + +echo "=== Configuring (x86_64-softmmu only) ===" +./configure \ + --prefix="$QEMU_DIR" \ + --target-list=x86_64-softmmu \ + --enable-kvm \ + --enable-slirp \ + --disable-docs \ + --disable-user \ + --disable-gtk \ + --disable-sdl \ + --disable-opengl \ + --disable-virglrenderer \ + --disable-xen \ + --disable-spice \ + --disable-vnc \ + --disable-curses + +echo "=== Building (${NPROC} jobs) ===" +make -j"$NPROC" +make install + +echo "=== Cleanup ===" +cd / +rm -rf /tmp/qemu-${QEMU_VERSION}* + +echo "QEMU installed: $($QEMU_DIR/bin/qemu-system-x86_64 --version | head -1)" diff --git a/vm-tests/kvm-unit-tests/common_lib.sh b/vm-tests/kvm-unit-tests/common_lib.sh new file mode 100644 index 0000000..226d63f --- /dev/null +++ b/vm-tests/kvm-unit-tests/common_lib.sh @@ -0,0 +1,458 @@ +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Common functions for the kvm-unit-tests test. +# +# This test exercises the KVM hypervisor *inside* an EC2 guest that has nested +# virtualization enabled (c8i/c7i families, see +# _supports_nested_virtualization() in launch_vm.py). The daily KernelCI cloud +# pipeline installs the target kernel, reboots into it, then runs the upstream +# kvm-unit-tests suite against that kernel's KVM -- a per-kernel regression +# signal for the virtualization stack. + +# --------------------------------------------------------------------------- +# Tunables (overridable via environment for experimentation) +# --------------------------------------------------------------------------- +# Upstream source and pinned revision for reproducible runs. +KVMUT_REPO="${KVMUT_REPO:-https://gitlab.com/kvm-unit-tests/kvm-unit-tests.git}" +KVMUT_REPO_MIRROR="${KVMUT_REPO_MIRROR:-https://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git}" +# Pin to a tag/commit for reproducibility so upstream test-suite changes cannot +# break the pipeline. Validated on c8i 2026-07-10 (upstream v2026-04-17-72-g1da1819e). +# Set KVMUT_REF="" to track the default branch tip instead. +KVMUT_REF="${KVMUT_REF:-1da1819e49fc4938985edca67df669099b4c87a7}" +# Directory (persists across reboots in the client work dir). +KVMUT_DIR="${KVMUT_DIR:-${PWD}/kvm-unit-tests}" +# Observe-only mode: when "true", the test reports counts but always exits 0. +# Default is "false" because the default KVMUT_TESTS subset below is a curated, +# proven-stable set (6/6 PASS on c8i 2026-07-10) suitable as a real gate. Set +# "true" when running the FULL suite (KVMUT_TESTS="") to characterise results +# without failing the pipeline. +KVMUT_OBSERVE_ONLY="${KVMUT_OBSERVE_ONLY:-false}" +# Space-separated test names excluded from the pass/fail decision (known flaky +# or nested-virt-sensitive). Defaults to the full-suite failures observed on +# c8i 2026-07-10 so that a full-suite run still gates on the stable core: +# timeouts (xapic, access, vmx_apicv_test, vmx_posted_intr_test, +# vmx_pf_exception_test), MSR/PMU emulation gaps (msr, msr64, pmu), +# nested-VMX instability (vmx SIGSEGV, la57). +KVMUT_IGNORE_FAILURES="${KVMUT_IGNORE_FAILURES-xapic access vmx_apicv_test vmx_posted_intr_test vmx_pf_exception_test msr msr64 pmu vmx la57}" +# Optional explicit test group to run (passed to run_tests.sh -g). Empty = all. +KVMUT_GROUP="${KVMUT_GROUP:-}" +# Explicit subset of test names to run (space-separated, passed positionally to +# run_tests.sh). Defaults to a small, fast, nested-virt-stable smoke set proven +# to pass on c8i (2026-07-10). Set KVMUT_TESTS="" to run the full suite, or +# KVMUT_TESTS="a b c" to pick your own. +KVMUT_TESTS="${KVMUT_TESTS-debug intel_iommu lam vmx_init_signal_test vmx_sipi_signal_test hyperv_clock}" + +# Error trap handler to show the line where an error occurred +error_trap() +{ + local exit_code=$? + local line_number=$1 + echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" + echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" + exit $exit_code +} +trap 'error_trap $LINENO' ERR + +# Install a single given package (yum first, dnf fallback) +install_package() +{ + local pkg="$1" + local output + echo "Installing package $pkg ..." + if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then + return 0 + else + echo "Failed to install package $pkg:" + echo "$output" + return 1 + fi +} + +# Install all packages listed in dependencies.txt +install_test_dependencies() +{ + local deps_file="${SOURCE_DIR}/dependencies.txt" + if [ -f "$deps_file" ]; then + while IFS= read -r pkg || [ -n "$pkg" ]; do + [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue + pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -n "$pkg" ] && { install_package "$pkg" || return 1; } + done <"$deps_file" + fi +} + +# --------------------------------------------------------------------------- +# Kernel management (reused from unixbench-kernel-regression): install the +# target kernel from the pipeline's shared kernel-rpms area and boot into it, +# so the suite runs against the kernel under test rather than the AMI default. +# --------------------------------------------------------------------------- +# Results bucket + kernel paths, populated from the pipeline environment. +RESULTS_BUCKET="${S3_BUCKET:-}" +ARCH="$(uname -m)" +KERNEL_RPM_DIR="/tmp/kernel-rpms" +KERNEL_FILE="${SOURCE_DIR}/kernel_version_before.txt" + +get_running_kernel() +{ + uname -r +} + +save_kernel_version() +{ + local version="$1" + local out_file="$2" + if [ -z "$version" ] || [ -z "$out_file" ]; then + echo "ERROR: save_kernel_version requires version and file" >&2 + return 1 + fi + echo "$version" >"$out_file" +} + +load_kernel_version() +{ + local in_file="$1" + if [ ! -f "$in_file" ]; then + echo "ERROR: Kernel version file not found: $in_file" >&2 + return 1 + fi + cat "$in_file" +} + +assert_kernel_changed() +{ + local before="$1" + local after="$2" + if [ "$before" = "$after" ]; then + echo "ERROR: kernel version did not change (still $after)" >&2 + return 1 + fi + echo "Kernel version changed from $before to $after" +} + +# List available kernel RPMs from the shared S3 area. +list_kernels_from_s3() +{ + S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX:-}/shared/kernel-rpms/binary/${ARCH}/" + aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' +} + +# Download a specific kernel RPM from S3. +download_kernel_rpm() +{ + if [ -z "${1:-}" ]; then + echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 + return 1 + fi + local kernel_name="$1" + S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX:-}/shared/kernel-rpms/binary/${ARCH}/" + mkdir -p "$KERNEL_RPM_DIR" + local local_path="${KERNEL_RPM_DIR}/${kernel_name}" + if [ -f "$local_path" ]; then + echo "$local_path" + return 0 + fi + if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then + echo "$local_path" + return 0 + else + echo "ERROR: Failed to download kernel" >&2 + return 1 + fi +} + +# Return the lowest-versioned kernel RPM (downloads it from S3). +get_first_kernel_rpm_from_dir() +{ + local kernels=$(list_kernels_from_s3 | sort -V) + local first_kernel=$(echo "$kernels" | head -n 1) + if [ -z "$first_kernel" ]; then + return 1 + fi + download_kernel_rpm "$first_kernel" +} + +# Dump boot configuration for debugging kernel install issues. +dump_boot_info() +{ + echo "=== Boot Debug Info ===" + echo "--- OS ---" + head -2 /etc/os-release 2>/dev/null || true + echo "--- Running kernel ---" + uname -r + echo "--- Installed kernel packages ---" + rpm -qa 'kernel*' | sort + echo "--- vmlinuz files in /boot ---" + ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" + echo "--- BLS entries ---" + ls -la /boot/loader/entries/ 2>/dev/null || echo "(no BLS directory)" + echo "--- grubby default ---" + sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" + echo "=== End Boot Debug Info ===" +} + +# Install a kernel RPM and make it the default boot target. +install_kernel_rpm() +{ + if [ -z "${1:-}" ]; then + echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 + return 1 + fi + local kernel_rpm="$1" + + local host_arch=$(uname -m) + local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) + if [ "$rpm_arch" != "$host_arch" ]; then + echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 + return 1 + fi + + echo "kernel before installation: $(uname -r)" + echo "Installing kernel from $kernel_rpm (arch: $rpm_arch)" + + if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then + dump_boot_info + + # Set the newly installed kernel as default boot target. Without this, + # GRUB may boot a different kernel than the one just installed. + local installed_version + installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) + + local grub_kernel + grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ + | grep "^kernel=" \ + | grep "$installed_version" \ + | head -1 \ + | sed 's/^kernel=//' \ + | tr -d '"' \ + || true) + + if [ -z "$grub_kernel" ]; then + # Upstream make binrpm-pkg kernels don't register with grubby; find + # the vmlinuz file and add a boot entry manually. + local vmlinuz + vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) + if [ -n "$vmlinuz" ]; then + echo "Adding grubby entry for $vmlinuz" + local initrd="/boot/initramfs-${installed_version}.img" + if [ ! -f "$initrd" ]; then + echo "Generating initramfs at $initrd" + sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ + || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ + || true + fi + if [ -f "$initrd" ]; then + sudo grubby --add-kernel="$vmlinuz" \ + --initrd="$initrd" \ + --title="Linux $installed_version" \ + --copy-default \ + --make-default + echo "Added and set default: $vmlinuz" + else + echo "WARNING: No initramfs for $installed_version, trying set-default anyway" + sudo grubby --set-default="$vmlinuz" || true + fi + grub_kernel="$vmlinuz" + else + echo "WARNING: No vmlinuz found for version $installed_version" + fi + else + echo "Setting default boot kernel to $grub_kernel" + sudo grubby --set-default="$grub_kernel" + fi + + if [ -n "$grub_kernel" ]; then + echo "Verifying default kernel:" + sudo grubby --default-kernel + fi + return 0 + else + echo "ERROR: Failed to install new kernel" >&2 + return 1 + fi +} + +# Install a given kernel RPM (path passed as argument). +install_specified_kernel_rpm() +{ + local kernel_rpm="$1" + if [ -z "$kernel_rpm" ]; then + echo "ERROR: install_specified_kernel_rpm requires a kernel RPM path" >&2 + return 1 + fi + echo "Installing kernel RPM: $(basename "$kernel_rpm")" + install_kernel_rpm "$kernel_rpm" +} + +# AL2023 core repos ship no qemu system emulator (only qemu-img), and SPAL also +# does not ship it. So build a minimal x86_64-softmmu QEMU from source via the +# vendored build-qemu.sh (--enable-kvm, installs to /opt/qemu). +# Sets and exports QEMU for run_tests.sh. +install_qemu() +{ + QEMU_BIN="" + if command -v qemu-system-x86_64 >/dev/null 2>&1; then + QEMU_BIN="$(command -v qemu-system-x86_64)" + echo "QEMU already present: $QEMU_BIN" + else + echo "No distro qemu-system-x86_64 on AL2023; building QEMU from source ..." + sudo -E QEMU_VERSION="${QEMU_VERSION:-9.2.0}" bash "${SOURCE_DIR}/build-qemu.sh" + [ -x /opt/qemu/bin/qemu-system-x86_64 ] && QEMU_BIN=/opt/qemu/bin/qemu-system-x86_64 + fi + if [ -z "$QEMU_BIN" ] || [ ! -x "$QEMU_BIN" ]; then + echo "ERROR: no usable qemu-system-x86_64 after build" >&2 + return 1 + fi + export QEMU="$QEMU_BIN" + export PATH="$(dirname "$QEMU_BIN"):$PATH" + echo "Using QEMU: $QEMU ($("$QEMU" --version | head -1))" + return 0 +} + +# Hardware gate: this test profile targets x86_64 with Intel VT-x (VMX) and a +# usable /dev/kvm -- the only configuration AWS EC2 nested virtualization +# supports (Intel C8i/M8i/R8i/C7i/... families). Fail fast and clearly on +# anything else, before the expensive QEMU build. +verify_hardware() +{ + echo "=== Hardware gate ===" + local arch; arch="$(uname -m)" + if [ "$arch" != "x86_64" ]; then + echo "ERROR: unsupported architecture '$arch'." >&2 + echo " This kvm-unit-tests profile supports x86_64 (Intel) only. EC2 nested" >&2 + echo " virtualization is Intel-VT-x only (C8i/M8i/R8i/C7i/... families)." >&2 + echo " aarch64/Graviton and AMD KVM require a bare-metal instance plus an arch-" >&2 + echo " generalized qemu build (aarch64-softmmu / SVM test set), not yet implemented." >&2 + return 1 + fi + if [ ! -e /dev/kvm ]; then + echo "ERROR: /dev/kvm is missing -- hardware virtualization is not available." >&2 + echo " Launch a nested-virt-capable Intel family (C8i/C7i/...) with" >&2 + echo " CpuOptions={NestedVirtualization:enabled}, or use a bare-metal host." >&2 + return 1 + fi + if grep -qw vmx /proc/cpuinfo; then + echo "OK: x86_64 + Intel VT-x (vmx) + /dev/kvm present ($(ls -l /dev/kvm))." + return 0 + elif grep -qw svm /proc/cpuinfo; then + echo "ERROR: AMD SVM detected, but this profile targets Intel VMX." >&2 + echo " The default subset exercises Intel vmx_* tests. Running on AMD needs the" >&2 + echo " SVM test set on a bare-metal AMD host (EC2 nested virt is Intel-only)." >&2 + return 1 + else + echo "ERROR: no CPU virtualization flag (vmx/svm) exposed to the guest CPU." >&2 + return 1 + fi +} + +# Confirm nested virtualization actually reached the guest. This doubles as a +# regression check on the EC2 nested-virt enablement itself. +verify_nested_virt() +{ + echo "=== Verifying nested virtualization is available ===" + if [ ! -e /dev/kvm ]; then + echo "ERROR: /dev/kvm is missing -- nested virtualization is NOT enabled on this instance." >&2 + echo " Ensure the VM uses a supported family (c8i/c7i/...) so launch_vm.py sets" >&2 + echo " CpuOptions={NestedVirtualization: enabled}." >&2 + return 1 + fi + echo "/dev/kvm present:" + ls -l /dev/kvm + grep -m1 -oE 'vmx|svm' /proc/cpuinfo | head -1 || { + echo "ERROR: no vmx/svm virtualization flag exposed to the guest CPU." >&2 + return 1 + } + echo "Virtualization CPU flag exposed to guest: OK" + return 0 +} + +# Fetch kvm-unit-tests (primary repo, then kernel.org mirror as fallback). +fetch_kvm_unit_tests() +{ + if [ -d "${KVMUT_DIR}/.git" ]; then + echo "kvm-unit-tests already checked out at ${KVMUT_DIR}" + return 0 + fi + echo "=== Cloning kvm-unit-tests ===" + if ! git clone "${KVMUT_REPO}" "${KVMUT_DIR}"; then + echo "Primary clone failed, trying mirror ${KVMUT_REPO_MIRROR}" + git clone "${KVMUT_REPO_MIRROR}" "${KVMUT_DIR}" + fi + if [ -n "${KVMUT_REF}" ]; then + echo "Checking out pinned ref ${KVMUT_REF}" + git -C "${KVMUT_DIR}" checkout --detach "${KVMUT_REF}" + fi + ( cd "${KVMUT_DIR}" && git submodule update --init >/dev/null 2>&1 || true ) + echo "kvm-unit-tests revision: $(git -C "${KVMUT_DIR}" rev-parse --short HEAD)" +} + +# Configure + build the test binaries. +build_kvm_unit_tests() +{ + echo "=== Building kvm-unit-tests ===" + ( cd "${KVMUT_DIR}" && ./configure && make -j"$(nproc)" ) +} + +# Run the suite, tee-ing full output to a log. Uses standalone tests so a single +# invocation runs the whole set. Returns run_tests.sh's own exit code. +run_kvm_unit_tests() +{ + local logfile="$1" + echo "=== Running kvm-unit-tests (accel=kvm) ===" + local args=(-v) + [ -n "${KVMUT_GROUP}" ] && args+=(-g "${KVMUT_GROUP}") + # Optional explicit subset of test names (positional args to run_tests.sh). + local tests=() + [ -n "${KVMUT_TESTS}" ] && read -r -a tests <<< "${KVMUT_TESTS}" + [ ${#tests[@]} -gt 0 ] && echo "Running subset: ${tests[*]}" + # ACCEL=kvm forces hardware acceleration; the run fails loudly if KVM is + # unavailable rather than silently falling back to slow TCG emulation. + ( cd "${KVMUT_DIR}" && ACCEL=kvm ./run_tests.sh "${args[@]}" "${tests[@]}" ) 2>&1 | tee "$logfile" + return "${PIPESTATUS[0]}" +} + +# Parse run_tests.sh output into PASS/FAIL/SKIP counts and a per-test CSV. +# Writes: +# result.txt human summary +# kvm-unit-tests.csv machine-readable per-test rows for KCIDB ingestion +# Sets globals: KVMUT_PASS KVMUT_FAIL KVMUT_SKIP KVMUT_REAL_FAIL +summarize_kvm_unit_tests() +{ + local logfile="$1" + local csv="$2" + local kver arch clean + kver="$(uname -r)" + arch="$(uname -m)" + + # run_tests.sh emits ANSI colour codes (e.g. \e[32mPASS\e[0m), so strip them + # to a clean copy before matching line-anchored PASS/FAIL/SKIP. + clean="$(mktemp)" + sed -E 's/\x1b\[[0-9;]*m//g' "$logfile" >"$clean" + + KVMUT_PASS=$(grep -cE '^PASS ' "$clean" || true) + KVMUT_FAIL=$(grep -cE '^FAIL ' "$clean" || true) + KVMUT_SKIP=$(grep -cE '^SKIP ' "$clean" || true) + + # Per-test CSV, KCIDB-friendly: test_name,status,kernel_version,arch + echo "test_name,status,kernel_version,arch" >"$csv" + # Guard against pipefail when there are zero matches. + grep -E '^(PASS|FAIL|SKIP) ' "$clean" | while read -r status name _rest; do + echo "${name},${status},${kver},${arch}" >>"$csv" + done || true + + # Count "real" failures excluding the ignore list. + KVMUT_REAL_FAIL=0 + local name + while read -r _status name _rest; do + [ -z "$name" ] && continue + local ignored=0 + for ig in ${KVMUT_IGNORE_FAILURES}; do + [ "$name" = "$ig" ] && ignored=1 && break + done + [ "$ignored" -eq 0 ] && KVMUT_REAL_FAIL=$((KVMUT_REAL_FAIL + 1)) + done < <(grep -E '^FAIL ' "$clean" || true) + + rm -f "$clean" + echo "kvm-unit-tests summary: PASS=${KVMUT_PASS} FAIL=${KVMUT_FAIL} SKIP=${KVMUT_SKIP} (real failures after ignore list: ${KVMUT_REAL_FAIL})" +} diff --git a/vm-tests/kvm-unit-tests/dependencies.txt b/vm-tests/kvm-unit-tests/dependencies.txt new file mode 100644 index 0000000..3f96be8 --- /dev/null +++ b/vm-tests/kvm-unit-tests/dependencies.txt @@ -0,0 +1,7 @@ +# Packages required to build and run kvm-unit-tests. +# QEMU is installed separately by install_qemu() in common_lib.sh because the +# package name varies across distros/versions. +git +gcc +make +binutils diff --git a/vm-tests/kvm-unit-tests/external_requirements.json b/vm-tests/kvm-unit-tests/external_requirements.json new file mode 100644 index 0000000..fd217a4 --- /dev/null +++ b/vm-tests/kvm-unit-tests/external_requirements.json @@ -0,0 +1,4 @@ +{ + "kernel-rpms/src": false, + "kernel-rpms/binary": true +} diff --git a/vm-tests/kvm-unit-tests/run-01-install-kernel.sh b/vm-tests/kvm-unit-tests/run-01-install-kernel.sh new file mode 100755 index 0000000..8f542c5 --- /dev/null +++ b/vm-tests/kvm-unit-tests/run-01-install-kernel.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# First stage: install the target kernel from the pipeline's shared kernel-rpms +# area and boot into it, so run-02 exercises kvm-unit-tests against the kernel +# under test rather than the AMI default. The kernel-management flow is reused +# from unixbench-kernel-regression. After this script the client reboots (a +# second run-*.sh follows), bringing up the installed kernel. + +set -euxo pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# Record the kernel we booted from so run-02 can confirm the switch. +kernel_before="$(get_running_kernel)" +echo "Kernel before installation: $kernel_before" +save_kernel_version "$kernel_before" "$KERNEL_FILE" + +# Install the target kernel (lowest version in the shared kernel-rpms area). +# If no kernel RPMs are provided (e.g. a standalone run without the pipeline's +# S3 layout), keep the current kernel and let run-02 test that instead. +if first_kernel="$(get_first_kernel_rpm_from_dir)"; then + install_specified_kernel_rpm "$first_kernel" + echo "Installed target kernel; the client reboots into it before run-02." +else + echo "No kernel RPMs found in S3; keeping the current kernel for run-02." +fi + +# Re-execution continues in run-02-kvm-unit-tests.sh after the reboot. diff --git a/vm-tests/kvm-unit-tests/run-02-kvm-unit-tests.sh b/vm-tests/kvm-unit-tests/run-02-kvm-unit-tests.sh new file mode 100755 index 0000000..6a6efc2 --- /dev/null +++ b/vm-tests/kvm-unit-tests/run-02-kvm-unit-tests.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Run the upstream kvm-unit-tests suite against the guest KVM, exercising the +# kernel's virtualization stack on an EC2 instance with nested virtualization +# enabled. Second stage: runs after run-01 installed the target kernel and the +# client rebooted into it. + +set -euxo pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# 0. Confirm we booted the target kernel that run-01 installed. When run-01 did +# not install one (no kernel RPMs provided), test the current kernel. +if [ -f "${KERNEL_FILE}" ]; then + kernel_before="$(load_kernel_version "${KERNEL_FILE}")" + kernel_now="$(get_running_kernel)" + echo "Kernel before run-01 install: ${kernel_before}; now running: ${kernel_now}" + assert_kernel_changed "${kernel_before}" "${kernel_now}" +else + echo "No recorded pre-install kernel; testing the current kernel ($(get_running_kernel))." +fi + +# 1. Hardware gate -- fail fast and clearly on wrong arch / no KVM, before +# installing anything or building QEMU. +verify_hardware || exit 1 + +# 2. Dependencies (build toolchain + QEMU) +install_test_dependencies +install_qemu + +# 3. Confirm nested virt actually reached this guest (regression check). +verify_nested_virt + +# 4. Fetch + build the suite. +fetch_kvm_unit_tests +build_kvm_unit_tests + +# 5. Run and capture output. +LOG="${PWD}/kvm-unit-tests.log" +set +e +run_kvm_unit_tests "$LOG" +RUN_RC=$? +set -e + +# 6. Summarise into a results file + CSV for KCIDB. The `results_` prefix makes +# test-vm-client.sh upload both from the test dir (it globs results_* and +# benchmark-*.csv). The authoritative SUCCESS/FAILED verdict is written by +# the client from this script's exit code. +summarize_kvm_unit_tests "$LOG" "${PWD}/results_kvm-unit-tests.csv" + +{ + echo "kvm-unit-tests result" + echo "kernel: $(uname -r) arch: $(uname -m)" + echo "PASS=${KVMUT_PASS} FAIL=${KVMUT_FAIL} SKIP=${KVMUT_SKIP}" + echo "real_failures=${KVMUT_REAL_FAIL} (ignore list: ${KVMUT_IGNORE_FAILURES:-})" + echo "run_tests.sh exit code: ${RUN_RC}" +} >"${PWD}/results_kvm-unit-tests.txt" +cat "${PWD}/results_kvm-unit-tests.txt" + +echo "Done executing kvm-unit-tests after $SECONDS seconds" + +# 7. Decide pass/fail. +# Observe-only mode reports but never fails the pipeline -- used while the +# stable subset of tests under nested virt is still being characterised. +if [ "${KVMUT_OBSERVE_ONLY}" = "true" ]; then + echo "OBSERVE-ONLY mode: not gating on failures (real_failures=${KVMUT_REAL_FAIL})." + exit 0 +fi + +if [ "${KVMUT_REAL_FAIL}" -gt 0 ]; then + echo "ERROR: ${KVMUT_REAL_FAIL} kvm-unit-tests failed (after ignore list)." >&2 + exit 1 +fi + +echo "kvm-unit-tests passed: no unexpected failures." +exit 0