diff --git a/Release.toml b/Release.toml index 802c3b6a33d..0e8ae7a6ec8 100644 --- a/Release.toml +++ b/Release.toml @@ -477,5 +477,6 @@ version = "1.64.0" ] "(1.63.0, 1.64.0)" = [ "migrate_v1.64.0_kubernetes-container-runtime-endpoint.lz4", - "migrate_v1.64.0_hugepages-settings.lz4" + "migrate_v1.64.0_hugepages-settings.lz4", + "migrate_v1.64.0_kubelet-dra-drivers-nvidia-enabled.lz4" ] diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 578573211e6..49699860f39 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1178,6 +1178,13 @@ dependencies = [ "serde_json", ] +[[package]] +name = "kubelet-dra-drivers-nvidia-enabled" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "kubelet-setting-container-log-single-process-oom-kill" version = "0.1.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index f3318f335c0..77a7c6f0e39 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -67,6 +67,7 @@ members = [ "settings-migrations/v1.63.0/nvidia-k8s-device-plugin-enabled", "settings-migrations/v1.64.0/kubernetes-container-runtime-endpoint", "settings-migrations/v1.64.0/hugepages-settings", + "settings-migrations/v1.64.0/kubelet-dra-drivers-nvidia-enabled", "settings-plugins/aws-dev", "settings-plugins/aws-ecs-2", diff --git a/sources/settings-defaults/aws-k8s-1.33-nvidia/defaults.d/83-nvidia-dra-driver-gpu.toml b/sources/settings-defaults/aws-k8s-1.33-nvidia/defaults.d/83-nvidia-dra-driver-gpu.toml new file mode 120000 index 00000000000..d2464895e30 --- /dev/null +++ b/sources/settings-defaults/aws-k8s-1.33-nvidia/defaults.d/83-nvidia-dra-driver-gpu.toml @@ -0,0 +1 @@ +../../../shared-defaults/nvidia-dra-driver-gpu.toml \ No newline at end of file diff --git a/sources/settings-defaults/aws-k8s-1.36-nvidia/defaults.d/83-nvidia-dra-driver-gpu.toml b/sources/settings-defaults/aws-k8s-1.36-nvidia/defaults.d/83-nvidia-dra-driver-gpu.toml new file mode 120000 index 00000000000..d2464895e30 --- /dev/null +++ b/sources/settings-defaults/aws-k8s-1.36-nvidia/defaults.d/83-nvidia-dra-driver-gpu.toml @@ -0,0 +1 @@ +../../../shared-defaults/nvidia-dra-driver-gpu.toml \ No newline at end of file diff --git a/sources/settings-migrations/v1.64.0/kubelet-dra-drivers-nvidia-enabled/Cargo.toml b/sources/settings-migrations/v1.64.0/kubelet-dra-drivers-nvidia-enabled/Cargo.toml new file mode 100644 index 00000000000..d2a660e47d6 --- /dev/null +++ b/sources/settings-migrations/v1.64.0/kubelet-dra-drivers-nvidia-enabled/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "kubelet-dra-drivers-nvidia-enabled" +version = "0.1.0" +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false +exclude = ["README.md"] + +[dependencies] +migration-helpers.workspace = true diff --git a/sources/settings-migrations/v1.64.0/kubelet-dra-drivers-nvidia-enabled/src/main.rs b/sources/settings-migrations/v1.64.0/kubelet-dra-drivers-nvidia-enabled/src/main.rs new file mode 100644 index 00000000000..efe177f778b --- /dev/null +++ b/sources/settings-migrations/v1.64.0/kubelet-dra-drivers-nvidia-enabled/src/main.rs @@ -0,0 +1,27 @@ +use migration_helpers::common_migrations::AddSettingsMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +/// We added a new setting for enabling/disabling the NVIDIA DRA driver +/// (gpu-kubelet-plugin) host service: +/// - settings.kubelet-dra-drivers.nvidia.enabled +/// +/// Forward (upgrade): no-op. The `enabled` field defaults to `None` (the +/// service treats unset as disabled), so existing data is compatible. +/// Backward (downgrade): removes settings.kubelet-dra-drivers.nvidia.enabled from the +/// datastore so older versions don't encounter an unknown field. +fn run() -> Result<()> { + migrate(AddSettingsMigration(&[ + "settings.kubelet-dra-drivers.nvidia.enabled", + ])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{e}"); + process::exit(1); + } +} diff --git a/sources/settings-plugins/aws-k8s-nvidia/src/lib.rs b/sources/settings-plugins/aws-k8s-nvidia/src/lib.rs index 973317a808c..11f5329b463 100644 --- a/sources/settings-plugins/aws-k8s-nvidia/src/lib.rs +++ b/sources/settings-plugins/aws-k8s-nvidia/src/lib.rs @@ -27,4 +27,5 @@ struct AwsK8sSettings { autoscaling: bottlerocket_settings_models::AutoScalingSettingsV1, nvidia_container_runtime: bottlerocket_settings_models::NvidiaContainerRuntimeSettingsV1, kubelet_device_plugins: bottlerocket_settings_models::KubeletDevicePluginsV1, + kubelet_dra_drivers: bottlerocket_settings_models::KubeletDraDriversV1, } diff --git a/sources/shared-defaults/nvidia-dra-driver-gpu.toml b/sources/shared-defaults/nvidia-dra-driver-gpu.toml new file mode 100644 index 00000000000..f383ba7de1f --- /dev/null +++ b/sources/shared-defaults/nvidia-dra-driver-gpu.toml @@ -0,0 +1,27 @@ +# NVIDIA DRA driver (GPU kubelet plugin) service. +# +# The plugin runs as a host systemd service (not a Helm-installed DaemonSet). +# Its ExecStart is rendered from settings because it needs the node name from +# settings.kubernetes.hostname-override; enablement is the dedicated +# settings.kubelet-dra-drivers.nvidia.enabled toggle (independent of the classic +# device plugin's settings.kubelet-device-plugins.nvidia.enabled). +[services.nvidia-dra-driver-gpu] +restart-commands = [ + "/bin/systemctl try-reload-or-restart nvidia-dra-driver-gpu.service", +] +configuration-files = [ + "nvidia-dra-driver-gpu-exec-start-conf", +] + +[configuration-files.nvidia-dra-driver-gpu-exec-start-conf] +path = "/etc/systemd/system/nvidia-dra-driver-gpu.service.d/exec-start.conf" +template-path = "/usr/share/templates/nvidia-dra-driver-gpu-exec-start-conf" + +# Opt-in by default: the DRA driver is off unless explicitly enabled, so it does +# not conflict with the classic device plugin out of the box. +[settings.kubelet-dra-drivers.nvidia] +enabled = false + +# Re-render + restart the service when the toggle (or node name) changes. +[metadata.settings.kubelet-dra-drivers.nvidia.enabled] +affected-services = ["nvidia-dra-driver-gpu"] diff --git a/variants/aws-k8s-1.34-nvidia/Cargo.toml b/variants/aws-k8s-1.34-nvidia/Cargo.toml index 98c1cb671bd..2d380b364a6 100644 --- a/variants/aws-k8s-1.34-nvidia/Cargo.toml +++ b/variants/aws-k8s-1.34-nvidia/Cargo.toml @@ -42,6 +42,7 @@ included-packages = [ # nvidia "nvidia-container-toolkit-k8s", "nvidia-k8s-device-plugin", + "nvidia-dra-driver-gpu", "kmod-6.12-nvidia-r580-tesla", ] kernel-parameters = [ diff --git a/variants/aws-k8s-1.35-nvidia/Cargo.toml b/variants/aws-k8s-1.35-nvidia/Cargo.toml index d6dccb3d6d2..f585f5daafc 100644 --- a/variants/aws-k8s-1.35-nvidia/Cargo.toml +++ b/variants/aws-k8s-1.35-nvidia/Cargo.toml @@ -42,6 +42,7 @@ included-packages = [ # nvidia "nvidia-container-toolkit-k8s", "nvidia-k8s-device-plugin", + "nvidia-dra-driver-gpu", "kmod-6.12-nvidia-r580-tesla", ] kernel-parameters = [ diff --git a/variants/aws-k8s-1.36-nvidia/Cargo.toml b/variants/aws-k8s-1.36-nvidia/Cargo.toml index b4083c1b30b..d649ce698b8 100644 --- a/variants/aws-k8s-1.36-nvidia/Cargo.toml +++ b/variants/aws-k8s-1.36-nvidia/Cargo.toml @@ -42,6 +42,7 @@ included-packages = [ # nvidia "nvidia-container-toolkit-k8s", "nvidia-k8s-device-plugin", + "nvidia-dra-driver-gpu", "kmod-6.18-nvidia-r580-tesla", ] kernel-parameters = [