Cherry-pick build/test updates from main

builds/checkin/e2e-checkin.yaml

@@ -75,7 +75,7 @@ stages:
             containerRegistryUsername: '$(acrUsername)'
             dpsIdScope: '$(dps.idScope)'
             eventHubName: '$(eventhub.name)'
-            eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+            eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
             iotHubHostName: '$(iothub.hostname)'
             iotHubResourceId: '$(iothub.resourceid)'
             rootCaCertificate: '$(rootCaCertificate)'

builds/e2e/e2e.yaml

@@ -80,7 +80,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -131,7 +131,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -179,7 +179,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -229,7 +229,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -280,7 +280,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -330,7 +330,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -380,7 +380,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -428,7 +428,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -491,7 +491,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -539,7 +539,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -587,7 +587,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -637,7 +637,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -692,7 +692,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'
@@ -745,7 +745,7 @@ jobs:
         containerRegistryUsername: '$(containerRegistryUsername)'
         dpsIdScope: '$(dps.idScope)'
         eventHubName: '$(eventhub.name)'
-        eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+        eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
         iotHubHostName: '$(iothub.hostname)'
         iotHubResourceId: '$(iothub.resourceid)'
         rootCaCertificate: '$(rootCaCertificate)'

builds/e2e/isa-95-smoke-test.yaml

@@ -77,8 +77,8 @@ stages:
           artifactName: iotedged-ubuntu24.04-amd64
         arm64:
           arch: ARM64
-          identityArtifactName: aziot-identity-ubuntu22.04-aarch64
-          artifactName: iotedged-ubuntu22.04-aarch64
+          identityArtifactName: aziot-identity-ubuntu24.04-aarch64
+          artifactName: iotedged-ubuntu24.04-aarch64
     variables:
       level: 5
       upstreamProtocol: amqp
@@ -145,8 +145,8 @@ stages:
           parentDeviceId: $(x64ParentDeviceId)
         arm64:
           arch: ARM64
-          identityArtifactName: aziot-identity-ubuntu22.04-aarch64
-          artifactName: iotedged-ubuntu22.04-aarch64
+          identityArtifactName: aziot-identity-ubuntu24.04-aarch64
+          artifactName: iotedged-ubuntu24.04-aarch64
           parentName: $(arm64ParentName)
           parentDeviceId: $(arm64ParentDeviceId)
     pool:
@@ -206,8 +206,8 @@ stages:
           parentDeviceId: $(x64ParentDeviceId)
         arm64:
           arch: ARM64
-          identityArtifactName: aziot-identity-ubuntu22.04-aarch64
-          artifactName: iotedged-ubuntu22.04-aarch64
+          identityArtifactName: aziot-identity-ubuntu24.04-aarch64
+          artifactName: iotedged-ubuntu24.04-aarch64
           parentName: $(arm64ParentName)
           parentDeviceId: $(arm64ParentDeviceId)
     pool:
@@ -327,7 +327,7 @@ stages:
           containerRegistryServer: '$(containerRegistryServer)'
           containerRegistryUsername: '$(containerRegistryUsername)'
           eventHubName: '$(eventhub.name)'
-          eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+          eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
           iotHubHostName: '$(iothub.hostname)'
           rootCaCertificate: '$(rootCaCertificate)'
           rootCaKey: '$(rootCaKey)'

builds/e2e/nested-e2e.yaml

@@ -103,7 +103,7 @@ stages:
           containerRegistryServer: '$(containerRegistryServer)'
           containerRegistryUsername: '$(containerRegistryUsername)'
           eventHubName: '$(eventhub.name)'
-          eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)'
+          eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)'
           iotHubHostName: '$(iothub.hostname)'
           iotHubResourceId: '$(iothub.resourceid)'
           rootCaCertificate: '$(rootCaCertificate)'
@@ -152,8 +152,52 @@ stages:
     - LockAgents
     - RunNestedTests
   jobs:
+  # Capture support bundles from the upper nested levels (L4/L5) after the test stage,
+  # so the L4/L5 edgeHub/relayer logs are available to debug nested routing failures
+  # (for example RouteMessageL3LeafToL4Module). The existing per-test teardown already
+  # publishes L3, so L3 is intentionally omitted here. Runs before Clean_images so
+  # containers/logs still exist.
+  - job: Collect_Nested_Bundles
+    displayName: Collect nested support bundles
+    condition: always()
+    strategy:
+      matrix:
+        L4:
+          level: 4
+        L5:
+          level: 5
+    pool:
+      name: $(pool.name)
+      demands:
+        - agent-group -equals $(agent.group)
+        - Agent.OS -equals Linux
+        - Agent.OSArchitecture -equals X64
+        - status -equals locked_$(Build.BuildId)_L$(level)
+    steps:
+      - bash: |
+          set +e
+          out="$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)"
+          mkdir -p "$out"
+          echo "Collecting support bundle on L$(level) ($(hostname))"
+          sudo iotedge support-bundle --output "$out/support_bundle_L$(level).zip" || echo "support-bundle failed on L$(level)"
+          # Also grab raw module logs as a fallback in case support-bundle errors
+          for m in edgeHub edgeAgent relayer1; do
+            sudo docker logs "$m" > "$out/$m-L$(level).log" 2>&1 || true
+          done
+          ls -la "$out" || true
+        displayName: 'Generate support bundle (L$(level))'
+        condition: always()
+      - task: PublishBuildArtifacts@1
+        displayName: 'Publish nested bundle (L$(level))'
+        condition: always()
+        inputs:
+          PathtoPublish: '$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)'
+          ArtifactName: 'nested-support-bundles'
+
   - job: Clean_images
     displayName: Clean up Docker images
+    dependsOn: Collect_Nested_Bundles
+    condition: always()
     strategy:
       matrix:
         L3:
@@ -176,7 +220,7 @@ stages:
     displayName: Unlock agents
     dependsOn: Clean_images
     condition: always()
-    timeoutInMinutes: 2
+    timeoutInMinutes: 5
     pool:
       name: $(pool.name)
       demands:
@@ -191,7 +235,7 @@ stages:
     displayName: Clean up identities
     dependsOn: Clean_images
     condition: always()
-    timeoutInMinutes: 2
+    timeoutInMinutes: 5
     variables:    
       deviceLvl5DeviceId: $[ stageDependencies.RunNestedTests.SetupVM_level5_mqtt.outputs['createIdentity.parentDeviceId'] ] 
       deviceLvl4DeviceId: $[ stageDependencies.RunNestedTests.SetupVM_level4_mqtt.outputs['createIdentity.parentDeviceId'] ] 

builds/e2e/templates/e2e-setup.yaml

@@ -3,7 +3,7 @@ parameters:
   containerRegistryUsername: ''
   dpsIdScope: 'optional'
   eventHubName: ''
-  eventHubFullyQualitifiedNamespace: ''
+  eventHubFullyQualifiedNamespace: ''
   iotHubHostName: ''
   iotHubResourceId: 'optional'
   rootCaCertificate: ''
@@ -168,7 +168,7 @@ steps:
 
       $context = @{
         eventHubName = '${{ parameters.eventHubName }}';
-        eventHubFullyQualitifiedNamespace = '${{ parameters.eventHubFullyQualitifiedNamespace }}';
+        eventHubFullyQualifiedNamespace = '${{ parameters.eventHubFullyQualifiedNamespace }}';
         iotHubHostName = '${{ parameters.iotHubHostName }}';
         nestededge = "$nestededge";
         dpsIdScope = '${{ parameters.dpsIdScope }}';

builds/e2e/templates/nested-deploy-config.yaml

@@ -47,3 +47,64 @@ steps:
         # 5/22/2024 - Temporary work around the issue where the az cli command cannot authorize itself within *.sh script using the service principal's service connection
         deployment_working_file="$(Agent.HomeDirectory)/../working/deployment.json"
         az iot edge set-modules --auth-type login --device-id "${{ parameters.deviceId }}" --hub-name "$(iotHubName)" --content ${deployment_working_file} --output none
+
+        # Wait for the device's edgeAgent to finish reconciling this deployment before
+        # returning. set-modules only writes the desired twin; the device applies it
+        # asynchronously (which restarts edgeHub). Without this barrier the next test
+        # group can start while a parent is still reconciling, dropping a leaf's upstream
+        # link mid-test (see RouteMessageL3LeafToL4Module ~4min stall). Gate on edgeAgent's
+        # reported lastDesiredStatus.code == 200 (Success) for the desired version.
+        deviceId="${{ parameters.deviceId }}"
+        hubName="$(iotHubName)"
+        desiredVersion=$(az iot hub module-twin show --auth-type login --device-id "$deviceId" --module-id '$edgeAgent' --hub-name "$hubName" --query 'properties.desired."$version"' --output tsv)
+        echo "Waiting for edgeAgent on $deviceId to reconcile desired version $desiredVersion ..."
+        reconcileTimeoutSecs=300
+        reconcilePollSecs=5
+        elapsed=0
+        while true; do
+          # Single twin read per poll: az emits the array query as three lines (one per
+          # element) with --output tsv, so read them into an array. Fewer az invocations
+          # than one call per field = fewer potential failure points per iteration.
+          mapfile -t twin < <(
+            az iot hub module-twin show \
+              --auth-type login \
+              --device-id "$deviceId" \
+              --module-id '$edgeAgent' \
+              --hub-name "$hubName" \
+              --query '[properties.reported.lastDesiredVersion, properties.reported.lastDesiredStatus.code, properties.reported.lastDesiredStatus.description]' \
+              --output tsv \
+              2>/dev/null
+          )
+          reportedVersion="${twin[0]}"
+          reportedCode="${twin[1]}"
+          reportedDescription="${twin[2]}"
+          echo "  [$elapsed s] reported version=$reportedVersion status=$reportedCode (want version=$desiredVersion status=200)"
+          if [ "$reportedVersion" = "$desiredVersion" ] && [ "$reportedCode" = "200" ]; then
+            echo "edgeAgent on $deviceId reconciled desired version $desiredVersion successfully."
+            break
+          fi
+          if [ "$elapsed" -ge "$reconcileTimeoutSecs" ]; then
+            # Two distinct failure modes, both diagnosable from the values already read above:
+            #  - reportedVersion never reached desiredVersion => edgeAgent never picked up the
+            #    deployment (twin write lost, agent down, or hub connectivity issue).
+            #  - reportedVersion matched but code stayed non-200 => deployment applied but a
+            #    module is unhealthy (bad image ref, crash loop, pull failure).
+            echo "----- edgeAgent reconcile timeout diagnostics for $deviceId -----"
+            echo "desired version : $desiredVersion"
+            echo "reported version: $reportedVersion"
+            echo "reported status : $reportedCode"
+            echo "status detail   : $reportedDescription"
+            if [ "$reportedVersion" != "$desiredVersion" ]; then
+              echo "diagnosis       : edgeAgent never picked up desired version $desiredVersion (still on $reportedVersion) - check agent connectivity/liveness"
+            else
+              echo "diagnosis       : deployment applied (version matched) but stuck at status $reportedCode - a module is likely unhealthy (bad image, crash loop, or pull failure)"
+            fi
+            echo "reported module states:"
+            az iot hub module-twin show --auth-type login --device-id "$deviceId" --module-id '$edgeAgent' --hub-name "$hubName" --query 'properties.reported.modules' --output json 2>/dev/null || true
+            echo "----------------------------------------------------------------"
+            echo "##vso[task.logissue type=error]edgeAgent on $deviceId did not reconcile desired version $desiredVersion within ${reconcileTimeoutSecs}s (last reported version=$reportedVersion status=$reportedCode: $reportedDescription)"
+            exit 1
+          fi
+          sleep "$reconcilePollSecs"
+          elapsed=$((elapsed + reconcilePollSecs))
+        done

builds/service/service-deployment.yaml

@@ -56,7 +56,7 @@ steps:
     $context = @{
       logFile = Join-Path '$(binDir)' 'testoutput.log';
       eventHubName = '$(eventhub.name)';
-      eventHubFullyQualitifiedNamespace = '$(eventhub.namespace.hostname)';
+      eventHubFullyQualifiedNamespace = '$(eventhub.namespace.hostname)';
       iotHubHostName = '$(iothub.hostname)';
       caCertScriptPath = Convert-Path '$(Build.SourcesDirectory)/tools/CACertificates';
       rootCaCertificatePath = Convert-Path '$(certsDir)/rsa_root_ca.cert.pem';

edgelet/build/linux/package-mariner.sh

@@ -30,8 +30,8 @@ apt-get install -y \
     llvm-dev make pigz pkg-config python3-distutils python3-pip qemu-utils rpm tar \
     wget zstd
 
-# Install Go 1.23
-GO_VERSION=1.23.0
+# Install Go 1.24
+GO_VERSION=1.24.0
 [ "$ARCH" == 'aarch64' ] && GO_ARCH='arm64' || GO_ARCH='amd64'
 mkdir -p /usr/local/go
 curl -sSL "https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | tar -C /usr/local -xzf -