test: persist nested-e2e edgeHub store across container recreate (diagnostic)

builds/e2e/nested-e2e.yaml

@@ -152,8 +152,52 @@ stages:
     - LockAgents
     - RunNestedTests
   jobs:
+  # TEMP (investigation, not for merge): capture support bundles from ALL levels (L3/L4/L5)
+  # after the test stage, so the L4/L5 edgeHub/relayer logs are available to debug
+  # RouteMessageL3LeafToL4Module. Runs before Clean_images so containers/logs still exist.
+  - job: Collect_Nested_Bundles
+    displayName: Collect nested support bundles
+    condition: always()
+    strategy:
+      matrix:
+        L3:
+          level: 3
+        L4:
+          level: 4
+        L5:
+          level: 5
+    pool:
+      name: $(pool.name)
+      demands:
+        - agent-group -equals $(agent.group)
+        - Agent.OS -equals Linux
+        - Agent.OSArchitecture -equals X64
+        - status -equals locked_$(Build.BuildId)_L$(level)
+    steps:
+      - bash: |
+          set +e
+          out="$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)"
+          mkdir -p "$out"
+          echo "Collecting support bundle on L$(level) ($(hostname))"
+          sudo iotedge support-bundle --output "$out/support_bundle_L$(level).zip" || echo "support-bundle failed on L$(level)"
+          # Also grab raw module logs as a fallback in case support-bundle errors
+          for m in edgeHub edgeAgent relayer1; do
+            sudo docker logs "$m" > "$out/$m-L$(level).log" 2>&1 || true
+          done
+          ls -la "$out" || true
+        displayName: 'Generate support bundle (L$(level))'
+        condition: always()
+      - task: PublishBuildArtifacts@1
+        displayName: 'Publish nested bundle (L$(level))'
+        condition: always()
+        inputs:
+          PathtoPublish: '$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)'
+          ArtifactName: 'nested-support-bundles'
+
   - job: Clean_images
     displayName: Clean up Docker images
+    dependsOn: Collect_Nested_Bundles
+    condition: always()
     strategy:
       matrix:
         L3:

e2e_deployment_files/nestededge_middleLayerBaseDeployment_amqp.json

@@ -27,9 +27,12 @@
             "type": "docker",
             "settings": {
               "image": "$upstream:443/microsoft/azureiotedge-hub:<Build.BuildNumber>-linux-<Architecture>",
-              "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}}}"
+              "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}, \"Binds\": [\"/etc/iotedge/storage/:/iotedge/storage/\"]}}"
             },
             "env": {
+              "StorageFolder": {
+                "value": "/iotedge/storage"
+              },
               "DeviceScopeCacheRefreshDelaySecs": {
                 "value": 0
               },

e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json

@@ -27,9 +27,12 @@
                         "type": "docker",
                         "settings": {
                             "image": "$upstream:443/microsoft/azureiotedge-hub:<Build.BuildNumber>-linux-<Architecture>",
-                            "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}}}"
+                            "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}, \"Binds\": [\"/etc/iotedge/storage/:/iotedge/storage/\"]}}"
                         },
                         "env": {
+                            "StorageFolder": {
+                                "value": "/iotedge/storage"
+                            },
                             "experimentalFeatures__enabled": {
                                 "value": "true"
                             },

test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs

@@ -97,7 +97,19 @@ public Task<Device> GetDeviceIdentityAsync(string deviceId, CancellationToken to
 
         public async Task<Device> CreateDeviceIdentityAsync(Device device, CancellationToken token)
         {
-            return await this.RegistryManager.AddDeviceAsync(device, token);
+            try
+            {
+                return await this.RegistryManager.AddDeviceAsync(device, token);
+            }
+            catch (DeviceAlreadyExistsException)
+            {
+                // A prior test run can leave an orphaned identity behind (for example when the
+                // job is killed during artifact upload before cleanup runs). Remove the stale
+                // identity and recreate so the repro run isn't blocked by leftover state.
+                Log.Warning($"Device identity '{device.Id}' already exists; deleting orphaned identity and recreating.");
+                await this.RegistryManager.RemoveDeviceAsync(device.Id, token);
+                return await this.RegistryManager.AddDeviceAsync(device, token);
+            }
         }
 
         public async Task<Device> CreateEdgeDeviceIdentityAsync(string deviceId, Option<string> parentDeviceId, AuthenticationType authType, X509Thumbprint x509Thumbprint, CancellationToken token)