Test snaps on Ubuntu Core

builds/e2e/e2e.yaml

@@ -384,6 +384,39 @@ jobs:
       parameters:
         sas_uri: $(sas_uri)
 
+################################################################################
+  - job: snaps_ubuntu_core
+################################################################################
+    displayName: Snaps (Ubuntu Core)
+    dependsOn: Token
+    condition: succeeded('Token')
+
+    pool:
+      name: $(pool.custom.name)
+      demands: ubucore-e2e-tests
+
+    variables:
+      os: linux
+      arch: arm64v8
+      artifactName: iotedged-snap-aarch64
+      identityServiceArtifactName: packages_snap_aarch64
+      identityServicePackageFilter: azure-iot-identity_*_arm64.snap
+      sas_uri: $[ dependencies.Token.outputs['generate.sas_uri'] ]
+
+    timeoutInMinutes: 90
+
+    steps:
+    - script: |
+        sudo snap install docker
+      displayName: Install Docker as a snap
+    - template: templates/e2e-clean-directory.yaml
+    - template: templates/e2e-setup.yaml
+    - template: templates/e2e-clear-docker-cached-images.yaml
+    - template: templates/e2e-run.yaml
+      parameters:
+        continue_on_error: true
+        sas_uri: $(sas_uri)
+
   ################################################################################
   - job: redhat8_amd64
   ################################################################################

builds/e2e/templates/e2e-run.yaml

@@ -1,4 +1,5 @@
 parameters:
+  continue_on_error: false
   EventHubCompatibleEndpoint: '$(TestEventHubCompatibleEndpoint)'
   IotHubConnectionString: '$(TestIotHubConnectionString)'
   test_type: ''
@@ -67,6 +68,7 @@ steps:
     sudo --preserve-env dotnet test $testFile --no-build --logger 'trx' --filter "$filter"
 
   displayName: Run tests ${{ parameters.test_type }}
+  continueOnError: ${{ parameters.continue_on_error }}
   env:
     E2E_DPS_GROUP_KEY: $(TestDpsGroupKeySymmetric)
     E2E_EVENT_HUB_ENDPOINT: ${{ parameters['EventHubCompatibleEndpoint'] }}

test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs

@@ -122,6 +122,7 @@ IEnumerable<Registry> GetAndValidateRegistries()
             this.DeviceId = Option.Maybe(Get("deviceId"));
             this.ISA95Tag = context.GetValue("isa95Tag", false);
             this.GetSupportBundle = context.GetValue("getSupportBundle", false);
+            this.EnableSdkLoggingForLeafDevice = context.GetValue("enableSdkLoggingForLeafDevice", false);
         }
 
         static readonly Lazy<Context> Default = new Lazy<Context>(() => new Context());
@@ -211,5 +212,7 @@ IEnumerable<Registry> GetAndValidateRegistries()
         public bool ISA95Tag { get; }
 
         public bool GetSupportBundle { get; }
+
+        public bool EnableSdkLoggingForLeafDevice { get; }
     }
 }

test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs

@@ -4,6 +4,7 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common
     using System;
     using System.Collections.Generic;
     using System.ComponentModel;
+    using System.Diagnostics.Tracing;
     using System.Globalization;
     using System.IO;
     using System.Linq;
@@ -14,25 +15,29 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common
     using System.Threading;
     using System.Threading.Tasks;
     using Microsoft.Azure.Devices.Client;
+    using Microsoft.Azure.Devices.Client.Exceptions;
     using Microsoft.Azure.Devices.Edge.Test.Common.Certs;
     using Microsoft.Azure.Devices.Edge.Test.Common.Config;
     using Microsoft.Azure.Devices.Edge.Util;
     using Microsoft.Azure.Devices.Edge.Util.TransientFaultHandling;
+    using Microsoft.Extensions.Logging;
     using Serilog;
 
-    public class LeafDevice
+    public class LeafDevice : IDisposable
     {
-        readonly DeviceClient client;
         readonly Device device;
         readonly IotHub iotHub;
         readonly string messageId;
+        DeviceClient client;
+        Option<LeafDeviceSdkLogger> sdkLogger;
 
-        LeafDevice(Device device, DeviceClient client, IotHub iotHub)
+        LeafDevice(Device device, DeviceClient client, IotHub iotHub, Option<LeafDeviceSdkLogger> sdkLogger)
         {
             this.client = client;
             this.device = device;
             this.iotHub = iotHub;
             this.messageId = Guid.NewGuid().ToString();
+            this.sdkLogger = sdkLogger;
         }
 
         public static Task<LeafDevice> CreateAsync(
@@ -316,19 +321,86 @@ static async Task<LeafDevice> DeleteIdentityIfFailedAsync(Device device, IotHub
 
         static async Task<LeafDevice> CreateLeafDeviceAsync(Device device, Func<DeviceClient> clientFactory, IotHub iotHub, CancellationToken token)
         {
-            DeviceClient client = clientFactory();
+            DeviceClient client;
+            Option<LeafDeviceSdkLogger> logger = Option.None<LeafDeviceSdkLogger>();
+            ConnectionStatus status = ConnectionStatus.Disconnected;
+            ConnectionStatusChangeReason reason = ConnectionStatusChangeReason.Connection_Ok;
 
-            client.SetConnectionStatusChangesHandler((status, reason) =>
+            while (true)
             {
-                Log.Verbose($"Detected change in connection status:{Environment.NewLine}Changed Status: {status} Reason: {reason}");
-            });
+                client = clientFactory();
+                logger = Option.Maybe(Context.Current.EnableSdkLoggingForLeafDevice
+                    ? new LeafDeviceSdkLogger(new string[]
+                    {
+                        "DotNetty-Default",
+                        "Microsoft-Azure-Devices",
+                        "Azure-Core", "Azure-Identity"
+                    })
+                    : null);
+
+                client.SetConnectionStatusChangesHandler((s, r) =>
+                {
+                    status = s;
+                    reason = r;
+                    Log.Verbose($"Detected change in connection status:{Environment.NewLine}Changed Status: {status} Reason: {reason}");
+                });
+
+                using var innerCts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+                using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(innerCts.Token, token);
+                try
+                {
+                    await client.SetMethodHandlerAsync(nameof(DirectMethod), DirectMethod, null, linkedCts.Token);
+                    break;
+                }
+                catch (OperationCanceledException)
+                {
+                    await client.CloseAsync();
+                    client.Dispose();
+                    logger.ForEach(l => l.Dispose());
+
+                    // Only throw if the caller-supplied token was cancelled. If the inner (30 second) token was
+                    // cancelled, fall through and allow the device client to retry.
+                    if (token.IsCancellationRequested)
+                    {
+                        token.ThrowIfCancellationRequested();
+                    }
+                }
+                catch (IotHubCommunicationException)
+                {
+                    await client.CloseAsync();
+                    client.Dispose();
+                    logger.ForEach(l => l.Dispose());
 
-            await client.SetMethodHandlerAsync(nameof(DirectMethod), DirectMethod, null, token);
+                    // In the {status == Disconnected, reason == Retry_Expired } scenario, fall through and allow the
+                    // client to retry, otherwise throw.
+                    if (status != ConnectionStatus.Disconnected || reason != ConnectionStatusChangeReason.Retry_Expired)
+                    {
+                        throw;
+                    }
+                }
+            }
 
-            return new LeafDevice(device, client, iotHub);
+            return new LeafDevice(device, client, iotHub, logger);
         }
 
-        public Task Close() => this.client.CloseAsync();
+        public Task CloseAsync() => this.client.CloseAsync();
+
+        public void Dispose()
+        {
+            if (this.client != null)
+            {
+                this.client.Dispose();
+                this.client = null;
+            }
+
+            this.sdkLogger.ForEach(l => l.Dispose());
+            this.sdkLogger = Option.None<LeafDeviceSdkLogger>();
+        }
+
+        ~LeafDevice()
+        {
+            this.Dispose();
+        }
 
         public Task SendEventAsync(CancellationToken token)
         {

test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs

@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft. All rights reserved.
+namespace Microsoft.Azure.Devices.Edge.Test.Common
+{
+    using System;
+    using System.Diagnostics.Tracing;
+    using System.Globalization;
+    using System.Linq;
+    using Microsoft.Extensions.Logging;
+    using Serilog;
+
+    /// <summary>
+    /// Prints SDK events to Console output - the log level is set to INFORMATION
+    /// </summary>
+    public sealed class LeafDeviceSdkLogger : EventListener
+    {
+        private readonly string[] eventFilters;
+        private readonly object @lock = new object();
+
+        public LeafDeviceSdkLogger(string filter)
+            : this(new string[] { filter })
+        {
+        }
+
+        public LeafDeviceSdkLogger(string[] filters)
+        {
+            this.eventFilters = filters ?? throw new ArgumentNullException(nameof(filters));
+            if (this.eventFilters.Length == 0)
+            {
+                throw new ArgumentException("Filters cannot be empty", nameof(filters));
+            }
+
+            foreach (string filter in this.eventFilters)
+            {
+                if (string.IsNullOrWhiteSpace(filter))
+                {
+                    throw new ArgumentNullException(nameof(filters));
+                }
+            }
+
+            foreach (EventSource source in EventSource.GetSources())
+            {
+                this.EnableEvents(source, EventLevel.LogAlways);
+            }
+        }
+
+        protected override void OnEventSourceCreated(EventSource eventSource)
+        {
+            base.OnEventSourceCreated(eventSource);
+            this.EnableEvents(eventSource, EventLevel.LogAlways, EventKeywords.All);
+        }
+
+        protected override void OnEventWritten(EventWrittenEventArgs eventData)
+        {
+            if (this.eventFilters == null)
+            {
+                return;
+            }
+
+            lock (this.@lock)
+            {
+                if (this.eventFilters.Any(ef => eventData.EventSource.Name.StartsWith(ef, StringComparison.Ordinal)))
+                {
+                    string text = $"[{eventData.EventSource.Name}:{eventData.EventName}]{(eventData.Payload != null ? $" ({string.Join(", ", eventData.Payload)})." : string.Empty)}";
+                    Log.Verbose(text);
+                }
+            }
+        }
+    }
+}

test/Microsoft.Azure.Devices.Edge.Test.Common/linux/EdgeDaemon.cs

@@ -62,6 +62,15 @@ void ThrowUnsupportedOs() =>
                         ? SupportedPackageExtension.Snap
                         : SupportedPackageExtension.Deb;
                     break;
+                case "ubuntu-core":
+                    if (!detectedSnap)
+                    {
+                        throw new ArgumentException(
+                            "packagesPath parameter is required on Ubuntu Core, and it must point to snap packages");
+                    }
+
+                    packageExtension = SupportedPackageExtension.Snap;
+                    break;
                 case "debian":
                     if (version != "11" && version != "12")
                     {

test/Microsoft.Azure.Devices.Edge.Test.Common/linux/OsPlatform.cs

@@ -15,13 +15,12 @@ public class OsPlatform : Common.OsPlatform, IOsPlatform
     {
         public async Task<string> CollectDaemonLogsAsync(DateTime testStartTime, string filePrefix, CancellationToken token)
         {
+            // TODO: Support snaps AND non-snap services
             string args = string.Join(
                 " ",
-                "-u aziot-keyd",
-                "-u aziot-certd",
-                "-u aziot-identityd",
-                "-u aziot-edged",
-                "-u docker",
+                "-u snap.azure-iot-*",
+                "-u snap.docker.dockerd",
+                "-u snapd",
                 $"--since \"{testStartTime:yyyy-MM-dd HH:mm:ss}\"",
                 "--no-pager");
             string[] output = await Process.RunAsync("journalctl", args, token, logCommand: true, logOutput: false);

test/Microsoft.Azure.Devices.Edge.Test/Device.cs

@@ -25,7 +25,7 @@ public async Task QuickstartCerts()
 
             string leafDeviceId = DeviceId.Current.Generate();
 
-            var leaf = await LeafDevice.CreateAsync(
+            using var leaf = await LeafDevice.CreateAsync(
                 leafDeviceId,
                 Protocol.Amqp,
                 AuthenticationType.Sas,
@@ -49,6 +49,7 @@ await TryFinally.DoAsync(
                 },
                 async () =>
                 {
+                    await leaf.CloseAsync();
                     await leaf.DeleteIdentityAsync(token);
                 });
         }
@@ -65,7 +66,7 @@ public async Task QuickstartChangeSasKey()
             string leafDeviceId = DeviceId.Current.Generate();
 
             // Create leaf and send message
-            var leaf = await LeafDevice.CreateAsync(
+            using var leaf = await LeafDevice.CreateAsync(
                 leafDeviceId,
                 Protocol.Amqp,
                 AuthenticationType.Sas,
@@ -89,13 +90,13 @@ await TryFinally.DoAsync(
                 },
                 async () =>
                 {
-                    await leaf.Close();
+                    await leaf.CloseAsync();
                     await leaf.DeleteIdentityAsync(token);
                 });
 
             // Re-create the leaf with the same device ID, for our purposes this is
             // the equivalent of updating the SAS keys
-            var leafUpdated = await LeafDevice.CreateAsync(
+            using var leafUpdated = await LeafDevice.CreateAsync(
                 leafDeviceId,
                 Protocol.Amqp,
                 AuthenticationType.Sas,
@@ -119,7 +120,7 @@ await TryFinally.DoAsync(
                 },
                 async () =>
                 {
-                    await leafUpdated.Close();
+                    await leafUpdated.CloseAsync();
                     await leafUpdated.DeleteIdentityAsync(token);
                 });
         }
@@ -141,7 +142,7 @@ public async Task RouteMessageL3LeafToL4Module()
             string relayerModuleId = "relayer1";
 
             // Create leaf and send message
-            var leaf = await LeafDevice.CreateAsync(
+            using var leaf = await LeafDevice.CreateAsync(
                 leafDeviceId,
                 Protocol.Amqp,
                 AuthenticationType.Sas,
@@ -186,7 +187,7 @@ await Profiler.Run(
                 },
                 async () =>
                 {
-                    await leaf.Close();
+                    await leaf.CloseAsync();
                     await leaf.DeleteIdentityAsync(token);
                 });
         }
@@ -212,7 +213,7 @@ public async Task DisableReenableParentEdge()
 
             // Try connecting
             string leafDeviceId = DeviceId.Current.Generate();
-            var leaf = await LeafDevice.CreateAsync(
+            using var leaf = await LeafDevice.CreateAsync(
                 leafDeviceId,
                 Protocol.Amqp,
                 AuthenticationType.Sas,
@@ -236,6 +237,7 @@ await TryFinally.DoAsync(
                 },
                 async () =>
                 {
+                    await leaf.CloseAsync();
                     await leaf.DeleteIdentityAsync(token);
                 });
         }