diff --git a/builds/e2e/e2e.yaml b/builds/e2e/e2e.yaml index 09518e1409b..fdfbe3668ca 100644 --- a/builds/e2e/e2e.yaml +++ b/builds/e2e/e2e.yaml @@ -384,6 +384,39 @@ jobs: parameters: sas_uri: $(sas_uri) +################################################################################ + - job: snaps_ubuntu_core +################################################################################ + displayName: Snaps (Ubuntu Core) + dependsOn: Token + condition: succeeded('Token') + + pool: + name: $(pool.custom.name) + demands: ubucore-e2e-tests + + variables: + os: linux + arch: arm64v8 + artifactName: iotedged-snap-aarch64 + identityServiceArtifactName: packages_snap_aarch64 + identityServicePackageFilter: azure-iot-identity_*_arm64.snap + sas_uri: $[ dependencies.Token.outputs['generate.sas_uri'] ] + + timeoutInMinutes: 90 + + steps: + - script: | + sudo snap install docker + displayName: Install Docker as a snap + - template: templates/e2e-clean-directory.yaml + - template: templates/e2e-setup.yaml + - template: templates/e2e-clear-docker-cached-images.yaml + - template: templates/e2e-run.yaml + parameters: + continue_on_error: true + sas_uri: $(sas_uri) + ################################################################################ - job: redhat8_amd64 ################################################################################ diff --git a/builds/e2e/templates/e2e-run.yaml b/builds/e2e/templates/e2e-run.yaml index 1d46257b7cf..464657750c6 100644 --- a/builds/e2e/templates/e2e-run.yaml +++ b/builds/e2e/templates/e2e-run.yaml @@ -1,4 +1,5 @@ parameters: + continue_on_error: false EventHubCompatibleEndpoint: '$(TestEventHubCompatibleEndpoint)' IotHubConnectionString: '$(TestIotHubConnectionString)' test_type: '' @@ -67,6 +68,7 @@ steps: sudo --preserve-env dotnet test $testFile --no-build --logger 'trx' --filter "$filter" displayName: Run tests ${{ parameters.test_type }} + continueOnError: ${{ parameters.continue_on_error }} env: E2E_DPS_GROUP_KEY: $(TestDpsGroupKeySymmetric) E2E_EVENT_HUB_ENDPOINT: ${{ parameters['EventHubCompatibleEndpoint'] }} diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs index 43b3a9da7cf..4cda40ddf63 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs @@ -122,6 +122,7 @@ IEnumerable GetAndValidateRegistries() this.DeviceId = Option.Maybe(Get("deviceId")); this.ISA95Tag = context.GetValue("isa95Tag", false); this.GetSupportBundle = context.GetValue("getSupportBundle", false); + this.EnableSdkLoggingForLeafDevice = context.GetValue("enableSdkLoggingForLeafDevice", false); } static readonly Lazy Default = new Lazy(() => new Context()); @@ -211,5 +212,7 @@ IEnumerable GetAndValidateRegistries() public bool ISA95Tag { get; } public bool GetSupportBundle { get; } + + public bool EnableSdkLoggingForLeafDevice { get; } } } diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs index bd6af625d19..f09034c0f7f 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs @@ -4,6 +4,7 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common using System; using System.Collections.Generic; using System.ComponentModel; + using System.Diagnostics.Tracing; using System.Globalization; using System.IO; using System.Linq; @@ -14,25 +15,29 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Devices.Client; + using Microsoft.Azure.Devices.Client.Exceptions; using Microsoft.Azure.Devices.Edge.Test.Common.Certs; using Microsoft.Azure.Devices.Edge.Test.Common.Config; using Microsoft.Azure.Devices.Edge.Util; using Microsoft.Azure.Devices.Edge.Util.TransientFaultHandling; + using Microsoft.Extensions.Logging; using Serilog; - public class LeafDevice + public class LeafDevice : IDisposable { - readonly DeviceClient client; readonly Device device; readonly IotHub iotHub; readonly string messageId; + DeviceClient client; + Option sdkLogger; - LeafDevice(Device device, DeviceClient client, IotHub iotHub) + LeafDevice(Device device, DeviceClient client, IotHub iotHub, Option sdkLogger) { this.client = client; this.device = device; this.iotHub = iotHub; this.messageId = Guid.NewGuid().ToString(); + this.sdkLogger = sdkLogger; } public static Task CreateAsync( @@ -316,19 +321,86 @@ static async Task DeleteIdentityIfFailedAsync(Device device, IotHub static async Task CreateLeafDeviceAsync(Device device, Func clientFactory, IotHub iotHub, CancellationToken token) { - DeviceClient client = clientFactory(); + DeviceClient client; + Option logger = Option.None(); + ConnectionStatus status = ConnectionStatus.Disconnected; + ConnectionStatusChangeReason reason = ConnectionStatusChangeReason.Connection_Ok; - client.SetConnectionStatusChangesHandler((status, reason) => + while (true) { - Log.Verbose($"Detected change in connection status:{Environment.NewLine}Changed Status: {status} Reason: {reason}"); - }); + client = clientFactory(); + logger = Option.Maybe(Context.Current.EnableSdkLoggingForLeafDevice + ? new LeafDeviceSdkLogger(new string[] + { + "DotNetty-Default", + "Microsoft-Azure-Devices", + "Azure-Core", "Azure-Identity" + }) + : null); + + client.SetConnectionStatusChangesHandler((s, r) => + { + status = s; + reason = r; + Log.Verbose($"Detected change in connection status:{Environment.NewLine}Changed Status: {status} Reason: {reason}"); + }); + + using var innerCts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(innerCts.Token, token); + try + { + await client.SetMethodHandlerAsync(nameof(DirectMethod), DirectMethod, null, linkedCts.Token); + break; + } + catch (OperationCanceledException) + { + await client.CloseAsync(); + client.Dispose(); + logger.ForEach(l => l.Dispose()); + + // Only throw if the caller-supplied token was cancelled. If the inner (30 second) token was + // cancelled, fall through and allow the device client to retry. + if (token.IsCancellationRequested) + { + token.ThrowIfCancellationRequested(); + } + } + catch (IotHubCommunicationException) + { + await client.CloseAsync(); + client.Dispose(); + logger.ForEach(l => l.Dispose()); - await client.SetMethodHandlerAsync(nameof(DirectMethod), DirectMethod, null, token); + // In the {status == Disconnected, reason == Retry_Expired } scenario, fall through and allow the + // client to retry, otherwise throw. + if (status != ConnectionStatus.Disconnected || reason != ConnectionStatusChangeReason.Retry_Expired) + { + throw; + } + } + } - return new LeafDevice(device, client, iotHub); + return new LeafDevice(device, client, iotHub, logger); } - public Task Close() => this.client.CloseAsync(); + public Task CloseAsync() => this.client.CloseAsync(); + + public void Dispose() + { + if (this.client != null) + { + this.client.Dispose(); + this.client = null; + } + + this.sdkLogger.ForEach(l => l.Dispose()); + this.sdkLogger = Option.None(); + } + + ~LeafDevice() + { + this.Dispose(); + } public Task SendEventAsync(CancellationToken token) { diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs new file mode 100644 index 00000000000..42bddffbc2c --- /dev/null +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Test.Common +{ + using System; + using System.Diagnostics.Tracing; + using System.Globalization; + using System.Linq; + using Microsoft.Extensions.Logging; + using Serilog; + + /// + /// Prints SDK events to Console output - the log level is set to INFORMATION + /// + public sealed class LeafDeviceSdkLogger : EventListener + { + private readonly string[] eventFilters; + private readonly object @lock = new object(); + + public LeafDeviceSdkLogger(string filter) + : this(new string[] { filter }) + { + } + + public LeafDeviceSdkLogger(string[] filters) + { + this.eventFilters = filters ?? throw new ArgumentNullException(nameof(filters)); + if (this.eventFilters.Length == 0) + { + throw new ArgumentException("Filters cannot be empty", nameof(filters)); + } + + foreach (string filter in this.eventFilters) + { + if (string.IsNullOrWhiteSpace(filter)) + { + throw new ArgumentNullException(nameof(filters)); + } + } + + foreach (EventSource source in EventSource.GetSources()) + { + this.EnableEvents(source, EventLevel.LogAlways); + } + } + + protected override void OnEventSourceCreated(EventSource eventSource) + { + base.OnEventSourceCreated(eventSource); + this.EnableEvents(eventSource, EventLevel.LogAlways, EventKeywords.All); + } + + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + if (this.eventFilters == null) + { + return; + } + + lock (this.@lock) + { + if (this.eventFilters.Any(ef => eventData.EventSource.Name.StartsWith(ef, StringComparison.Ordinal))) + { + string text = $"[{eventData.EventSource.Name}:{eventData.EventName}]{(eventData.Payload != null ? $" ({string.Join(", ", eventData.Payload)})." : string.Empty)}"; + Log.Verbose(text); + } + } + } + } +} diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/EdgeDaemon.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/EdgeDaemon.cs index a67606e5332..e9e4cee5dc0 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/EdgeDaemon.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/EdgeDaemon.cs @@ -62,6 +62,15 @@ void ThrowUnsupportedOs() => ? SupportedPackageExtension.Snap : SupportedPackageExtension.Deb; break; + case "ubuntu-core": + if (!detectedSnap) + { + throw new ArgumentException( + "packagesPath parameter is required on Ubuntu Core, and it must point to snap packages"); + } + + packageExtension = SupportedPackageExtension.Snap; + break; case "debian": if (version != "11" && version != "12") { diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/OsPlatform.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/OsPlatform.cs index 4e71976eccc..ef57b2e24d7 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/OsPlatform.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/OsPlatform.cs @@ -15,13 +15,12 @@ public class OsPlatform : Common.OsPlatform, IOsPlatform { public async Task CollectDaemonLogsAsync(DateTime testStartTime, string filePrefix, CancellationToken token) { + // TODO: Support snaps AND non-snap services string args = string.Join( " ", - "-u aziot-keyd", - "-u aziot-certd", - "-u aziot-identityd", - "-u aziot-edged", - "-u docker", + "-u snap.azure-iot-*", + "-u snap.docker.dockerd", + "-u snapd", $"--since \"{testStartTime:yyyy-MM-dd HH:mm:ss}\"", "--no-pager"); string[] output = await Process.RunAsync("journalctl", args, token, logCommand: true, logOutput: false); diff --git a/test/Microsoft.Azure.Devices.Edge.Test/Device.cs b/test/Microsoft.Azure.Devices.Edge.Test/Device.cs index 87fee9a0bbe..0898ec085e1 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/Device.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/Device.cs @@ -25,7 +25,7 @@ public async Task QuickstartCerts() string leafDeviceId = DeviceId.Current.Generate(); - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -49,6 +49,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } @@ -65,7 +66,7 @@ public async Task QuickstartChangeSasKey() string leafDeviceId = DeviceId.Current.Generate(); // Create leaf and send message - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -89,13 +90,13 @@ await TryFinally.DoAsync( }, async () => { - await leaf.Close(); + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); // Re-create the leaf with the same device ID, for our purposes this is // the equivalent of updating the SAS keys - var leafUpdated = await LeafDevice.CreateAsync( + using var leafUpdated = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -119,7 +120,7 @@ await TryFinally.DoAsync( }, async () => { - await leafUpdated.Close(); + await leafUpdated.CloseAsync(); await leafUpdated.DeleteIdentityAsync(token); }); } @@ -141,7 +142,7 @@ public async Task RouteMessageL3LeafToL4Module() string relayerModuleId = "relayer1"; // Create leaf and send message - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -186,7 +187,7 @@ await Profiler.Run( }, async () => { - await leaf.Close(); + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } @@ -212,7 +213,7 @@ public async Task DisableReenableParentEdge() // Try connecting string leafDeviceId = DeviceId.Current.Generate(); - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -236,6 +237,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } diff --git a/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs b/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs index 5f311963e54..4781888f1e4 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs @@ -29,34 +29,24 @@ public async Task TransparentGateway( ? Option.None() : Option.Some(this.runtime.DeviceId); - LeafDevice leaf = null; - try - { - leaf = await LeafDevice.CreateAsync( - leafDeviceId, - protocol, - testAuth.ToAuthenticationType(), - parentId, - testAuth.UseSecondaryCertificate(), - this.ca, - this.daemon.GetCertificatesPath(), - this.IotHub, - this.device.NestedEdge.DeviceHostname, - token, - Option.None(), - this.device.NestedEdge.IsNestedEdge); - } - catch (Exception) when (!parentId.HasValue) - { - return; - } - if (!parentId.HasValue) { Assert.Fail("Expected to fail when not in scope."); } - Assert.NotNull(leaf); + using var leaf = await LeafDevice.CreateAsync( + leafDeviceId, + protocol, + testAuth.ToAuthenticationType(), + parentId, + testAuth.UseSecondaryCertificate(), + this.ca, + this.daemon.GetCertificatesPath(), + this.IotHub, + this.device.NestedEdge.DeviceHostname, + token, + Option.None(), + this.device.NestedEdge.IsNestedEdge); await TryFinally.DoAsync( async () => @@ -68,6 +58,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } @@ -89,6 +80,10 @@ public async Task GrandparentScopeDevice( } Option parentId = Option.Some(this.runtime.DeviceId); + if (!parentId.HasValue) + { + Assert.Fail("Expected to fail when not in scope."); + } CancellationToken token = this.TestToken; @@ -96,34 +91,19 @@ public async Task GrandparentScopeDevice( string leafDeviceId = DeviceId.Current.Generate(); - LeafDevice leaf = null; - try - { - leaf = await LeafDevice.CreateAsync( - leafDeviceId, - protocol, - testAuth.ToAuthenticationType(), - parentId, - testAuth.UseSecondaryCertificate(), - this.ca, - this.daemon.GetCertificatesPath(), - this.IotHub, - this.device.NestedEdge.ParentHostname, - token, - Option.None(), - this.device.NestedEdge.IsNestedEdge); - } - catch (Exception) when (!parentId.HasValue) - { - return; - } - - if (!parentId.HasValue) - { - Assert.Fail("Expected to fail when not in scope."); - } - - Assert.NotNull(leaf); + using var leaf = await LeafDevice.CreateAsync( + leafDeviceId, + protocol, + testAuth.ToAuthenticationType(), + parentId, + testAuth.UseSecondaryCertificate(), + this.ca, + this.daemon.GetCertificatesPath(), + this.IotHub, + this.device.NestedEdge.ParentHostname, + token, + Option.None(), + this.device.NestedEdge.IsNestedEdge); await TryFinally.DoAsync( async () => @@ -135,8 +115,8 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); - await Task.CompletedTask; }); } } diff --git a/test/Microsoft.Azure.Devices.Edge.Test/PlugAndPlay.cs b/test/Microsoft.Azure.Devices.Edge.Test/PlugAndPlay.cs index 45eaf14d4f3..ced20ae5734 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/PlugAndPlay.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/PlugAndPlay.cs @@ -34,7 +34,7 @@ public async Task PlugAndPlayDeviceClient(Protocol protocol) token, Context.Current.NestedEdge); - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, protocol, AuthenticationType.Sas, @@ -58,6 +58,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } diff --git a/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs b/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs index 767c5cb999c..fa3a7386e2a 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs @@ -24,7 +24,7 @@ public async Task X509ManualProvision() string leafDeviceId = DeviceId.Current.Generate(); - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -48,6 +48,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } diff --git a/test/doc/e2e.md b/test/doc/e2e.md index 697626f22f2..1846c015aea 100644 --- a/test/doc/e2e.md +++ b/test/doc/e2e.md @@ -27,6 +27,7 @@ The end-to-end tests take several parameters, which they expect to find in a fil | `edgeAgentImage` || Docker image to pull/use for Edge Agent. If not given, the default value `mcr.microsoft.com/azureiotedge-agent:1.5` is used. This setting only applies to any configurations deployed by the tests. Note also that the default value is ALWAYS used in config.yaml to start IoT Edge; this setting only applies to any configurations deployed by the tests. | | `edgeHubImage` || Docker image to pull/use for Edge Hub. If not given, `mcr.microsoft.com/azureiotedge-hub:1.5` is used. | | `edgeHubSchemaVersion` || The schema version used for EdgeHub. | +| `enableSdkLoggingForLeafDevice` || Enable logging in the Device SDK when it is used to simulate a leaf device (used by tests: QuickstartCerts, QuickstartChangeSasKey, RouteMessageL3LeafToL4Module, DisableReenableParentEdge, TransparentGateway, GrandparentScopeDevice, PlugAndPlayDeviceClient, X509ManualProvision). This setting is useful for debugging issues in these tests. | | `iotHubResourceId` || Full resource ID (`/resource/subscriptions//resourceGroups//providers/Microsoft.Devices/IotHubs/`) of the IoT hub that will receive metrics messages. Required when running the test 'MetricsCollector', ignored otherwise. | | `loadGenImage` | * | LoadGen image to be used. Required when running PriorityQueue tests, ignored otherwise.| | `logFile` || Path to which all test output will be written, including verbose output. This setting allows the user to capture all the details of a test pass while keeping the shell window output free of visual clutter. Note that daemon logs and module logs are always written to the same directory as the test binaries (e.g., `test/Microsoft.Azure.Devices.Edge.Test/bin/Debug/netcoreapp2.1/*.log`), independent of this parameter. |