From 8b93a1f894051bc5a154ea1439e87daba2e12fe8 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Wed, 4 Sep 2024 17:42:28 +0000 Subject: [PATCH 01/10] Support Ubuntu Core in the end-to-end tests --- .../linux/EdgeDaemon.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/EdgeDaemon.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/EdgeDaemon.cs index a67606e5332..e9e4cee5dc0 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/EdgeDaemon.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/EdgeDaemon.cs @@ -62,6 +62,15 @@ void ThrowUnsupportedOs() => ? SupportedPackageExtension.Snap : SupportedPackageExtension.Deb; break; + case "ubuntu-core": + if (!detectedSnap) + { + throw new ArgumentException( + "packagesPath parameter is required on Ubuntu Core, and it must point to snap packages"); + } + + packageExtension = SupportedPackageExtension.Snap; + break; case "debian": if (version != "11" && version != "12") { From 608ce2fa63ce633d76e613d0306ec59c1f1f52a0 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Wed, 4 Sep 2024 17:44:52 +0000 Subject: [PATCH 02/10] Add retries and better error handling when creating a leaf device --- .../LeafDevice.cs | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs index bd6af625d19..0a8ae893bca 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs @@ -14,6 +14,7 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Devices.Client; + using Microsoft.Azure.Devices.Common.Exceptions; using Microsoft.Azure.Devices.Edge.Test.Common.Certs; using Microsoft.Azure.Devices.Edge.Test.Common.Config; using Microsoft.Azure.Devices.Edge.Util; @@ -316,14 +317,51 @@ static async Task DeleteIdentityIfFailedAsync(Device device, IotHub static async Task CreateLeafDeviceAsync(Device device, Func clientFactory, IotHub iotHub, CancellationToken token) { - DeviceClient client = clientFactory(); + DeviceClient client; + ConnectionStatus status = ConnectionStatus.Disconnected; + ConnectionStatusChangeReason reason = ConnectionStatusChangeReason.Connection_Ok; - client.SetConnectionStatusChangesHandler((status, reason) => + while (true) { - Log.Verbose($"Detected change in connection status:{Environment.NewLine}Changed Status: {status} Reason: {reason}"); - }); + client = clientFactory(); - await client.SetMethodHandlerAsync(nameof(DirectMethod), DirectMethod, null, token); + client.SetConnectionStatusChangesHandler((s, r) => + { + status = s; + reason = r; + Log.Verbose($"Detected change in connection status:{Environment.NewLine}Changed Status: {status} Reason: {reason}"); + }); + + using var innerCts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(innerCts.Token, token); + try + { + await client.SetMethodHandlerAsync(nameof(DirectMethod), DirectMethod, null, linkedCts.Token); + break; + } + catch (OperationCanceledException e) + { + Log.Information($"1 ### Caught exception: {e.GetType()}"); + await client.CloseAsync(); + client.Dispose(); + + if (token.IsCancellationRequested) + { + token.ThrowIfCancellationRequested(); + } + } + catch (Exception e) + { + Log.Information($"2 ### Caught exception: {e.GetType()}"); + await client.CloseAsync(); + client.Dispose(); + + if (status != ConnectionStatus.Disconnected || reason != ConnectionStatusChangeReason.Retry_Expired) + { + throw; + } + } + } return new LeafDevice(device, client, iotHub); } From 06c7983a7ab5f700f9052452d3f152937670f822 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Wed, 4 Sep 2024 17:46:14 +0000 Subject: [PATCH 03/10] Collect daemon logs for snaps --- .../linux/OsPlatform.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/OsPlatform.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/OsPlatform.cs index 4e71976eccc..ef57b2e24d7 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/OsPlatform.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/OsPlatform.cs @@ -15,13 +15,12 @@ public class OsPlatform : Common.OsPlatform, IOsPlatform { public async Task CollectDaemonLogsAsync(DateTime testStartTime, string filePrefix, CancellationToken token) { + // TODO: Support snaps AND non-snap services string args = string.Join( " ", - "-u aziot-keyd", - "-u aziot-certd", - "-u aziot-identityd", - "-u aziot-edged", - "-u docker", + "-u snap.azure-iot-*", + "-u snap.docker.dockerd", + "-u snapd", $"--since \"{testStartTime:yyyy-MM-dd HH:mm:ss}\"", "--no-pager"); string[] output = await Process.RunAsync("journalctl", args, token, logCommand: true, logOutput: false); From deda319541766e84d9a818c380eb052a51eb3905 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Wed, 4 Sep 2024 20:18:02 +0000 Subject: [PATCH 04/10] Target IotHubCommunicationException directly --- .../LeafDevice.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs index 0a8ae893bca..60aa8508a5a 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs @@ -14,7 +14,7 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Devices.Client; - using Microsoft.Azure.Devices.Common.Exceptions; + using Microsoft.Azure.Devices.Client.Exceptions; using Microsoft.Azure.Devices.Edge.Test.Common.Certs; using Microsoft.Azure.Devices.Edge.Test.Common.Config; using Microsoft.Azure.Devices.Edge.Util; @@ -341,21 +341,23 @@ static async Task CreateLeafDeviceAsync(Device device, Func Date: Wed, 4 Sep 2024 20:19:58 +0000 Subject: [PATCH 05/10] Remove unused variables --- test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs index 60aa8508a5a..7325919fd45 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs @@ -339,7 +339,7 @@ static async Task CreateLeafDeviceAsync(Device device, Func CreateLeafDeviceAsync(Device device, Func Date: Thu, 5 Sep 2024 22:41:53 +0000 Subject: [PATCH 06/10] Log SDK events in LeafDevice --- .../LeafDevice.cs | 45 ++++++++-- .../LeafDeviceSdkLogger.cs | 71 ++++++++++++++++ .../Device.cs | 18 ++-- .../DeviceWithCustomCertificates.cs | 84 +++++++------------ .../PlugAndPlay.cs | 3 +- .../X509Device.cs | 3 +- 6 files changed, 157 insertions(+), 67 deletions(-) create mode 100644 test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs index 7325919fd45..5b7a31ba168 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs @@ -4,6 +4,7 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common using System; using System.Collections.Generic; using System.ComponentModel; + using System.Diagnostics.Tracing; using System.Globalization; using System.IO; using System.Linq; @@ -19,21 +20,24 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common using Microsoft.Azure.Devices.Edge.Test.Common.Config; using Microsoft.Azure.Devices.Edge.Util; using Microsoft.Azure.Devices.Edge.Util.TransientFaultHandling; + using Microsoft.Extensions.Logging; using Serilog; - public class LeafDevice + public class LeafDevice : IDisposable { - readonly DeviceClient client; readonly Device device; readonly IotHub iotHub; readonly string messageId; + DeviceClient client; + LeafDeviceSdkLogger sdkLogger; - LeafDevice(Device device, DeviceClient client, IotHub iotHub) + LeafDevice(Device device, DeviceClient client, IotHub iotHub, LeafDeviceSdkLogger sdkLogger) { this.client = client; this.device = device; this.iotHub = iotHub; this.messageId = Guid.NewGuid().ToString(); + this.sdkLogger = sdkLogger; } public static Task CreateAsync( @@ -317,13 +321,22 @@ static async Task DeleteIdentityIfFailedAsync(Device device, IotHub static async Task CreateLeafDeviceAsync(Device device, Func clientFactory, IotHub iotHub, CancellationToken token) { + LeafDeviceSdkLogger CreateSdkLogger() + { + string[] eventFilter = new string[] { "DotNetty-Default", "Microsoft-Azure-Devices", "Azure-Core", "Azure-Identity" }; + var loggerFactory = new LoggerFactory().AddSerilog(Log.Logger); + return new LeafDeviceSdkLogger(eventFilter, loggerFactory.CreateLogger("LeafDevice")); + } + DeviceClient client; + LeafDeviceSdkLogger logger; ConnectionStatus status = ConnectionStatus.Disconnected; ConnectionStatusChangeReason reason = ConnectionStatusChangeReason.Connection_Ok; while (true) { client = clientFactory(); + logger = CreateSdkLogger(); client.SetConnectionStatusChangesHandler((s, r) => { @@ -343,6 +356,7 @@ static async Task CreateLeafDeviceAsync(Device device, Func CreateLeafDeviceAsync(Device device, Func CreateLeafDeviceAsync(Device device, Func this.client.CloseAsync(); + + public void Dispose() + { + if (this.client != null) + { + this.client.Dispose(); + this.client = null; + } + + if (this.sdkLogger != null) + { + this.sdkLogger.Dispose(); + this.sdkLogger = null; + } } - public Task Close() => this.client.CloseAsync(); + ~LeafDevice() + { + this.Dispose(); + } public Task SendEventAsync(CancellationToken token) { diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs new file mode 100644 index 00000000000..d6978bfa444 --- /dev/null +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Test.Common +{ + using System; + using System.Diagnostics.Tracing; + using System.Globalization; + using System.Linq; + using Microsoft.Extensions.Logging; + + /// + /// Prints SDK events to Console output - the log level is set to INFORMATION + /// + public sealed class LeafDeviceSdkLogger : EventListener + { + private readonly string[] eventFilters; + private readonly ILogger logger; + private readonly object @lock = new object(); + + public LeafDeviceSdkLogger(string filter, ILogger logger) + : this(new string[] { filter }, logger) + { + } + + public LeafDeviceSdkLogger(string[] filters, ILogger logger) + { + this.eventFilters = filters ?? throw new ArgumentNullException(nameof(filters)); + if (this.eventFilters.Length == 0) + { + throw new ArgumentException("Filters cannot be empty", nameof(filters)); + } + + foreach (string filter in this.eventFilters) + { + if (string.IsNullOrWhiteSpace(filter)) + { + throw new ArgumentNullException(nameof(filters)); + } + } + + this.logger = logger; + + foreach (EventSource source in EventSource.GetSources()) + { + this.EnableEvents(source, EventLevel.LogAlways); + } + } + + protected override void OnEventSourceCreated(EventSource eventSource) + { + base.OnEventSourceCreated(eventSource); + this.EnableEvents(eventSource, EventLevel.LogAlways, EventKeywords.All); + } + + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + if (this.eventFilters == null) + { + return; + } + + lock (this.@lock) + { + if (this.eventFilters.Any(ef => eventData.EventSource.Name.StartsWith(ef, StringComparison.Ordinal))) + { + string text = $"{DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture)} [SDK] [{eventData.EventSource.Name}-{eventData.EventName}]{(eventData.Payload != null ? $" ({string.Join(", ", eventData.Payload)})." : string.Empty)}"; + this.logger.LogInformation(text); + } + } + } + } +} diff --git a/test/Microsoft.Azure.Devices.Edge.Test/Device.cs b/test/Microsoft.Azure.Devices.Edge.Test/Device.cs index 87fee9a0bbe..0898ec085e1 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/Device.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/Device.cs @@ -25,7 +25,7 @@ public async Task QuickstartCerts() string leafDeviceId = DeviceId.Current.Generate(); - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -49,6 +49,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } @@ -65,7 +66,7 @@ public async Task QuickstartChangeSasKey() string leafDeviceId = DeviceId.Current.Generate(); // Create leaf and send message - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -89,13 +90,13 @@ await TryFinally.DoAsync( }, async () => { - await leaf.Close(); + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); // Re-create the leaf with the same device ID, for our purposes this is // the equivalent of updating the SAS keys - var leafUpdated = await LeafDevice.CreateAsync( + using var leafUpdated = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -119,7 +120,7 @@ await TryFinally.DoAsync( }, async () => { - await leafUpdated.Close(); + await leafUpdated.CloseAsync(); await leafUpdated.DeleteIdentityAsync(token); }); } @@ -141,7 +142,7 @@ public async Task RouteMessageL3LeafToL4Module() string relayerModuleId = "relayer1"; // Create leaf and send message - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -186,7 +187,7 @@ await Profiler.Run( }, async () => { - await leaf.Close(); + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } @@ -212,7 +213,7 @@ public async Task DisableReenableParentEdge() // Try connecting string leafDeviceId = DeviceId.Current.Generate(); - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -236,6 +237,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } diff --git a/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs b/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs index 5f311963e54..4781888f1e4 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs @@ -29,34 +29,24 @@ public async Task TransparentGateway( ? Option.None() : Option.Some(this.runtime.DeviceId); - LeafDevice leaf = null; - try - { - leaf = await LeafDevice.CreateAsync( - leafDeviceId, - protocol, - testAuth.ToAuthenticationType(), - parentId, - testAuth.UseSecondaryCertificate(), - this.ca, - this.daemon.GetCertificatesPath(), - this.IotHub, - this.device.NestedEdge.DeviceHostname, - token, - Option.None(), - this.device.NestedEdge.IsNestedEdge); - } - catch (Exception) when (!parentId.HasValue) - { - return; - } - if (!parentId.HasValue) { Assert.Fail("Expected to fail when not in scope."); } - Assert.NotNull(leaf); + using var leaf = await LeafDevice.CreateAsync( + leafDeviceId, + protocol, + testAuth.ToAuthenticationType(), + parentId, + testAuth.UseSecondaryCertificate(), + this.ca, + this.daemon.GetCertificatesPath(), + this.IotHub, + this.device.NestedEdge.DeviceHostname, + token, + Option.None(), + this.device.NestedEdge.IsNestedEdge); await TryFinally.DoAsync( async () => @@ -68,6 +58,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } @@ -89,6 +80,10 @@ public async Task GrandparentScopeDevice( } Option parentId = Option.Some(this.runtime.DeviceId); + if (!parentId.HasValue) + { + Assert.Fail("Expected to fail when not in scope."); + } CancellationToken token = this.TestToken; @@ -96,34 +91,19 @@ public async Task GrandparentScopeDevice( string leafDeviceId = DeviceId.Current.Generate(); - LeafDevice leaf = null; - try - { - leaf = await LeafDevice.CreateAsync( - leafDeviceId, - protocol, - testAuth.ToAuthenticationType(), - parentId, - testAuth.UseSecondaryCertificate(), - this.ca, - this.daemon.GetCertificatesPath(), - this.IotHub, - this.device.NestedEdge.ParentHostname, - token, - Option.None(), - this.device.NestedEdge.IsNestedEdge); - } - catch (Exception) when (!parentId.HasValue) - { - return; - } - - if (!parentId.HasValue) - { - Assert.Fail("Expected to fail when not in scope."); - } - - Assert.NotNull(leaf); + using var leaf = await LeafDevice.CreateAsync( + leafDeviceId, + protocol, + testAuth.ToAuthenticationType(), + parentId, + testAuth.UseSecondaryCertificate(), + this.ca, + this.daemon.GetCertificatesPath(), + this.IotHub, + this.device.NestedEdge.ParentHostname, + token, + Option.None(), + this.device.NestedEdge.IsNestedEdge); await TryFinally.DoAsync( async () => @@ -135,8 +115,8 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); - await Task.CompletedTask; }); } } diff --git a/test/Microsoft.Azure.Devices.Edge.Test/PlugAndPlay.cs b/test/Microsoft.Azure.Devices.Edge.Test/PlugAndPlay.cs index 45eaf14d4f3..ced20ae5734 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/PlugAndPlay.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/PlugAndPlay.cs @@ -34,7 +34,7 @@ public async Task PlugAndPlayDeviceClient(Protocol protocol) token, Context.Current.NestedEdge); - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, protocol, AuthenticationType.Sas, @@ -58,6 +58,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } diff --git a/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs b/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs index 767c5cb999c..fa3a7386e2a 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs @@ -24,7 +24,7 @@ public async Task X509ManualProvision() string leafDeviceId = DeviceId.Current.Generate(); - var leaf = await LeafDevice.CreateAsync( + using var leaf = await LeafDevice.CreateAsync( leafDeviceId, Protocol.Amqp, AuthenticationType.Sas, @@ -48,6 +48,7 @@ await TryFinally.DoAsync( }, async () => { + await leaf.CloseAsync(); await leaf.DeleteIdentityAsync(token); }); } From b1063e0d517487b29bcc67202765fd0bbaee95b6 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Fri, 6 Sep 2024 16:30:51 +0000 Subject: [PATCH 07/10] Use Serilog directly --- .../LeafDevice.cs | 9 +-------- .../LeafDeviceSdkLogger.cs | 12 +++++------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs index 5b7a31ba168..c6a875d467c 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs @@ -321,13 +321,6 @@ static async Task DeleteIdentityIfFailedAsync(Device device, IotHub static async Task CreateLeafDeviceAsync(Device device, Func clientFactory, IotHub iotHub, CancellationToken token) { - LeafDeviceSdkLogger CreateSdkLogger() - { - string[] eventFilter = new string[] { "DotNetty-Default", "Microsoft-Azure-Devices", "Azure-Core", "Azure-Identity" }; - var loggerFactory = new LoggerFactory().AddSerilog(Log.Logger); - return new LeafDeviceSdkLogger(eventFilter, loggerFactory.CreateLogger("LeafDevice")); - } - DeviceClient client; LeafDeviceSdkLogger logger; ConnectionStatus status = ConnectionStatus.Disconnected; @@ -336,7 +329,7 @@ LeafDeviceSdkLogger CreateSdkLogger() while (true) { client = clientFactory(); - logger = CreateSdkLogger(); + logger = new LeafDeviceSdkLogger(new string[] { "DotNetty-Default", "Microsoft-Azure-Devices", "Azure-Core", "Azure-Identity" }); client.SetConnectionStatusChangesHandler((s, r) => { diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs index d6978bfa444..f6dfc5daa57 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common using System.Globalization; using System.Linq; using Microsoft.Extensions.Logging; + using Serilog; /// /// Prints SDK events to Console output - the log level is set to INFORMATION @@ -13,15 +14,14 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common public sealed class LeafDeviceSdkLogger : EventListener { private readonly string[] eventFilters; - private readonly ILogger logger; private readonly object @lock = new object(); - public LeafDeviceSdkLogger(string filter, ILogger logger) - : this(new string[] { filter }, logger) + public LeafDeviceSdkLogger(string filter) + : this(new string[] { filter }) { } - public LeafDeviceSdkLogger(string[] filters, ILogger logger) + public LeafDeviceSdkLogger(string[] filters) { this.eventFilters = filters ?? throw new ArgumentNullException(nameof(filters)); if (this.eventFilters.Length == 0) @@ -37,8 +37,6 @@ public LeafDeviceSdkLogger(string[] filters, ILogger logger) } } - this.logger = logger; - foreach (EventSource source in EventSource.GetSources()) { this.EnableEvents(source, EventLevel.LogAlways); @@ -63,7 +61,7 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData) if (this.eventFilters.Any(ef => eventData.EventSource.Name.StartsWith(ef, StringComparison.Ordinal))) { string text = $"{DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture)} [SDK] [{eventData.EventSource.Name}-{eventData.EventName}]{(eventData.Payload != null ? $" ({string.Join(", ", eventData.Payload)})." : string.Empty)}"; - this.logger.LogInformation(text); + Log.Verbose(text); } } } From 53f5720dc0f9c58e4b4c9424326e5011d22e2edd Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Fri, 6 Sep 2024 16:32:17 +0000 Subject: [PATCH 08/10] Add flag to enable SDK logging for leaf device --- .../Context.cs | 3 +++ .../LeafDevice.cs | 26 +++++++++++-------- .../LeafDeviceSdkLogger.cs | 2 +- test/doc/e2e.md | 1 + 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs index 43b3a9da7cf..4cda40ddf63 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs @@ -122,6 +122,7 @@ IEnumerable GetAndValidateRegistries() this.DeviceId = Option.Maybe(Get("deviceId")); this.ISA95Tag = context.GetValue("isa95Tag", false); this.GetSupportBundle = context.GetValue("getSupportBundle", false); + this.EnableSdkLoggingForLeafDevice = context.GetValue("enableSdkLoggingForLeafDevice", false); } static readonly Lazy Default = new Lazy(() => new Context()); @@ -211,5 +212,7 @@ IEnumerable GetAndValidateRegistries() public bool ISA95Tag { get; } public bool GetSupportBundle { get; } + + public bool EnableSdkLoggingForLeafDevice { get; } } } diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs index c6a875d467c..f09034c0f7f 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDevice.cs @@ -29,9 +29,9 @@ public class LeafDevice : IDisposable readonly IotHub iotHub; readonly string messageId; DeviceClient client; - LeafDeviceSdkLogger sdkLogger; + Option sdkLogger; - LeafDevice(Device device, DeviceClient client, IotHub iotHub, LeafDeviceSdkLogger sdkLogger) + LeafDevice(Device device, DeviceClient client, IotHub iotHub, Option sdkLogger) { this.client = client; this.device = device; @@ -322,14 +322,21 @@ static async Task DeleteIdentityIfFailedAsync(Device device, IotHub static async Task CreateLeafDeviceAsync(Device device, Func clientFactory, IotHub iotHub, CancellationToken token) { DeviceClient client; - LeafDeviceSdkLogger logger; + Option logger = Option.None(); ConnectionStatus status = ConnectionStatus.Disconnected; ConnectionStatusChangeReason reason = ConnectionStatusChangeReason.Connection_Ok; while (true) { client = clientFactory(); - logger = new LeafDeviceSdkLogger(new string[] { "DotNetty-Default", "Microsoft-Azure-Devices", "Azure-Core", "Azure-Identity" }); + logger = Option.Maybe(Context.Current.EnableSdkLoggingForLeafDevice + ? new LeafDeviceSdkLogger(new string[] + { + "DotNetty-Default", + "Microsoft-Azure-Devices", + "Azure-Core", "Azure-Identity" + }) + : null); client.SetConnectionStatusChangesHandler((s, r) => { @@ -349,7 +356,7 @@ static async Task CreateLeafDeviceAsync(Device device, Func l.Dispose()); // Only throw if the caller-supplied token was cancelled. If the inner (30 second) token was // cancelled, fall through and allow the device client to retry. @@ -362,7 +369,7 @@ static async Task CreateLeafDeviceAsync(Device device, Func l.Dispose()); // In the {status == Disconnected, reason == Retry_Expired } scenario, fall through and allow the // client to retry, otherwise throw. @@ -386,11 +393,8 @@ public void Dispose() this.client = null; } - if (this.sdkLogger != null) - { - this.sdkLogger.Dispose(); - this.sdkLogger = null; - } + this.sdkLogger.ForEach(l => l.Dispose()); + this.sdkLogger = Option.None(); } ~LeafDevice() diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs index f6dfc5daa57..42bddffbc2c 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/LeafDeviceSdkLogger.cs @@ -60,7 +60,7 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData) { if (this.eventFilters.Any(ef => eventData.EventSource.Name.StartsWith(ef, StringComparison.Ordinal))) { - string text = $"{DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture)} [SDK] [{eventData.EventSource.Name}-{eventData.EventName}]{(eventData.Payload != null ? $" ({string.Join(", ", eventData.Payload)})." : string.Empty)}"; + string text = $"[{eventData.EventSource.Name}:{eventData.EventName}]{(eventData.Payload != null ? $" ({string.Join(", ", eventData.Payload)})." : string.Empty)}"; Log.Verbose(text); } } diff --git a/test/doc/e2e.md b/test/doc/e2e.md index 697626f22f2..1846c015aea 100644 --- a/test/doc/e2e.md +++ b/test/doc/e2e.md @@ -27,6 +27,7 @@ The end-to-end tests take several parameters, which they expect to find in a fil | `edgeAgentImage` || Docker image to pull/use for Edge Agent. If not given, the default value `mcr.microsoft.com/azureiotedge-agent:1.5` is used. This setting only applies to any configurations deployed by the tests. Note also that the default value is ALWAYS used in config.yaml to start IoT Edge; this setting only applies to any configurations deployed by the tests. | | `edgeHubImage` || Docker image to pull/use for Edge Hub. If not given, `mcr.microsoft.com/azureiotedge-hub:1.5` is used. | | `edgeHubSchemaVersion` || The schema version used for EdgeHub. | +| `enableSdkLoggingForLeafDevice` || Enable logging in the Device SDK when it is used to simulate a leaf device (used by tests: QuickstartCerts, QuickstartChangeSasKey, RouteMessageL3LeafToL4Module, DisableReenableParentEdge, TransparentGateway, GrandparentScopeDevice, PlugAndPlayDeviceClient, X509ManualProvision). This setting is useful for debugging issues in these tests. | | `iotHubResourceId` || Full resource ID (`/resource/subscriptions//resourceGroups//providers/Microsoft.Devices/IotHubs/`) of the IoT hub that will receive metrics messages. Required when running the test 'MetricsCollector', ignored otherwise. | | `loadGenImage` | * | LoadGen image to be used. Required when running PriorityQueue tests, ignored otherwise.| | `logFile` || Path to which all test output will be written, including verbose output. This setting allows the user to capture all the details of a test pass while keeping the shell window output free of visual clutter. Note that daemon logs and module logs are always written to the same directory as the test binaries (e.g., `test/Microsoft.Azure.Devices.Edge.Test/bin/Debug/netcoreapp2.1/*.log`), independent of this parameter. | From 0d9d8ac5110667b2abdcc45c51c96b9cd7445bc3 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Mon, 30 Sep 2024 15:57:13 -0700 Subject: [PATCH 09/10] Add Ubuntu Core to end-to-end pipeline --- builds/e2e/e2e.yaml | 29 +++++++++++++++++++++++++++++ builds/e2e/templates/e2e-run.yaml | 2 ++ 2 files changed, 31 insertions(+) diff --git a/builds/e2e/e2e.yaml b/builds/e2e/e2e.yaml index 09518e1409b..654adb1fab2 100644 --- a/builds/e2e/e2e.yaml +++ b/builds/e2e/e2e.yaml @@ -384,6 +384,35 @@ jobs: parameters: sas_uri: $(sas_uri) +################################################################################ + - job: snaps_ubuntu_core +################################################################################ + displayName: Snaps (Ubuntu Core) + dependsOn: Token + condition: succeeded('Token') + + variables: + os: linux + arch: arm64v8 + artifactName: iotedged-snap-aarch64 + identityServiceArtifactName: packages_snap_aarch64 + identityServicePackageFilter: azure-iot-identity_*_arm64.snap + sas_uri: $[ dependencies.Token.outputs['generate.sas_uri'] ] + + timeoutInMinutes: 90 + + steps: + - script: | + sudo snap install docker + displayName: Install Docker as a snap + - template: templates/e2e-clean-directory.yaml + - template: templates/e2e-setup.yaml + - template: templates/e2e-clear-docker-cached-images.yaml + - template: templates/e2e-run.yaml + parameters: + continue_on_error: true + sas_uri: $(sas_uri) + ################################################################################ - job: redhat8_amd64 ################################################################################ diff --git a/builds/e2e/templates/e2e-run.yaml b/builds/e2e/templates/e2e-run.yaml index 1d46257b7cf..464657750c6 100644 --- a/builds/e2e/templates/e2e-run.yaml +++ b/builds/e2e/templates/e2e-run.yaml @@ -1,4 +1,5 @@ parameters: + continue_on_error: false EventHubCompatibleEndpoint: '$(TestEventHubCompatibleEndpoint)' IotHubConnectionString: '$(TestIotHubConnectionString)' test_type: '' @@ -67,6 +68,7 @@ steps: sudo --preserve-env dotnet test $testFile --no-build --logger 'trx' --filter "$filter" displayName: Run tests ${{ parameters.test_type }} + continueOnError: ${{ parameters.continue_on_error }} env: E2E_DPS_GROUP_KEY: $(TestDpsGroupKeySymmetric) E2E_EVENT_HUB_ENDPOINT: ${{ parameters['EventHubCompatibleEndpoint'] }} From 6bd223c3bf114842fc6c22441fee0dbd4d37a00e Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Wed, 2 Oct 2024 13:14:52 -0700 Subject: [PATCH 10/10] Add pool to ubuntu core job --- builds/e2e/e2e.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builds/e2e/e2e.yaml b/builds/e2e/e2e.yaml index 654adb1fab2..fdfbe3668ca 100644 --- a/builds/e2e/e2e.yaml +++ b/builds/e2e/e2e.yaml @@ -391,6 +391,10 @@ jobs: dependsOn: Token condition: succeeded('Token') + pool: + name: $(pool.custom.name) + demands: ubucore-e2e-tests + variables: os: linux arch: arm64v8