diff --git a/Directory.Build.props b/Directory.Build.props index 58334de25..a136f3c66 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -43,7 +43,7 @@ - + diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index 20d650658..2e9d95388 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -8,13 +8,16 @@ Current package versions: ## Unreleased -- Remove `[Experimental]` 8.8 `GCRA` feature ([#3074 by @mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3074)) +IMPORTANT: for AMR users, this changes the default protocol to RESP3. In some cases, this may require code changes. Please see [this topic](https://stackexchange.github.io/StackExchange.Redis/Resp3) for more information. + +- Remove experimental Redis 8.8 `GCRA` feature ([#3074 by @mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3074)) +- Recognize Azure Managed Redis (AMR) resources in new Azure clouds ([#3068 by @philon-msft](https://github.com/StackExchange/StackExchange.Redis/pull/3068)) +- Prefer RESP3 and avoid opening a separate subscription connection for Azure Managed Redis endpoints ([#3067 by @mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3067)) - Detect server-mode correctly on Valkey 8+ instances ([#3050 by @wipiano](https://github.com/StackExchange/StackExchange.Redis/pull/3050)) - Add Redis 8.8 stream negative acknowledgements (`XNACK`) ([#3058 by @mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3058)) - Add experimental `Aggregate.Count` support for sorted-set combination operations against Redis 8.8 ([#3059 by @mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3059)) -- Support sub-key (hash field) notifications ([#3062 by @mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3062)) +- Support Redis 8.8 sub-key (hash field) notifications ([#3062 by @mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3062)) - Add `ValueCondition` overloads for `SortedSetIncrement`/`SortedSetIncrementAsync`, supporting `ZADD INCR` with existence conditions ([#3071 by @mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3071)) -- Recognize Azure Managed Redis (AMR) resources in new Azure clouds ([#3068 by @philon-msft](https://github.com/StackExchange/StackExchange.Redis/pull/3068)) ## 2.12.14 diff --git a/docs/Resp3.md b/docs/Resp3.md index 126b460f4..6433c168d 100644 --- a/docs/Resp3.md +++ b/docs/Resp3.md @@ -2,35 +2,80 @@ RESP2 and RESP3 are evolutions of the Redis protocol, with RESP3 existing from Redis server version 6 onwards (v7.2+ for Redis Enterprise). The main differences are: -1. RESP3 can carry out-of-band / "push" messages on a single connection, where-as RESP2 requires a separate connection for these messages -2. RESP3 can (when appropriate) convey additional semantic meaning about returned payloads inside the same result structure -3. Some commands (see [this topic](https://github.com/redis/redis-doc/issues/2511)) return different result structures in RESP3 mode; for example a flat interleaved array might become a jagged array +1. RESP3 can carry out-of-band / "push" messages on a single connection, where-as RESP2 requires a separate connection for out-of-band (pub/sub) messages + - this single connection can be of huge benefit in high-usage servers, as it halves the number of connections required +2. RESP3 supports *additional* out-of-band messages that cannot be expressed in RESP2, which allows advanced features such as "smart client handoffs" (a family of + server maintenance notifications) + - these features (not yet implemented in SE.Redis) allow for greater stability in complex deployments +3. RESP3 can (when appropriate) convey additional semantic meaning about returned payloads inside the same result structure + - this is *mostly* relevant to client libraries that do not explicitly interpret the results before exposing to the user, so this does not directly impact SE.Redis itself, + but it is relevant to consumers of SE.Redis that use Lua scripts or ad-hoc commands -For most people, #1 is the main reason to consider RESP3, as in high-usage servers - this can halve the number of connections required. -This is particularly useful in hosted environments where the number of inbound connections to the server is capped as part of a service plan. -Alternatively, where users are currently choosing to disable the out-of-band connection to achieve this, they may now be able to re-enable this -(for example, to receive server maintenance notifications) *without* incurring any additional connection overhead. +For many users, using RESP3 is a "no-brainer" - it offers significant benefits with no real downsides. However, there are some important things to be aware of, and some +migration work that may be required. In particular, some commands *return different result structures* in RESP3 mode; for example a jagged (nested) array might become a "map" +(essentially an interleaved flat array). SE.Redis has been updated to handle these cases transparently, but if you are using `Execute[Async]` or `ScriptEvaluate[Async]` (or if +you are using an additional library that issues ad-hoc commands or scripts on your behalf) you may need to update your processing code to compensate for this. This is discussed more below. -Because of the significance of #3 (and to avoid breaking your code), the library does not currently default to RESP3 mode. This must be enabled explicitly -via `ConfigurationOptions.Protocol` or by adding `,protocol=resp3` (or `,protocol=3`) to the configuration string. +# Enabling RESP3 ---- +RESP2 and RESP3 are both supported options (if the server does not support RESP3, RESP2 will always be used). To make full use of the benefits of RESP3, +the library is moving in the direction of *preferring* RESP3. The default behaviour is: -#3 is a critical one - the library *should* already handle all documented commands that have revised results in RESP3, but if you're using -`Execute[Async]` to issue ad-hoc commands, you may need to update your processing code to compensate for this, ideally using detection to handle -*either* format so that the same code works in both REP2 and RESP3. Since the impacted commands are handled internally by the library, in reality -this should not usually present a difficulty. +| Library version | Endpoint | Default protocol +|-------------------------|-----------------------------------------------------------------|- +| < 2.13 | (any) | RESP2 +| >= 2.13 and < 3.0 | (non-AMR) | RESP2 +| >= 2.13 and < 3.0 | [AMR](https://azure.microsoft.com/products/managed-redis) | RESP3 +| > 3.0† | (any) | RESP3 -The minor (#2) and major (#3) differences to results are only visible to your code when using: +† = planned + +You can override this behaviour by setting the `protocol` option in the connection string, or by setting the `ConfigurationOptions.Protocol` property: + +```csharp +var options = ConfigurationOptions.Parse("someserver"); +options.Protocol = RedisProtocol.Resp3; // or .Resp2 +var muxer = await ConnectionMultiplexer.ConnectAsync(options); +``` + +or + +```csharp +var options = ConfigurationOptions.Parse("someserver,protocol=resp3"); // or =resp2 +var muxer = await ConnectionMultiplexer.ConnectAsync(options); +``` + +You can use this configuration to *explicitly enable* RESP3 on earlier library versions, or to *explicitly disable* RESP3 on later versions, if you encounter issues. + +# Handling RESP3 + +For most users, *no additional work will be required*, or the additional work may be limited to updating libraries; for example, For example, [NRedisStack](https://www.nuget.org/packages/NRedisStack/) +now fully supports RESP3 for the commands it exposes (search, json, time-series, etc). + +Scenarios impacted by RESP3 include: - Lua scripts invoked via the `ScriptEvaluate[Async](...)` or related APIs, that either: - Uses the `redis.setresp(3)` API and returns a value from `redis.[p]call(...)` - Returns a value that satisfies the [LUA to RESP3 type conversion rules](https://redis.io/docs/manual/programmability/lua-api/#lua-to-resp3-type-conversion) -- Ad-hoc commands (in particular: *modules*) that are invoked via the `Execute[Async](string command, ...)` API +- Ad-hoc commands that are invoked via the `Execute[Async](string command, ...)` API + +This delta is *especially* pronounced for some of the "modules" in Redis, even those that now ship by default in OSS Redis, including: +- "search" (`FT.SEARCH`, `FT.AGGREGATE`, etc.) +- "time-series" (`TS.RANGE`, etc.) +- "json" (`JSON.NUMINCRBY`, etc.) + +Note that NRedisStack wraps most of these common modules, and has been updated to understand RESP3; if you are using these modules via NRedisStack, you should update to the latest version; if +you are using these modules via ad-hoc commands, you may need to update your processing code to compensate for this, or consider using NRedisStack instead, which will handle the RESP3 conversion for you. -...both which return `RedisResult`. **If you are not using these APIs, you should not need to do anything additional.** +This leaves a small category of users who are currently using the `RedisResult` type directly (via `Execute[Async](...)` or `ScriptEvaluate[Async](...)`). -Historically, you could use the `RedisResult.Type` property to query the type of data returned (integer, string, etc). In particular: +## Impact on RedisResult + +Firstly, note that it is possible that the *structure* of the data changes between RESP2 and RESP3; for example, a jagged array might become a map, or a single string value might become an array. You will +need to identify these changes (typically via integration tests) and update your code accordingly, ideally with detection code to handle *either* structure so that the same code works in both REP2 and RESP3. + +This is usually combined by using the `RedisResult.Resp3Type` property to query the type of data returned (integer, string, etc). Historically, you could use the `RedisResult.Type` property to query the type of data returned (integer, string, etc). +With RESP3, this is extended: - Two new properties are added: `RedisResult.Resp2Type` and `RedisResult.Resp3Type` - The `Resp3Type` property exposes the new semantic data (when using RESP3) - for example, it can indicate that a value is a double-precision number, a boolean, a map, etc (types that did not historically exist) @@ -42,4 +87,7 @@ Possible changes required due to RESP3: 1. To prevent build warnings, replace usage of `ResultType.MultiBulk` with `ResultType.Array`, and usage of `RedisResult.Type` with `RedisResult.Resp2Type` 2. If you wish to exploit the additional semantic data when enabling RESP3, use `RedisResult.Resp3Type` where appropriate -3. If you are enabling RESP3, you must verify whether the commands you are using can give different result shapes on RESP3 connections \ No newline at end of file +3. If you are enabling RESP3, you must verify whether the commands you are using can give different result shapes on RESP3 connections + +An example of the types of changes required may be seen in the [NRedisStack #471](https://github.com/redis/NRedisStack/pull/471) pull-request, which updates result processing for multiple modules +(and changes the integration tests to run on RESP2 and RESP3 separately). diff --git a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs index cdb55ef2d..42bb2fcbf 100644 --- a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs @@ -1,7 +1,6 @@ using System; using System.Net; using System.Threading.Tasks; -using StackExchange.Redis.Maintenance; namespace StackExchange.Redis.Configuration { @@ -56,9 +55,15 @@ private bool IsHostInDomains(string hostName, string[] domains) /// public override Task AfterConnectAsync(ConnectionMultiplexer muxer, Action log) - => AzureMaintenanceEvent.AddListenerAsync(muxer, log); + => Task.CompletedTask; /// public override bool GetDefaultSsl(EndPointCollection endPoints) => true; + + /// + public override RedisProtocol? Protocol => RedisProtocol.Resp3; // prefer RESP3 on AMR + + /// + public override string ConfigurationChannel => ""; // disable on AMR } } diff --git a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs index e4fa25891..f560c8ce4 100644 --- a/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/DefaultOptionsProvider.cs @@ -259,6 +259,11 @@ protected virtual string GetDefaultClientName() => /// public virtual bool SetClientLibrary => true; + /// + /// Gets the preferred protocol to use for the connection. + /// + public virtual RedisProtocol? Protocol => null; + /// /// Tries to get the RoleInstance Id if Microsoft.WindowsAzure.ServiceRuntime is loaded. /// In case of any failure, swallows the exception and returns null. diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 641fccc95..58a281ddb 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -1169,13 +1169,18 @@ private ConfigurationOptions DoParse(string configuration, bool ignoreUnknown) /// /// Specify the redis protocol type. /// - public RedisProtocol? Protocol { get; set; } + public RedisProtocol? Protocol + { + get => field ?? Defaults.Protocol; + set; + } internal bool TryResp3() { + var protocol = Protocol; // note: deliberately leaving the IsAvailable duplicated to use short-circuit - // if (Protocol is null) + // if (protocol is null) // { // // if not specified, lean on the server version and whether HELLO is available // return new RedisFeatures(DefaultVersion).Resp3 && CommandMap.IsAvailable(RedisCommand.HELLO); @@ -1187,7 +1192,7 @@ internal bool TryResp3() // edge case in the library itself, the break is still visible to external callers via Execute[Async]; with an // abundance of caution, we are therefore making RESP3 explicit opt-in only for now; we may revisit this in a major { - return Protocol.GetValueOrDefault() >= RedisProtocol.Resp3 && CommandMap.IsAvailable(RedisCommand.HELLO); + return protocol.GetValueOrDefault() >= RedisProtocol.Resp3 && CommandMap.IsAvailable(RedisCommand.HELLO); } } diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 36f459f14..9590720c2 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1370,10 +1370,11 @@ internal void GetStatus(ILogger? log) private void ActivateAllServers(ILogger? log) { + bool hasSubscriptions = GetSubscriptionsCount() != 0; foreach (var server in GetServerSnapshot()) { server.Activate(ConnectionType.Interactive, log); - if (server.SupportsSubscriptions && !server.KnowOrAssumeResp3()) + if (hasSubscriptions && server.SupportsSubscriptions && !server.KnowOrAssumeResp3()) { // Intentionally not logging the sub connection server.Activate(ConnectionType.Subscription, null); diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index e515b6e0c..ec93df20e 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,4 +1,7 @@ #nullable enable +override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.ConfigurationChannel.get -> string! +override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? +virtual StackExchange.Redis.Configuration.DefaultOptionsProvider.Protocol.get -> StackExchange.Redis.RedisProtocol? [SER006]static StackExchange.Redis.RedisChannel.SubKeyEvent(StackExchange.Redis.KeyNotificationType type, int? database = null) -> StackExchange.Redis.RedisChannel [SER006]static StackExchange.Redis.RedisChannel.SubKeyEvent(System.ReadOnlySpan type, int? database) -> StackExchange.Redis.RedisChannel [SER006]static StackExchange.Redis.RedisChannel.SubKeySpaceEvent(StackExchange.Redis.KeyNotificationType type, in StackExchange.Redis.RedisKey key, int? database = null) -> StackExchange.Redis.RedisChannel diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 0da791d3c..129cfbfed 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -107,7 +107,6 @@ public int Databases public bool IsConnecting => interactive?.IsConnecting == true; public bool IsConnected => interactive?.IsConnected == true; public bool IsSubscriberConnected => KnowOrAssumeResp3() ? IsConnected : subscription?.IsConnected == true; - public bool KnowOrAssumeResp3() { var protocol = interactive?.Protocol; @@ -627,7 +626,7 @@ internal bool IsSelectable(RedisCommand command, bool allowDisconnected = false) { // Until we've connected at least once, we're going to have a DidNotRespond unselectable reason present var bridge = unselectableReasons == 0 || (allowDisconnected && unselectableReasons == UnselectableFlags.DidNotRespond) - ? GetBridge(command, false) + ? GetBridge(command, true) : null; return bridge != null && (allowDisconnected || bridge.IsConnected); diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index a0ec70ae1..77f670420 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -166,6 +166,26 @@ public void ConfigurationOptionsDefaultForAzureManagedRedis(string hostAndPort, Assert.Equal(sslShouldBeEnabled, options.Ssl); } + [Theory] + // azure managed redis, no overrides + [InlineData("contoso.redis.azure.net:10000", RedisProtocol.Resp3, true)] // default + [InlineData("contoso.redis.azure.net:10000,protocol=resp2", RedisProtocol.Resp2, false)] // opt-out + [InlineData("contoso.redis.azure.net:10000,protocol=resp3", RedisProtocol.Resp3, true)] // opt-in + // azure redis cache, no overrides (we expect this to change in v3) + [InlineData("contoso.redis.cache.windows.net:6380", null, false)] // default + [InlineData("contoso.redis.cache.windows.net:6380,protocol=resp2", RedisProtocol.Resp2, false)] // opt-out + [InlineData("contoso.redis.cache.windows.net:6380,protocol=resp3", RedisProtocol.Resp3, true)] // opt-in + // arbitrary endpoint (we expect this to change in v3) + [InlineData("myserver:6379", null, false)] // default + [InlineData("myserver:6379,protocol=resp2", RedisProtocol.Resp2, false)] // opt-out + [InlineData("myserver:6379,protocol=resp3", RedisProtocol.Resp3, true)] // opt-in + public void CorrectRespProtocol(string config, RedisProtocol? expected, bool useResp3) + { + var options = ConfigurationOptions.Parse(config); + Assert.Equal(expected, options.Protocol); + Assert.Equal(useResp3, options.TryResp3()); + } + [Fact] public void ConfigurationOptionsForAzureWhenSpecified() { diff --git a/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs b/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs index a01e845da..8f6047e03 100644 --- a/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs +++ b/tests/StackExchange.Redis.Tests/DefaultOptionsTests.cs @@ -86,6 +86,93 @@ public void IsMatchOnAzureManagedRedisDomain(string hostName) Assert.IsType(provider); } + [Theory] + [InlineData(RedisProtocol.Resp2)] + [InlineData(RedisProtocol.Resp3)] + public async Task AzureManagedRedisConnectsWithoutSubscriptionConnection(RedisProtocol protocol) + { + using var serverObj = new InProcessTestServer(Output, new DnsEndPoint("contoso.redis.azure.net", 10000), useSsl: true); + var config = serverObj.GetClientConfig(); + config.ClientName = Guid.NewGuid().ToString().Replace("-", ""); + config.Protocol = protocol; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config, Writer); + + var server = conn.GetServer(conn.GetEndPoints().Single()); + var interactiveId = ((IInternalConnectionMultiplexer)conn).GetConnectionId(server.EndPoint, ConnectionType.Interactive); + var clients = await server.ClientListAsync(); + var namedClients = clients.Where(x => x.Name == config.ClientName).ToArray(); + + Assert.Equal(protocol, server.Protocol); + Assert.Equal(1, serverObj.ClientCount); + Assert.NotNull(interactiveId); + Assert.Single(namedClients); + var self = Assert.Single(clients, x => x.Id == interactiveId); + Assert.Equal(ClientType.Normal, self.ClientType); + Assert.Equal(0, self.SubscriptionCount); + Assert.Equal(0, self.PatternSubscriptionCount); + Assert.Equal(0, self.ShardedSubscriptionCount); + Assert.Equal(protocol, self.Protocol); + + await AssertCanPubSubAsync(conn, $"{nameof(AzureManagedRedisConnectsWithoutSubscriptionConnection)}:{protocol}"); + } + + [Fact] + public async Task VanillaResp2ConnectsWithSeparatePubSubConnection() + { + using var serverObj = new InProcessTestServer(Output, new DnsEndPoint("redis.contoso.com", 10000), useSsl: true); + var config = serverObj.GetClientConfig(); + config.Protocol = RedisProtocol.Resp2; + Log($"QueueWhileDisconnected: {config.BacklogPolicy.QueueWhileDisconnected}"); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config, Writer); + var sub = conn.GetSubscriber(); + await sub.SubscribeAsync(RedisChannel.Literal(nameof(VanillaResp2ConnectsWithSeparatePubSubConnection)), (_, _) => { }); + + var server = conn.GetServer(conn.GetEndPoints().Single()); + var mux = (IInternalConnectionMultiplexer)conn; + var interactiveId = mux.GetConnectionId(server.EndPoint, ConnectionType.Interactive); + var subscriptionId = mux.GetConnectionId(server.EndPoint, ConnectionType.Subscription); + var clients = server.ClientList(); + var namedClients = clients.Where(x => x.Name == conn.ClientName).ToArray(); + + Assert.Equal(RedisProtocol.Resp2, server.Protocol); + Assert.Equal(2, serverObj.ClientCount); + Assert.NotNull(interactiveId); + Assert.NotNull(subscriptionId); + Assert.NotEqual(interactiveId, subscriptionId); + Assert.Equal(2, namedClients.Length); + + var interactive = Assert.Single(clients, x => x.Id == interactiveId); + var subscription = Assert.Single(clients, x => x.Id == subscriptionId); + Assert.Equal(ClientType.Normal, interactive.ClientType); + Assert.Equal(ClientType.PubSub, subscription.ClientType); + Assert.True(subscription.SubscriptionCount > 0); + + await AssertCanPubSubAsync(conn, nameof(VanillaResp2ConnectsWithSeparatePubSubConnection)); + } + + private static async Task AssertCanPubSubAsync(ConnectionMultiplexer conn, string channelName) + { + var sub = conn.GetSubscriber(); + var channel = RedisChannel.Literal(channelName); + var payload = (RedisValue)("payload:" + channelName); + TaskCompletionSource tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + + await sub.SubscribeAsync(channel, (_, message) => tcs.TrySetResult(message)); + try + { + await sub.PublishAsync(channel, payload); + var completed = await Task.WhenAny(tcs.Task, Task.Delay(5000, TestContext.Current.CancellationToken)); + Assert.Same(tcs.Task, completed); + Assert.Equal(payload, await tcs.Task); + } + finally + { + await sub.UnsubscribeAsync(channel); + } + } + [Fact] public void AllOverridesFromDefaultsProp() { diff --git a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs index af9f1ee44..e861bb6d3 100644 --- a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs +++ b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs @@ -3,7 +3,13 @@ using System.IO; using System.IO.Pipelines; using System.Net; +using System.Net.Security; using System.Net.Sockets; +#if !NETFRAMEWORK +using System.Security.Authentication; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +#endif using System.Text; using System.Threading; using System.Threading.Tasks; @@ -17,11 +23,28 @@ namespace StackExchange.Redis.Tests; public class InProcessTestServer : MemoryCacheRedisServer { private readonly ITestOutputHelper? _log; - public InProcessTestServer(ITestOutputHelper? log = null, EndPoint? endpoint = null) +#if !NETFRAMEWORK + private readonly X509Certificate2? _serverCertificate; + private readonly string? _serverCertificateThumbprint; + private readonly RemoteCertificateValidationCallback? _certificateValidationCallback; +#endif + + public InProcessTestServer(ITestOutputHelper? log = null, EndPoint? endpoint = null, bool useSsl = false) : base(endpoint) { RedisVersion = RedisFeatures.v6_0_0; // for client to expect RESP3 _log = log; +#if NETFRAMEWORK + UseSsl = false; +#else + UseSsl = useSsl; + if (useSsl) + { + _serverCertificate = CreateServerCertificate(DefaultEndPoint); + _serverCertificateThumbprint = _serverCertificate.Thumbprint; + _certificateValidationCallback = ValidateServerCertificate; + } +#endif // ReSharper disable once VirtualMemberCallInConstructor _log?.WriteLine($"Creating in-process server: {ToString()}"); Tunnel = new InProcTunnel(this); @@ -90,6 +113,13 @@ public ConfigurationOptions GetClientConfig(bool withPubSub = true, bool default // WriteMode = (BufferedStreamWriter.WriteMode)writeMode, }; if (!string.IsNullOrEmpty(Password)) config.Password = Password; + config.Ssl = UseSsl; // explicitly, ignore provider defaults + if (UseSsl) + { +#if !NETFRAMEWORK + config.CertificateValidation += _certificateValidationCallback; +#endif + } /* useful for viewing *outbound* data in the log #if DEBUG @@ -121,6 +151,7 @@ public ConfigurationOptions GetClientConfig(bool withPubSub = true, bool default } public Tunnel Tunnel { get; } + public bool UseSsl { get; } public override void Log(string message) { @@ -200,6 +231,70 @@ protected override void OnSkippedReply(RedisClient client) base.OnSkippedReply(client); } + protected override void Dispose(bool disposing) + { +#if !NETFRAMEWORK + if (disposing) + { + _serverCertificate?.Dispose(); + } +#endif + base.Dispose(disposing); + } + +#if !NETFRAMEWORK + private bool ValidateServerCertificate(object sender, X509Certificate? certificate, X509Chain? chain, SslPolicyErrors errors) + { + if (errors == SslPolicyErrors.None) + { + return true; + } + + return certificate is not null + && _serverCertificateThumbprint is not null + && string.Equals(certificate.GetCertHashString(), _serverCertificateThumbprint, StringComparison.OrdinalIgnoreCase); + } + + private static X509Certificate2 CreateServerCertificate(EndPoint endpoint) + { + var now = DateTimeOffset.UtcNow; + var subjectName = GetCertificateSubjectName(endpoint); + + using var rsa = RSA.Create(2048); + var request = new CertificateRequest($"CN={subjectName}", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + request.CertificateExtensions.Add(new X509BasicConstraintsExtension(false, false, 0, false)); + request.CertificateExtensions.Add(new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, false)); + request.CertificateExtensions.Add( + new X509EnhancedKeyUsageExtension( + new OidCollection { new Oid("1.3.6.1.5.5.7.3.1") }, + false)); + + var san = new SubjectAlternativeNameBuilder(); + switch (endpoint) + { + case DnsEndPoint dns: + san.AddDnsName(dns.Host); + break; + case IPEndPoint ip: + san.AddIpAddress(ip.Address); + break; + } + request.CertificateExtensions.Add(san.Build()); + + using var certificate = request.CreateSelfSigned(now.AddMinutes(-5), now.AddDays(7)); +#pragma warning disable SYSLIB0057 + return new X509Certificate2(certificate.Export(X509ContentType.Pfx)); +#pragma warning restore SYSLIB0057 + + static string GetCertificateSubjectName(EndPoint endpoint) => endpoint switch + { + DnsEndPoint dns => dns.Host, + IPEndPoint ip => ip.Address.ToString(), + _ => "localhost", + }; + } +#endif + private sealed class InProcTunnel( InProcessTestServer server, PipeOptions? pipeOptions = null) : Tunnel @@ -225,16 +320,40 @@ private sealed class InProcTunnel( if (server.TryGetNode(endpoint, out var node)) { await server.OnAcceptClientAsync(endpoint); + server._log?.WriteLine( + $"[{endpoint}] accepting {connectionType} mapped to {server.ServerType} node {node} via {(server.UseSsl ? "TLS" : "plaintext")}"); var clientToServer = new Pipe(pipeOptions ?? PipeOptions.Default); var serverToClient = new Pipe(pipeOptions ?? PipeOptions.Default); - var serverSide = new Duplex(clientToServer.Reader, serverToClient.Writer); + var serverInput = clientToServer.Reader.AsStream(); + var serverOutput = serverToClient.Writer.AsStream(); + var serverTransport = new DuplexStream(serverInput, serverOutput); - TaskCompletionSource clientTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); - Task.Run(async () => await server.RunClientAsync(serverSide, node: node, state: clientTcs), cancellationToken).RedisFireAndForget(); - if (!clientTcs.Task.Wait(1000)) throw new TimeoutException("Client not connected"); - var client = clientTcs.Task.Result; - server._log?.WriteLine( - $"[{client}] connected ({connectionType} mapped to {server.ServerType} node {node})"); + if (server.UseSsl) + { +#if !NETFRAMEWORK + Task.Run( + async () => + { + using var ssl = new SslStream(serverTransport, leaveInnerStreamOpen: false); + await ssl.AuthenticateAsServerAsync( + server._serverCertificate!, + clientCertificateRequired: false, + enabledSslProtocols: SslProtocols.None, + checkCertificateRevocation: false).ConfigureAwait(false); + var serverSide = new StreamDuplexPipe(ssl); + await server.RunClientAsync(serverSide, node: node, state: null).ConfigureAwait(false); + }, + cancellationToken).RedisFireAndForget(); +#endif + } + else + { + var serverSide = new Duplex(clientToServer.Reader, serverToClient.Writer); + TaskCompletionSource clientTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + Task.Run(async () => await server.RunClientAsync(serverSide, node: node, state: clientTcs), cancellationToken).RedisFireAndForget(); + if (!clientTcs.Task.Wait(1000)) throw new TimeoutException("Client not connected"); + _ = clientTcs.Task.Result; + } var readStream = serverToClient.Reader.AsStream(); var writeStream = clientToServer.Writer.AsStream(); @@ -256,6 +375,12 @@ public ValueTask Dispose() return default; } } + + private sealed class StreamDuplexPipe(Stream stream) : IDuplexPipe + { + public PipeReader Input { get; } = PipeReader.Create(stream); + public PipeWriter Output { get; } = PipeWriter.Create(stream); + } } protected virtual ValueTask OnAcceptClientAsync(EndPoint endpoint) => default; diff --git a/tests/StackExchange.Redis.Tests/PubSubKeyNotificationTests.cs b/tests/StackExchange.Redis.Tests/PubSubKeyNotificationTests.cs index 75ee4f9b4..88f17e69c 100644 --- a/tests/StackExchange.Redis.Tests/PubSubKeyNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/PubSubKeyNotificationTests.cs @@ -85,6 +85,7 @@ private async Task ConnectAsync(KeyNotificationK var config = (await server.ConfigGetAsync("notify-keyspace-events")).Single(); var value = config.Value.ToString() ?? ""; + Log($"Server {ep} notify-keyspace-events config: {value}"); // Check that the config contains all required tokens foreach (var token in requiredTokens) { diff --git a/tests/StackExchange.Redis.Tests/SSLTests.cs b/tests/StackExchange.Redis.Tests/SSLTests.cs index 96d964b23..f068e0aae 100644 --- a/tests/StackExchange.Redis.Tests/SSLTests.cs +++ b/tests/StackExchange.Redis.Tests/SSLTests.cs @@ -416,6 +416,8 @@ public void Issue883_Exhaustive() Ssl = true, AbortOnConnectFail = false, }; + _ = a.Defaults; + _ = b.Defaults; // ensure the lazily materialized provider matches the parsed shape Log($"computed: {b.ToString(true)}"); Log("Checking endpoints..."); @@ -429,6 +431,14 @@ public void Issue883_Exhaustive() Array.Sort(fields, (x, y) => string.CompareOrdinal(x.Name, y.Name)); foreach (var field in fields) { + if (field.Name == "defaultOptions") + { + var x = field.GetValue(a); + var y = field.GetValue(b); + Log($"{field.Name}: {(x == null ? "(null)" : x.GetType().Name)} vs {(y == null ? "(null)" : y.GetType().Name)}"); + Check(field.Name + ".Type", x?.GetType(), y?.GetType()); + continue; + } Check(field.Name, field.GetValue(a), field.GetValue(b)); } } diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index 54a7fbe04..a3b52dec1 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -462,6 +462,32 @@ protected virtual TypedRedisValue ClientReply(RedisClient client, in RedisReques protected virtual TypedRedisValue ClientId(RedisClient client, in RedisRequest request) => TypedRedisValue.Integer(client.Id); + [RedisCommand(2, nameof(RedisCommand.CLIENT), "list", LockFree = true)] + protected virtual TypedRedisValue ClientList(RedisClient client, in RedisRequest request) + { + var sb = new StringBuilder(); + ForAllClients( + sb, + static (other, state) => + { + if (state.Length != 0) state.AppendLine(); + state.Append("id=").Append(other.Id) + .Append(" addr=").Append(other.Node.Host).Append(':').Append(other.Node.Port) + .Append(" age=0 idle=0") + .Append(" db=").Append(other.Database) + .Append(" sub=").Append(other.SubscriptionCount) + .Append(" psub=").Append(other.PatternSubscriptionCount) + .Append(" ssub=").Append(other.ShardedSubscriptionCount) + .Append(" multi=0") + .Append(" cmd=NULL") + .Append(" name=").Append(other.Name ?? "") + .Append(" resp=").Append(other.Protocol is RedisProtocol.Resp3 ? 3 : 2) + .Append(" flags=").Append(other.IsSubscriber ? "P" : "N"); + return 1; + }); + return TypedRedisValue.BulkString(sb.ToString()); + } + [RedisCommand(4, nameof(RedisCommand.CLIENT), "setinfo", LockFree = true)] protected virtual TypedRedisValue ClientSetInfo(RedisClient client, in RedisRequest request) => TypedRedisValue.OK; // only exists to keep logs clean diff --git a/version.json b/version.json index c2ded472b..9500cbb6f 100644 --- a/version.json +++ b/version.json @@ -1,5 +1,5 @@ { - "version": "2.12", + "version": "2.13", "versionHeightOffset": 0, "assemblyVersion": "2.0", "publicReleaseRefSpec": [