From 497aea58288e54dfa3952784028cdf8096dcc670 Mon Sep 17 00:00:00 2001 From: Graeme Foster Date: Mon, 3 Apr 2023 14:51:27 +0800 Subject: [PATCH 01/11] Up to net 4.52 and added a net 6 target (cannot find a 4.5 development pack anymore) --- .idea/.idea.Splunk.Logging/.idea/.name | 1 + .../.idea/indexLayout.xml | 8 + .../.idea/projectSettingsUpdater.xml | 6 + .idea/.idea.Splunk.Logging/.idea/vcs.xml | 6 + .../.idea.Splunk.Logging/.idea/workspace.xml | 166 ++++++++++++++++++ examples/examples.csproj | 2 +- .../Splunk.Logging.Common.csproj | 2 +- .../Splunk.Logging.SLAB.csproj | 2 +- .../Splunk.Logging.TraceListener.csproj | 2 +- standalone-test/standalone-test.csproj | 2 +- test/unit-tests/unit-tests.csproj | 2 +- 11 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 .idea/.idea.Splunk.Logging/.idea/.name create mode 100644 .idea/.idea.Splunk.Logging/.idea/indexLayout.xml create mode 100644 .idea/.idea.Splunk.Logging/.idea/projectSettingsUpdater.xml create mode 100644 .idea/.idea.Splunk.Logging/.idea/vcs.xml create mode 100644 .idea/.idea.Splunk.Logging/.idea/workspace.xml diff --git a/.idea/.idea.Splunk.Logging/.idea/.name b/.idea/.idea.Splunk.Logging/.idea/.name new file mode 100644 index 0000000..8216a96 --- /dev/null +++ b/.idea/.idea.Splunk.Logging/.idea/.name @@ -0,0 +1 @@ +Splunk.Logging \ No newline at end of file diff --git a/.idea/.idea.Splunk.Logging/.idea/indexLayout.xml b/.idea/.idea.Splunk.Logging/.idea/indexLayout.xml new file mode 100644 index 0000000..7b08163 --- /dev/null +++ b/.idea/.idea.Splunk.Logging/.idea/indexLayout.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/.idea.Splunk.Logging/.idea/projectSettingsUpdater.xml b/.idea/.idea.Splunk.Logging/.idea/projectSettingsUpdater.xml new file mode 100644 index 0000000..4bb9f4d --- /dev/null +++ b/.idea/.idea.Splunk.Logging/.idea/projectSettingsUpdater.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/.idea.Splunk.Logging/.idea/vcs.xml b/.idea/.idea.Splunk.Logging/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/.idea.Splunk.Logging/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/.idea.Splunk.Logging/.idea/workspace.xml b/.idea/.idea.Splunk.Logging/.idea/workspace.xml new file mode 100644 index 0000000..8ca5d9b --- /dev/null +++ b/.idea/.idea.Splunk.Logging/.idea/workspace.xml @@ -0,0 +1,166 @@ + + + + examples/examples.csproj + standalone-test/standalone-test.csproj + + + + + + + + + + + + + + + + + + + + { + "keyToString": { + "RunOnceActivity.OpenProjectViewOnStart": "true", + "RunOnceActivity.ShowReadmeOnStart": "true", + "WebServerToolWindowFactoryState": "false", + "XThreadsFramesViewSplitterKey": "0.37819025", + "nodejs_package_manager_path": "npm", + "settings.editor.selected.configurable": "SolutionBuilderGeneralOptionsPage", + "vue.rearranger.settings.migration": "true" + }, + "keyToStringList": { + "rider.external.source.directories": [ + "C:\\Users\\graemefoster\\AppData\\Roaming\\JetBrains\\Rider2022.3\\resharper-host\\DecompilerCache", + "C:\\Users\\graemefoster\\AppData\\Roaming\\JetBrains\\Rider2022.3\\resharper-host\\SourcesCache", + "C:\\Users\\graemefoster\\AppData\\Local\\Symbols\\src" + ] + } +} + + + + + + + + + + + 1680251006425 + + + + + + + + + + + + + file://$PROJECT_DIR$/standalone-test/Program.cs + 31 + + + + + + + + + file://$PROJECT_DIR$/src/Splunk.Logging.Common/HttpEventCollectorSender.cs + 337 + + + + + + + + + file://$PROJECT_DIR$/src/Splunk.Logging.Common/HttpEventCollectorSender.cs + 318 + + + + + + + + + file://$PROJECT_DIR$/src/Splunk.Logging.Common/HttpEventCollectorSender.cs + 325 + + + + + + + + + + + \ No newline at end of file diff --git a/examples/examples.csproj b/examples/examples.csproj index 7ac5b4d..c43378e 100644 --- a/examples/examples.csproj +++ b/examples/examples.csproj @@ -8,7 +8,7 @@ Exe examples examples - v4.5 + v4.5.2 512 true diff --git a/src/Splunk.Logging.Common/Splunk.Logging.Common.csproj b/src/Splunk.Logging.Common/Splunk.Logging.Common.csproj index d0e6016..4521ca8 100644 --- a/src/Splunk.Logging.Common/Splunk.Logging.Common.csproj +++ b/src/Splunk.Logging.Common/Splunk.Logging.Common.csproj @@ -1,7 +1,7 @@ - net45 + net452;net6.0 Splunk, Inc. Copyright © Splunk, Inc. 2015 1.7.2.0 diff --git a/src/Splunk.Logging.SLAB/Splunk.Logging.SLAB.csproj b/src/Splunk.Logging.SLAB/Splunk.Logging.SLAB.csproj index 95b5609..17e2be5 100644 --- a/src/Splunk.Logging.SLAB/Splunk.Logging.SLAB.csproj +++ b/src/Splunk.Logging.SLAB/Splunk.Logging.SLAB.csproj @@ -1,12 +1,12 @@  - net45 Splunk, Inc. Copyright © Splunk, Inc. 2015 1.7.2.0 1.7.2.0 1.7.2 + net452 diff --git a/src/Splunk.Logging.TraceListener/Splunk.Logging.TraceListener.csproj b/src/Splunk.Logging.TraceListener/Splunk.Logging.TraceListener.csproj index 7ea91f7..6e135d6 100644 --- a/src/Splunk.Logging.TraceListener/Splunk.Logging.TraceListener.csproj +++ b/src/Splunk.Logging.TraceListener/Splunk.Logging.TraceListener.csproj @@ -1,12 +1,12 @@  - net45 Splunk, Inc. Copyright © Splunk, Inc. 2015 1.7.2.0 1.7.2.0 1.7.2 + net452 diff --git a/standalone-test/standalone-test.csproj b/standalone-test/standalone-test.csproj index e1845ea..4f26ed8 100644 --- a/standalone-test/standalone-test.csproj +++ b/standalone-test/standalone-test.csproj @@ -9,7 +9,7 @@ Properties standalone_test standalone-test - v4.5 + v4.5.2 512 true diff --git a/test/unit-tests/unit-tests.csproj b/test/unit-tests/unit-tests.csproj index a826d43..9146710 100644 --- a/test/unit-tests/unit-tests.csproj +++ b/test/unit-tests/unit-tests.csproj @@ -1,7 +1,7 @@  - net45 + net452 From cf40fa6980b0f9ca1dbc1819f78cb828b6990ffb Mon Sep 17 00:00:00 2001 From: Graeme Foster Date: Mon, 3 Apr 2023 15:17:36 +0800 Subject: [PATCH 02/11] Start task --- .idea/.idea.Splunk.Logging/.idea/workspace.xml | 9 ++++++--- src/Splunk.Logging.Common/HttpEventCollectorSender.cs | 7 +++---- standalone-test/standalone-test.csproj | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.idea/.idea.Splunk.Logging/.idea/workspace.xml b/.idea/.idea.Splunk.Logging/.idea/workspace.xml index 8ca5d9b..70a3523 100644 --- a/.idea/.idea.Splunk.Logging/.idea/workspace.xml +++ b/.idea/.idea.Splunk.Logging/.idea/workspace.xml @@ -5,7 +5,10 @@ standalone-test/standalone-test.csproj - + + + + - + + + + - + + + + + - { - "keyToString": { - "RunOnceActivity.OpenProjectViewOnStart": "true", - "RunOnceActivity.ShowReadmeOnStart": "true", - "WebServerToolWindowFactoryState": "false", - "XThreadsFramesViewSplitterKey": "0.37819025", - "nodejs_package_manager_path": "npm", - "settings.editor.selected.configurable": "SolutionBuilderGeneralOptionsPage", - "vue.rearranger.settings.migration": "true" + +}]]> + - - - - - file://$PROJECT_DIR$/standalone-test/Program.cs - 31 - - - - - - - - - file://$PROJECT_DIR$/src/Splunk.Logging.Common/HttpEventCollectorSender.cs - 337 - - - - - - - - - file://$PROJECT_DIR$/src/Splunk.Logging.Common/HttpEventCollectorSender.cs - 318 - - - - - - - - - file://$PROJECT_DIR$/src/Splunk.Logging.Common/HttpEventCollectorSender.cs - 325 - - - - - - - - - + + + \ No newline at end of file diff --git a/src/Splunk.Logging.Common/BatchBuffers/IBuffer.cs b/src/Splunk.Logging.Common/BatchBuffers/IBuffer.cs new file mode 100644 index 0000000..35895b7 --- /dev/null +++ b/src/Splunk.Logging.Common/BatchBuffers/IBuffer.cs @@ -0,0 +1,13 @@ +using System; +using System.Net.Http; +using Newtonsoft.Json; + +namespace Splunk.Logging.BatchBuffers +{ + public interface IBuffer : IDisposable + { + void Append(HttpEventCollectorEventInfo serializedEventInfo); + long Length { get; } + HttpContent BuildHttpContent(string mediaType); + } +} \ No newline at end of file diff --git a/src/Splunk.Logging.Common/BatchBuffers/MemoryStreamBatchBuffer.cs b/src/Splunk.Logging.Common/BatchBuffers/MemoryStreamBatchBuffer.cs new file mode 100644 index 0000000..c2d8fb0 --- /dev/null +++ b/src/Splunk.Logging.Common/BatchBuffers/MemoryStreamBatchBuffer.cs @@ -0,0 +1,60 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Newtonsoft.Json; + +namespace Splunk.Logging.BatchBuffers +{ + public class MemoryStreamBatchBuffer : IBuffer + { + private readonly string filePath; + private readonly JsonSerializer serializer; + private readonly TextWriter writer; + private readonly FileStream fileStream; + + public MemoryStreamBatchBuffer() + { + filePath = Path.GetTempFileName(); + serializer = JsonSerializer.Create(); + fileStream = File.OpenWrite(filePath); + writer = new StreamWriter(fileStream); + } + + public void Append(HttpEventCollectorEventInfo serializedEventInfo) + { + serializer.Serialize(writer, serializedEventInfo); + writer.Flush(); + } + + public long Length => fileStream.Length; + + public HttpContent BuildHttpContent(string mediaType) + { + writer.Flush(); + writer.Close(); + return new StreamContent(File.OpenRead(filePath)) + { + Headers = + { + ContentType = new MediaTypeHeaderValue(mediaType) + } + }; + } + + public void Dispose() + { + writer?.Dispose(); + fileStream?.Dispose(); + try + { + File.Delete(filePath); + } + catch (Exception) + { + //Ignore + } + } + } +} \ No newline at end of file diff --git a/src/Splunk.Logging.Common/BatchBuffers/StringBuilderBatchBuffer.cs b/src/Splunk.Logging.Common/BatchBuffers/StringBuilderBatchBuffer.cs new file mode 100644 index 0000000..e502e93 --- /dev/null +++ b/src/Splunk.Logging.Common/BatchBuffers/StringBuilderBatchBuffer.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.Net.Http; +using System.Text; +using Newtonsoft.Json; + +namespace Splunk.Logging.BatchBuffers +{ + public class StringBuilderBatchBuffer : IBuffer + { + private readonly StringBuilder builder = new StringBuilder(); + private readonly StringWriter writer; + private readonly JsonSerializer serializer; + + public StringBuilderBatchBuffer() + { + writer = new StringWriter(builder); + serializer = JsonSerializer.Create(); + } + + public void Append(HttpEventCollectorEventInfo serializedEventInfo) + { + serializer.Serialize(writer, serializedEventInfo); + writer.Flush(); + } + + public long Length => builder.Length; + + public HttpContent BuildHttpContent(string mediaType) + { + return new StringContent(builder.ToString(), Encoding.UTF8, mediaType); + } + + public void Dispose() + { + } + } +} \ No newline at end of file diff --git a/src/Splunk.Logging.Common/HttpEventCollectorSender.cs b/src/Splunk.Logging.Common/HttpEventCollectorSender.cs index 842d460..47b205e 100644 --- a/src/Splunk.Logging.Common/HttpEventCollectorSender.cs +++ b/src/Splunk.Logging.Common/HttpEventCollectorSender.cs @@ -20,12 +20,14 @@ using Newtonsoft.Json.Serialization; using System; using System.Collections.Generic; +using System.IO; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; using System.Threading.Tasks; +using Splunk.Logging.BatchBuffers; namespace Splunk.Logging { @@ -99,6 +101,17 @@ public enum SendMode Sequential }; + /// + /// Where to buffer log messages before sending. Default is to use InMemory, but + /// to save memory use at the expense of slightly slower sends, TempFiles will write to the + /// file system, and delete the files on successful send. + /// + public enum BufferMode + { + InMemory, + TempFiles + }; + private const string HttpContentTypeMedia = "application/json"; private const string HttpEventCollectorPath = "/services/collector/event/1.0"; private const string AuthorizationHeaderScheme = "Splunk"; @@ -114,8 +127,8 @@ public enum SendMode private SendMode sendMode = SendMode.Parallel; private Task activePostTask = null; private object eventsBatchLock = new object(); - private List eventsBatch = new List(); - private StringBuilder serializedEventsBatch = new StringBuilder(); + private List eventsBatch; + private IBuffer serializedEventsBatch; private Timer timer; private HttpClient httpClient = null; @@ -148,7 +161,8 @@ public HttpEventCollectorSender( SendMode sendMode, int batchInterval, int batchSizeBytes, int batchSizeCount, HttpEventCollectorMiddleware middleware, - HttpEventCollectorFormatter formatter = null) + HttpEventCollectorFormatter formatter = null, + BufferMode bufferMode = BufferMode.InMemory) { this.serializer = new JsonSerializer(); serializer.NullValueHandling = NullValueHandling.Ignore; @@ -163,6 +177,9 @@ public HttpEventCollectorSender( this.token = token; this.middleware = middleware; this.formatter = formatter; + this.bufferMode = bufferMode; + + BuildBuffers(); // special case - if batch interval is specified without size and count // they are set to "infinity", i.e., batch may have any size @@ -243,16 +260,10 @@ public void Send( private void DoSerialization(HttpEventCollectorEventInfo ei) { - string serializedEventInfo; - if (formatter == null) - { - serializedEventInfo = SerializeEventInfo(ei); - } - else + if (formatter != null) { var formattedEvent = formatter(ei); ei.Event = formattedEvent; - serializedEventInfo = JsonConvert.SerializeObject(ei); } // we use lock serializedEventsBatch to synchronize both @@ -260,7 +271,7 @@ private void DoSerialization(HttpEventCollectorEventInfo ei) lock (eventsBatchLock) { eventsBatch.Add(ei); - serializedEventsBatch.Append(serializedEventInfo); + serializedEventsBatch.Append(ei); if (eventsBatch.Count >= batchSizeCount || serializedEventsBatch.Length >= batchSizeBytes) { @@ -295,16 +306,7 @@ public Task FlushAsync() task.Start(); return task; } - - /// - /// Serialize event info into a json string - /// - /// - /// - public static string SerializeEventInfo(HttpEventCollectorEventInfo eventInfo) - { - return JsonConvert.SerializeObject(eventInfo); - } + /// /// Flush all batched events immediately. @@ -327,20 +329,25 @@ private void FlushInternal() // flush events according to the system operation mode if (this.sendMode == SendMode.Sequential) - FlushInternalSequentialMode(this.eventsBatch, this.serializedEventsBatch.ToString()); + FlushInternalSequentialMode(this.eventsBatch, this.serializedEventsBatch); else - FlushInternalSingleBatch(this.eventsBatch, this.serializedEventsBatch.ToString()); + FlushInternalSingleBatch(this.eventsBatch, this.serializedEventsBatch); // we explicitly create new objects instead to clear and reuse // the old ones because Flush works in async mode // and can use "previous" containers - this.serializedEventsBatch = new StringBuilder(); + BuildBuffers(); + } + + private void BuildBuffers() + { + this.serializedEventsBatch = bufferMode == BufferMode.InMemory ? (IBuffer)new StringBuilderBatchBuffer() : new MemoryStreamBatchBuffer(); this.eventsBatch = new List(); } private void FlushInternalSequentialMode( List events, - String serializedEvents) + IBuffer serializedEventsBuilder) { // post data and update tasks counter Interlocked.Increment(ref activeAsyncTasksCount); @@ -350,23 +357,23 @@ private void FlushInternalSequentialMode( { this.activePostTask = Task.Factory.StartNew(() => { - FlushInternalSingleBatch(events, serializedEvents).Wait(); + FlushInternalSingleBatch(events, serializedEventsBuilder).Wait(); }); } else { this.activePostTask = this.activePostTask.ContinueWith((_) => { - FlushInternalSingleBatch(events, serializedEvents).Wait(); + FlushInternalSingleBatch(events, serializedEventsBuilder).Wait(); }); } } private Task FlushInternalSingleBatch( List events, - String serializedEvents) + IBuffer serializedEventsBuilder) { - Task task = PostEvents(events, serializedEvents); + Task task = PostEvents(events, serializedEventsBuilder); task.ContinueWith((_) => { Interlocked.Decrement(ref activeAsyncTasksCount); @@ -376,7 +383,7 @@ private Task FlushInternalSingleBatch( private async Task PostEvents( List events, - String serializedEvents) + IBuffer serializedEventsBuilder) { // encode data HttpResponseMessage response = null; @@ -385,18 +392,26 @@ private async Task PostEvents( try { // post data - HttpEventCollectorHandler next = (t, e) => + HttpEventCollectorHandler next = async (t, e) => { - HttpContent content = new StringContent(serializedEvents, Encoding.UTF8, HttpContentTypeMedia); - return httpClient.PostAsync(httpEventCollectorEndpointUri, content); + using (var content = serializedEventsBuilder.BuildHttpContent(HttpContentTypeMedia)) + { + var postResult = await httpClient.PostAsync(httpEventCollectorEndpointUri, content); + return postResult; + } }; HttpEventCollectorHandler postEvents = (t, e) => { - return middleware == null ? - next(t, e) : middleware(t, e, next); + return middleware == null ? next(t, e) : middleware(t, e, next); }; response = await postEvents(token, events); responseCode = response.StatusCode; + + if (response.IsSuccessStatusCode) + { + serializedEventsBuilder.Dispose(); + } + if (responseCode != HttpStatusCode.OK && response.Content != null) { // record server reply @@ -416,7 +431,7 @@ private async Task PostEvents( OnError(e); } catch (Exception e) - { + { OnError(new HttpEventCollectorException( code: responseCode, webException: e, @@ -425,6 +440,7 @@ private async Task PostEvents( events: events )); } + return responseCode; } @@ -436,6 +452,7 @@ private void OnTimer(object state) #region HttpClientHandler.IDispose private bool disposed = false; + private readonly BufferMode bufferMode; public void Dispose() { diff --git a/src/Splunk.Logging.Common/Splunk.Logging.Common.csproj b/src/Splunk.Logging.Common/Splunk.Logging.Common.csproj index 4521ca8..9853e81 100644 --- a/src/Splunk.Logging.Common/Splunk.Logging.Common.csproj +++ b/src/Splunk.Logging.Common/Splunk.Logging.Common.csproj @@ -7,6 +7,7 @@ 1.7.2.0 1.7.2.0 1.7.2 + Splunk.Logging From 2602836b31da502c0f74c877323e43ab9961920d Mon Sep 17 00:00:00 2001 From: Graeme Foster Date: Mon, 3 Apr 2023 20:01:04 +0800 Subject: [PATCH 06/11] Added some alternative buffers each which has a sweet spot compared to the original. --- .../.idea.Splunk.Logging/.idea/workspace.xml | 61 +++++++++++++--- .../BatchBuffers/IBuffer.cs | 2 +- .../BatchBuffers/PushStreamInMemoryBuffer.cs | 72 +++++++++++++++++++ ...rBatchBuffer.cs => StringBuilderBuffer.cs} | 18 ++--- .../StringBuilderOriginalBatchBuffer.cs | 33 +++++++++ ...hBuffer.cs => TemporaryFileBatchBuffer.cs} | 18 +++-- .../HttpEventCollectorSender.cs | 9 +-- standalone-test/Program.cs | 51 ++++++++----- standalone-test/standalone-test.csproj | 2 +- test/unit-tests/TestHttpEventCollector.cs | 3 +- 10 files changed, 217 insertions(+), 52 deletions(-) create mode 100644 src/Splunk.Logging.Common/BatchBuffers/PushStreamInMemoryBuffer.cs rename src/Splunk.Logging.Common/BatchBuffers/{StringBuilderBatchBuffer.cs => StringBuilderBuffer.cs} (58%) create mode 100644 src/Splunk.Logging.Common/BatchBuffers/StringBuilderOriginalBatchBuffer.cs rename src/Splunk.Logging.Common/BatchBuffers/{MemoryStreamBatchBuffer.cs => TemporaryFileBatchBuffer.cs} (71%) diff --git a/.idea/.idea.Splunk.Logging/.idea/workspace.xml b/.idea/.idea.Splunk.Logging/.idea/workspace.xml index 0728e45..b41fcdd 100644 --- a/.idea/.idea.Splunk.Logging/.idea/workspace.xml +++ b/.idea/.idea.Splunk.Logging/.idea/workspace.xml @@ -5,14 +5,16 @@ standalone-test/standalone-test.csproj - - - - + + + + + + - + + + + - + + - + + - + + + + + + + + @@ -139,6 +160,26 @@ + + + + + file://$PROJECT_DIR$/src/Splunk.Logging.Common/BatchBuffers/TemporaryFileBatchBuffer.cs + 55 + + + + + + + + + \ No newline at end of file diff --git a/src/Splunk.Logging.Common/BatchBuffers/IBuffer.cs b/src/Splunk.Logging.Common/BatchBuffers/IBuffer.cs index 35895b7..979fd9d 100644 --- a/src/Splunk.Logging.Common/BatchBuffers/IBuffer.cs +++ b/src/Splunk.Logging.Common/BatchBuffers/IBuffer.cs @@ -1,6 +1,5 @@ using System; using System.Net.Http; -using Newtonsoft.Json; namespace Splunk.Logging.BatchBuffers { @@ -9,5 +8,6 @@ public interface IBuffer : IDisposable void Append(HttpEventCollectorEventInfo serializedEventInfo); long Length { get; } HttpContent BuildHttpContent(string mediaType); + void SupportOriginalBehaviour(); } } \ No newline at end of file diff --git a/src/Splunk.Logging.Common/BatchBuffers/PushStreamInMemoryBuffer.cs b/src/Splunk.Logging.Common/BatchBuffers/PushStreamInMemoryBuffer.cs new file mode 100644 index 0000000..5224418 --- /dev/null +++ b/src/Splunk.Logging.Common/BatchBuffers/PushStreamInMemoryBuffer.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; + +namespace Splunk.Logging.BatchBuffers +{ + public class PushStreamInMemoryBuffer : IBuffer + { + private readonly List events; + + public PushStreamInMemoryBuffer(List events) + { + this.events = events; + } + + public void Append(HttpEventCollectorEventInfo serializedEventInfo) + { + } + + public long Length => + events.Count * 3 * 1024; //assume 3kb for a log entry. Not ideal, but we can get more finessed if we want. + + public HttpContent BuildHttpContent(string mediaType) + { + return new PushStreamContent(async s => + { + foreach (var evt in this.events) + { + var entry = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(evt)); + await s.WriteAsync(entry, 0, entry.Length); + } + }, mediaType); + } + + public void SupportOriginalBehaviour() + { + } + + public void Dispose() + { + } + + private class PushStreamContent : HttpContent + { + private readonly Func writeContent; + + public PushStreamContent(Func writeContent, string mediaType) + { + this.writeContent = writeContent; + Headers.ContentType = new MediaTypeHeaderValue(mediaType); + } + + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext context) + { + await writeContent(stream); + await stream.FlushAsync(); + } + + protected override bool TryComputeLength(out long length) + { + length = -1; + return false; + } + } + } +} \ No newline at end of file diff --git a/src/Splunk.Logging.Common/BatchBuffers/StringBuilderBatchBuffer.cs b/src/Splunk.Logging.Common/BatchBuffers/StringBuilderBuffer.cs similarity index 58% rename from src/Splunk.Logging.Common/BatchBuffers/StringBuilderBatchBuffer.cs rename to src/Splunk.Logging.Common/BatchBuffers/StringBuilderBuffer.cs index e502e93..9560cee 100644 --- a/src/Splunk.Logging.Common/BatchBuffers/StringBuilderBatchBuffer.cs +++ b/src/Splunk.Logging.Common/BatchBuffers/StringBuilderBuffer.cs @@ -1,26 +1,16 @@ -using System.IO; using System.Net.Http; using System.Text; using Newtonsoft.Json; namespace Splunk.Logging.BatchBuffers { - public class StringBuilderBatchBuffer : IBuffer + public class StringBuilderBuffer : IBuffer { private readonly StringBuilder builder = new StringBuilder(); - private readonly StringWriter writer; - private readonly JsonSerializer serializer; - - public StringBuilderBatchBuffer() - { - writer = new StringWriter(builder); - serializer = JsonSerializer.Create(); - } public void Append(HttpEventCollectorEventInfo serializedEventInfo) { - serializer.Serialize(writer, serializedEventInfo); - writer.Flush(); + builder.Append(JsonConvert.SerializeObject(serializedEventInfo)); } public long Length => builder.Length; @@ -30,6 +20,10 @@ public HttpContent BuildHttpContent(string mediaType) return new StringContent(builder.ToString(), Encoding.UTF8, mediaType); } + public void SupportOriginalBehaviour() + { + } + public void Dispose() { } diff --git a/src/Splunk.Logging.Common/BatchBuffers/StringBuilderOriginalBatchBuffer.cs b/src/Splunk.Logging.Common/BatchBuffers/StringBuilderOriginalBatchBuffer.cs new file mode 100644 index 0000000..78bec61 --- /dev/null +++ b/src/Splunk.Logging.Common/BatchBuffers/StringBuilderOriginalBatchBuffer.cs @@ -0,0 +1,33 @@ +using System.Net.Http; +using System.Text; +using Newtonsoft.Json; + +namespace Splunk.Logging.BatchBuffers +{ + public class StringBuilderOriginalBatchBuffer : IBuffer + { + private readonly StringBuilder builder = new StringBuilder(); + private string serializedEvents; + + public void Append(HttpEventCollectorEventInfo serializedEventInfo) + { + builder.Append(JsonConvert.SerializeObject(serializedEventInfo)); + } + + public long Length => builder.Length; + + public HttpContent BuildHttpContent(string mediaType) + { + return new StringContent(serializedEvents, Encoding.UTF8, mediaType); + } + + public void SupportOriginalBehaviour() + { + this.serializedEvents = builder.ToString(); + } + + public void Dispose() + { + } + } +} \ No newline at end of file diff --git a/src/Splunk.Logging.Common/BatchBuffers/MemoryStreamBatchBuffer.cs b/src/Splunk.Logging.Common/BatchBuffers/TemporaryFileBatchBuffer.cs similarity index 71% rename from src/Splunk.Logging.Common/BatchBuffers/MemoryStreamBatchBuffer.cs rename to src/Splunk.Logging.Common/BatchBuffers/TemporaryFileBatchBuffer.cs index c2d8fb0..6e692b2 100644 --- a/src/Splunk.Logging.Common/BatchBuffers/MemoryStreamBatchBuffer.cs +++ b/src/Splunk.Logging.Common/BatchBuffers/TemporaryFileBatchBuffer.cs @@ -2,23 +2,23 @@ using System.IO; using System.Net.Http; using System.Net.Http.Headers; -using System.Text; using Newtonsoft.Json; namespace Splunk.Logging.BatchBuffers { - public class MemoryStreamBatchBuffer : IBuffer + public class TemporaryFileBatchBuffer : IBuffer { + private static readonly string tempDir = Path.GetTempPath(); private readonly string filePath; private readonly JsonSerializer serializer; private readonly TextWriter writer; private readonly FileStream fileStream; - public MemoryStreamBatchBuffer() + public TemporaryFileBatchBuffer() { - filePath = Path.GetTempFileName(); + filePath = Path.GetFileName(Path.GetTempFileName()); serializer = JsonSerializer.Create(); - fileStream = File.OpenWrite(filePath); + fileStream = File.OpenWrite($"{tempDir}{filePath}"); writer = new StreamWriter(fileStream); } @@ -34,7 +34,7 @@ public HttpContent BuildHttpContent(string mediaType) { writer.Flush(); writer.Close(); - return new StreamContent(File.OpenRead(filePath)) + return new StreamContent(File.OpenRead($"{tempDir}{filePath}")) { Headers = { @@ -43,13 +43,17 @@ public HttpContent BuildHttpContent(string mediaType) }; } + public void SupportOriginalBehaviour() + { + } + public void Dispose() { writer?.Dispose(); fileStream?.Dispose(); try { - File.Delete(filePath); + File.Delete($"{tempDir}{filePath}"); } catch (Exception) { diff --git a/src/Splunk.Logging.Common/HttpEventCollectorSender.cs b/src/Splunk.Logging.Common/HttpEventCollectorSender.cs index 47b205e..6ac9750 100644 --- a/src/Splunk.Logging.Common/HttpEventCollectorSender.cs +++ b/src/Splunk.Logging.Common/HttpEventCollectorSender.cs @@ -108,8 +108,10 @@ public enum SendMode /// public enum BufferMode { - InMemory, - TempFiles + StringBuilderBuffer, + PushStreamInMemoryBuffer, + OriginalBuffer, + TempFilesBuffer }; private const string HttpContentTypeMedia = "application/json"; @@ -162,7 +164,7 @@ public HttpEventCollectorSender( int batchInterval, int batchSizeBytes, int batchSizeCount, HttpEventCollectorMiddleware middleware, HttpEventCollectorFormatter formatter = null, - BufferMode bufferMode = BufferMode.InMemory) + BufferMode bufferMode = BufferMode.OriginalBuffer) { this.serializer = new JsonSerializer(); serializer.NullValueHandling = NullValueHandling.Ignore; @@ -306,7 +308,6 @@ public Task FlushAsync() task.Start(); return task; } - /// /// Flush all batched events immediately. diff --git a/standalone-test/Program.cs b/standalone-test/Program.cs index 1d451e7..71ae277 100644 --- a/standalone-test/Program.cs +++ b/standalone-test/Program.cs @@ -1,6 +1,7 @@ using Splunk.Logging; using System; using System.Net; +using System.Threading.Tasks; namespace standalone_test { @@ -23,23 +24,41 @@ static async Task DoIt() }; var middleware = new HttpEventCollectorResendMiddleware(100); - var ecSender = new HttpEventCollectorSender(new Uri("https://localhost:8088"), "92A93306-354C-46A5-9790-055C688EB0C4", null, HttpEventCollectorSender.SendMode.Sequential, 5000, 0, 0, middleware.Plugin); + var ecSender = new HttpEventCollectorSender( + new Uri("https://localhost:8088"), + "92A93306-354C-46A5-9790-055C688EB0C4", + null, + HttpEventCollectorSender.SendMode.Sequential, + 1000, + 0, + 1000, + middleware.Plugin, + bufferMode: HttpEventCollectorSender.BufferMode.OriginalBuffer); + ecSender.OnError += o => Console.WriteLine(o.Message); - - for (var i = 0; i < 50000; i++) + + var rnd = new Random(Environment.TickCount); + for (var i = 0; i < 5000; i++) { - ecSender.Send(DateTime.UtcNow.AddDays(-1), Guid.NewGuid().ToString(), "INFO", null, - new - { - Foo = "Bar", test2 = "Testit2", time = ConvertToEpoch(DateTime.UtcNow.AddHours(-2)).ToString(), - anotherkey = "anothervalue" - }); - ecSender.Send(Guid.NewGuid().ToString(), "INFO", null, - new - { - Foo = "Bar", test2 = "Testit2", time = ConvertToEpoch(DateTime.UtcNow.AddHours(-2)).ToString(), - anotherkey = "anothervalue!!" - }); + for (var j = 0; j < rnd.Next(30, 50); j++) + { + ecSender.Send(DateTime.UtcNow.AddDays(-1), Guid.NewGuid().ToString(), "INFO", null, + new + { + Foo = "Bar", test2 = "Testit2", + time = ConvertToEpoch(DateTime.UtcNow.AddHours(-2)).ToString(), + anotherkey = "anothervalue" + }); + ecSender.Send(Guid.NewGuid().ToString(), "INFO", null, + new + { + Foo = "Bar", test2 = "Testit2", + time = ConvertToEpoch(DateTime.UtcNow.AddHours(-2)).ToString(), + anotherkey = "anothervalue!!" + }); + } + + Console.WriteLine(i.ToString()); } await ecSender.FlushAsync(); @@ -57,4 +76,4 @@ private static void EcSender_OnError(HttpEventCollectorException obj) throw new NotImplementedException(); } } -} +} \ No newline at end of file diff --git a/standalone-test/standalone-test.csproj b/standalone-test/standalone-test.csproj index 43bc1b4..4f26ed8 100644 --- a/standalone-test/standalone-test.csproj +++ b/standalone-test/standalone-test.csproj @@ -9,7 +9,7 @@ Properties standalone_test standalone-test - v4.8.1 + v4.5.2 512 true diff --git a/test/unit-tests/TestHttpEventCollector.cs b/test/unit-tests/TestHttpEventCollector.cs index 5e179e7..4e2af4e 100644 --- a/test/unit-tests/TestHttpEventCollector.cs +++ b/test/unit-tests/TestHttpEventCollector.cs @@ -9,6 +9,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Newtonsoft.Json; using Xunit; namespace Splunk.Logging @@ -394,7 +395,7 @@ public void HttpEventCollectorBatchingSizeTest() // estimate serialized event size HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(null, TraceEventType.Information.ToString(), "info ?", null, null); - int size = HttpEventCollectorSender.SerializeEventInfo(ei).Length; + int size = JsonConvert.SerializeObject(ei).Length; var trace = Trace( handler: (token, events) => From 88e1e0e2ed71d849d15f9672e5e1fdd7a5513037 Mon Sep 17 00:00:00 2001 From: Graeme Foster Date: Mon, 3 Apr 2023 20:11:48 +0800 Subject: [PATCH 07/11] Tests running --- .../.idea.Splunk.Logging/.idea/workspace.xml | 64 ++++++++++++++++--- .../HttpEventCollectorSender.cs | 16 +++-- test/unit-tests/TestHttpEventCollector.cs | 10 +-- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/.idea/.idea.Splunk.Logging/.idea/workspace.xml b/.idea/.idea.Splunk.Logging/.idea/workspace.xml index b41fcdd..6fe4a8a 100644 --- a/.idea/.idea.Splunk.Logging/.idea/workspace.xml +++ b/.idea/.idea.Splunk.Logging/.idea/workspace.xml @@ -5,16 +5,9 @@ standalone-test/standalone-test.csproj - - - + - - - - - @@ -161,7 +165,8 @@ - @@ -179,6 +184,45 @@ diff --git a/src/Splunk.Logging.Common/HttpEventCollectorSender.cs b/src/Splunk.Logging.Common/HttpEventCollectorSender.cs index 6ac9750..244b400 100644 --- a/src/Splunk.Logging.Common/HttpEventCollectorSender.cs +++ b/src/Splunk.Logging.Common/HttpEventCollectorSender.cs @@ -329,6 +329,11 @@ private void FlushInternal() return; // there is nothing to send // flush events according to the system operation mode + this.serializedEventsBatch.SupportOriginalBehaviour(); + + // post data and update tasks counter + Interlocked.Increment(ref activeAsyncTasksCount); + if (this.sendMode == SendMode.Sequential) FlushInternalSequentialMode(this.eventsBatch, this.serializedEventsBatch); else @@ -342,17 +347,20 @@ private void FlushInternal() private void BuildBuffers() { - this.serializedEventsBatch = bufferMode == BufferMode.InMemory ? (IBuffer)new StringBuilderBatchBuffer() : new MemoryStreamBatchBuffer(); this.eventsBatch = new List(); + this.serializedEventsBatch = bufferMode == BufferMode.StringBuilderBuffer + ? new StringBuilderBuffer() + : bufferMode == BufferMode.OriginalBuffer + ? new StringBuilderOriginalBatchBuffer() + : bufferMode == BufferMode.PushStreamInMemoryBuffer + ? (IBuffer)new PushStreamInMemoryBuffer(eventsBatch) + : new TemporaryFileBatchBuffer(); } private void FlushInternalSequentialMode( List events, IBuffer serializedEventsBuilder) { - // post data and update tasks counter - Interlocked.Increment(ref activeAsyncTasksCount); - // post events only after the current post task is done if (this.activePostTask == null) { diff --git a/test/unit-tests/TestHttpEventCollector.cs b/test/unit-tests/TestHttpEventCollector.cs index 4e2af4e..986ea78 100644 --- a/test/unit-tests/TestHttpEventCollector.cs +++ b/test/unit-tests/TestHttpEventCollector.cs @@ -16,7 +16,7 @@ namespace Splunk.Logging { public class TestHttpEventCollector { - private readonly Uri uri = new Uri("http://localhost:8089"); // a dummy uri + private readonly Uri uri = new Uri("http://localhost:5678"); // a dummy uri private const string token = "TOKEN-GUID"; #region Trace listener interceptor that replaces a real Splunk server for testing. @@ -272,7 +272,7 @@ public void HttpEventCollectorCoreTest() [Trait("integration-tests", "Splunk.Logging.HttpEventCollectorSerializationTest")] [Fact] - public void HttpEventCollectorSerializationTest() + public async Task HttpEventCollectorSerializationTest() { Func, Response> noopHandler = (token, events) => { @@ -338,7 +338,7 @@ public void HttpEventCollectorSerializationTest() trace.TraceInformation("hello2"); trace.TraceInformation("hello3"); - (trace.Listeners[trace.Listeners.Count - 1] as HttpEventCollectorTraceListener).FlushAsync().RunSynchronously(); + await (trace.Listeners[trace.Listeners.Count - 1] as HttpEventCollectorTraceListener).FlushAsync(); trace.Close(); Assert.Equal(numFormattedEvents, 3); @@ -558,7 +558,7 @@ public void HttpEventCollectorSinkBatchingTest() [Trait("integration-tests", "Splunk.Logging.HttpEventCollectorAsyncFlushTest")] [Fact] - public void HttpEventCollectorAsyncFlushTest() + public async Task HttpEventCollectorAsyncFlushTest() { var trace = Trace( handler: (token, events) => @@ -577,7 +577,7 @@ public void HttpEventCollectorAsyncFlushTest() trace.TraceInformation("info 3"); trace.TraceInformation("info 4"); HttpEventCollectorTraceListener listener = trace.Listeners[1] as HttpEventCollectorTraceListener; - listener.FlushAsync().RunSynchronously(); + await listener.FlushAsync(); } [Trait("integration-tests", "Splunk.Logging.HttpEventCollectorSeqModeTest")] From 8ebc3af855a354c6d415d16419f1341b2633c163 Mon Sep 17 00:00:00 2001 From: Graeme Foster Date: Mon, 3 Apr 2023 21:27:54 +0800 Subject: [PATCH 08/11] Serialize as they come in into a list. Minimal LOH allocation for messages < 85kb. --- .../.idea.Splunk.Logging/.idea/workspace.xml | 4 ++-- .../BatchBuffers/PushStreamInMemoryBuffer.cs | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.idea/.idea.Splunk.Logging/.idea/workspace.xml b/.idea/.idea.Splunk.Logging/.idea/workspace.xml index 6fe4a8a..7a23ee3 100644 --- a/.idea/.idea.Splunk.Logging/.idea/workspace.xml +++ b/.idea/.idea.Splunk.Logging/.idea/workspace.xml @@ -7,8 +7,8 @@ - - + + - + + + + - { + "keyToString": { + "RunOnceActivity.OpenProjectViewOnStart": "true", + "RunOnceActivity.ShowReadmeOnStart": "true", + "WebServerToolWindowFactoryState": "false", + "XThreadsFramesViewSplitterKey": "0.53677934", + "nodejs_package_manager_path": "npm", + "settings.editor.selected.configurable": "SolutionBuilderGeneralOptionsPage", + "vue.rearranger.settings.migration": "true" }, - "keyToStringList": { - "rider.external.source.directories": [ - "C:\\Users\\graemefoster\\AppData\\Roaming\\JetBrains\\Rider2022.3\\resharper-host\\DecompilerCache", - "C:\\Users\\graemefoster\\AppData\\Roaming\\JetBrains\\Rider2022.3\\resharper-host\\SourcesCache", - "C:\\Users\\graemefoster\\AppData\\Local\\Symbols\\src" + "keyToStringList": { + "rider.external.source.directories": [ + "C:\\Users\\graemefoster\\AppData\\Roaming\\JetBrains\\Rider2022.3\\resharper-host\\DecompilerCache", + "C:\\Users\\graemefoster\\AppData\\Roaming\\JetBrains\\Rider2022.3\\resharper-host\\SourcesCache", + "C:\\Users\\graemefoster\\AppData\\Local\\Symbols\\src" ] } -}]]> +}