From 487a8a4fd9604f5d2720ca9b97b98904c739152c Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Tue, 3 Mar 2026 19:30:24 +0000 Subject: [PATCH 01/14] Fix throttleLatest for empty flow --- .../datasourcex/util/flow/Throttle.kt | 2 +- .../datasourcex/util/flow/ThrottleKtTest.kt | 69 +++++++++++++++++-- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Throttle.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Throttle.kt index c89226d..765b614 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Throttle.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Throttle.kt @@ -22,7 +22,7 @@ import kotlinx.coroutines.selects.whileSelect fun Flow.throttleLatest(timeMillis: Long): Flow = channelFlow { val receiveChannel = produce { collect { send(it) } } - send(receiveChannel.receive()) + send(receiveChannel.receiveCatching().getOrNull() ?: return@channelFlow) var delayJob: Job? = launch { delay(timeMillis) } diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/ThrottleKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/ThrottleKtTest.kt index d215a7b..5a2197b 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/ThrottleKtTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/ThrottleKtTest.kt @@ -5,10 +5,13 @@ package com.caplin.integration.datasourcex.util.flow import app.cash.turbine.test import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.equals.shouldBeEqual +import io.kotest.matchers.shouldBe import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.delay import kotlinx.coroutines.flow.consumeAsFlow +import kotlinx.coroutines.flow.emptyFlow +import kotlinx.coroutines.flow.flow internal class ThrottleKtTest : FunSpec({ @@ -16,20 +19,20 @@ internal class ThrottleKtTest : val flow = Channel() flow.consumeAsFlow().throttleLatest(100).test { flow.send("A") - expectMostRecentItem() shouldBeEqual "A" + awaitItem() shouldBeEqual "A" flow.send("B") delay(99) expectNoEvents() delay(1) - expectMostRecentItem() shouldBeEqual "B" + awaitItem() shouldBeEqual "B" flow.send("X") delay(50) flow.send("Y") delay(50) - expectMostRecentItem() shouldBeEqual "Y" + awaitItem() shouldBeEqual "Y" delay(100) flow.send("Z") - expectMostRecentItem() shouldBeEqual "Z" + awaitItem() shouldBeEqual "Z" flow.close() awaitComplete() } @@ -39,12 +42,66 @@ internal class ThrottleKtTest : val flow = Channel() flow.consumeAsFlow().throttleLatest(100).test { flow.send("A") - expectMostRecentItem() shouldBeEqual "A" + awaitItem() shouldBeEqual "A" flow.send("B") flow.send("C") expectNoEvents() flow.close() - expectMostRecentItem() shouldBeEqual "C" + awaitItem() shouldBeEqual "C" + awaitComplete() + } + } + + test("Throttle latest with empty flow") { + emptyFlow().throttleLatest(100).test { awaitComplete() } + } + + test("Throttle latest with error") { + flow { + emit("A") + delay(10) + throw RuntimeException("Failure") + } + .throttleLatest(100) + .test { + awaitItem() shouldBeEqual "A" + awaitError().message shouldBe "Failure" + } + } + + test("Throttle latest drops intermediate items") { + val flow = Channel() + flow.consumeAsFlow().throttleLatest(100).test { + flow.send("1") + awaitItem() shouldBeEqual "1" + + flow.send("2") + flow.send("3") + flow.send("4") + + delay(150) + awaitItem() shouldBeEqual "4" + + flow.close() + awaitComplete() + } + } + + test("Throttle latest slow upstream") { + val flow = Channel() + flow.consumeAsFlow().throttleLatest(100).test { + flow.send("A") + awaitItem() shouldBeEqual "A" + + delay(150) + flow.send("B") + awaitItem() shouldBeEqual "B" + + delay(150) + flow.send("C") + awaitItem() shouldBeEqual "C" + + flow.close() awaitComplete() } } From 7f59f07d17bdc6e6fbd347ed49f8b6b2e76bcdf1 Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Tue, 3 Mar 2026 20:01:13 +0000 Subject: [PATCH 02/14] ThrottleLatest performance improvements --- .../datasourcex/util/flow/Throttle.kt | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Throttle.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Throttle.kt index 765b614..64ebf5a 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Throttle.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Throttle.kt @@ -6,10 +6,11 @@ import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.Job import kotlinx.coroutines.channels.onFailure import kotlinx.coroutines.channels.onSuccess -import kotlinx.coroutines.channels.produce +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.produceIn import kotlinx.coroutines.launch import kotlinx.coroutines.selects.whileSelect @@ -19,37 +20,41 @@ import kotlinx.coroutines.selects.whileSelect * immediately. */ @Suppress("UNCHECKED_CAST") -fun Flow.throttleLatest(timeMillis: Long): Flow = channelFlow { - val receiveChannel = produce { collect { send(it) } } +fun Flow.throttleLatest(timeMillis: Long): Flow = flow { + coroutineScope { + val receiveChannel = produceIn(this) - send(receiveChannel.receiveCatching().getOrNull() ?: return@channelFlow) + emit(receiveChannel.receiveCatching().getOrNull() ?: return@coroutineScope) - var delayJob: Job? = launch { delay(timeMillis) } + var delayJob: Job? = launch { delay(timeMillis) } - var queuedEvent: Any? = UNSET - whileSelect { - receiveChannel.onReceiveCatching { channelResult -> - channelResult - .onSuccess { event -> - if (delayJob == null) { - send(event) - delayJob = launch { delay(timeMillis) } - } else queuedEvent = event - } - .onFailure { - queuedEvent?.takeIf { event -> event != UNSET }?.let { event -> send(event as T) } - } - .isSuccess - } - delayJob?.onJoin?.invoke { - delayJob = - if (queuedEvent == UNSET) null - else { - send(queuedEvent as T) - queuedEvent = UNSET - launch { delay(timeMillis) } - } - true + var queuedEvent: Any? = UNSET + whileSelect { + receiveChannel.onReceiveCatching { channelResult -> + channelResult + .onSuccess { event -> + if (delayJob == null) { + emit(event) + delayJob = launch { delay(timeMillis) } + } else queuedEvent = event + } + .onFailure { + if (queuedEvent !== UNSET) { + emit(queuedEvent as T) + } + } + .isSuccess + } + delayJob?.onJoin?.invoke { + delayJob = + if (queuedEvent === UNSET) null + else { + emit(queuedEvent as T) + queuedEvent = UNSET + launch { delay(timeMillis) } + } + true + } } } } From 2a77484ab18a1b544856fba86206cbb1704033fe Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Tue, 3 Mar 2026 21:39:28 +0000 Subject: [PATCH 03/14] More improvements --- .../datasourcex/util/flow/Buffer.kt | 14 +- .../datasourcex/util/flow/Demultiplex.kt | 2 +- .../datasourcex/util/flow/MapEvent.kt | 82 +++--- .../datasourcex/util/flow/SetEvent.kt | 4 +- .../datasourcex/util/flow/BufferKtTest.kt | 39 ++- .../datasourcex/util/flow/CastKtTest.kt | 31 +++ .../datasourcex/util/flow/CommonKtTest.kt | 7 + .../util/flow/CompletingSharedFlowTest.kt | 246 +++++++++--------- .../util/flow/DemultiplexKtTest.kt | 50 ++++ .../datasourcex/util/flow/FlowMapTest.kt | 18 ++ .../datasourcex/util/flow/MapEventKtTest.kt | 72 +++++ .../datasourcex/util/flow/RetryKtTest.kt | 30 +++ .../datasourcex/util/flow/SetEventKtTest.kt | 46 ++++ .../datasourcex/util/flow/TimeoutKtTest.kt | 42 +++ 14 files changed, 522 insertions(+), 161 deletions(-) create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CastKtTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CommonKtTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/DemultiplexKtTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/MapEventKtTest.kt diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Buffer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Buffer.kt index 1394d33..fbee073 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Buffer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Buffer.kt @@ -11,6 +11,7 @@ import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.produceIn import kotlinx.coroutines.selects.onTimeout import kotlinx.coroutines.selects.whileSelect +import kotlinx.coroutines.yield /** * Buffers all elements emitted until there is a period of no emissions greater than @@ -18,7 +19,7 @@ import kotlinx.coroutines.selects.whileSelect * * If the upstream [Flow] completes, any remaining elements are emitted immediately. */ -fun Flow.bufferingDebounce(timeoutMillis: Long): Flow> = channelFlow { +fun Flow.bufferingDebounce(timeoutMillis: Long): Flow> = channelFlow { val itemChannel = produceIn(this) var bufferedItems = mutableListOf() whileSelect { @@ -31,7 +32,14 @@ fun Flow.bufferingDebounce(timeoutMillis: Long): Flow> = c itemChannel.onReceiveCatching { result -> result .onSuccess { item -> bufferedItems += item } - .onFailure { if (bufferedItems.isNotEmpty()) send(bufferedItems) } + .onFailure { + if (bufferedItems.isNotEmpty()) { + send(bufferedItems) + bufferedItems = mutableListOf() + yield() + } + it?.let { throw it } + } .isSuccess } } @@ -43,5 +51,5 @@ fun Flow.bufferingDebounce(timeoutMillis: Long): Flow> = c * * If the upstream [Flow] completes, any remaining elements are emitted immediately. */ -fun Flow.bufferingDebounce(timeout: Duration): Flow> = +fun Flow.bufferingDebounce(timeout: Duration): Flow> = bufferingDebounce(timeout.toMillis()) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Demultiplex.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Demultiplex.kt index 60e05d4..71eed27 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Demultiplex.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Demultiplex.kt @@ -45,7 +45,7 @@ fun Flow.demultiplexBy( keyResponseChannels[key] = keyResponseChannel flow { flowProducer(key, keyResponseChannel.consumeAsFlow()) } .onEach { response -> send(response) } - .onCompletion { completedKeysChannel.send(key) } + .onCompletion { completedKeysChannel.trySend(key) } .launchIn(this@callbackFlow) } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/MapEvent.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/MapEvent.kt index d00077c..5639281 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/MapEvent.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/MapEvent.kt @@ -10,11 +10,13 @@ import com.caplin.integration.datasourcex.util.serializable import java.io.Serializable import kotlinx.collections.immutable.persistentMapOf import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.channels.onFailure +import kotlinx.coroutines.channels.onSuccess import kotlinx.coroutines.channels.produce import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.flow -import kotlinx.coroutines.selects.whileSelect +import kotlinx.coroutines.selects.select /** * Events representing a mutation to a [Map]. @@ -209,45 +211,57 @@ fun Flow>.conflateKeys() = channelFlow { fun nextValueToSend(): MapEvent? = unsentValues.entries.firstOrNull()?.value ?: Populated.takeIf { unsentPopulated } - whileSelect { - nextValueToSend()?.let { value -> - onSend(value) { - when (value) { - is EntryEvent -> unsentValues.remove(value.key) - is Populated -> unsentPopulated = false + var upstreamClosed = false + while (true) { + val value = nextValueToSend() + if (upstreamClosed && value == null) break + + select { + if (value != null) { + onSend(value) { + when (value) { + is EntryEvent -> unsentValues.remove(value.key) + is Populated -> unsentPopulated = false + } } - true } - } - upstream.onReceive { event -> - when (event) { - is Populated -> unsentPopulated = true - is EntryEvent -> { - val key = event.key - val oldEvent = unsentValues[key] - if (oldEvent == null) { - unsentValues[key] = event // Nothing to conflate - } else { - val oldValue = oldEvent.oldValue - when (event) { - is Removed -> { - when (oldEvent) { - is Removed -> error("Two Removed events for the same key") - is Upsert -> if (oldValue != null) unsentValues[key] = Removed(key, oldValue) + if (!upstreamClosed) { + upstream.onReceiveCatching { result -> + result + .onSuccess { event -> + when (event) { + is Populated -> unsentPopulated = true + is EntryEvent -> { + val key = event.key + val oldEvent = unsentValues[key] + if (oldEvent == null) { + unsentValues[key] = event // Nothing to conflate + } else { + val oldValue = oldEvent.oldValue + when (event) { + is Removed -> { + when (oldEvent) { + is Removed -> error("Two Removed events for the same key") + is Upsert -> + if (oldValue != null) unsentValues[key] = Removed(key, oldValue) + else unsentValues.remove(key) + } + } + + is Upsert -> { + when (oldEvent) { + is Removed -> unsentValues[key] = Upsert(key, null, event.newValue) + is Upsert -> unsentValues[key] = Upsert(key, oldValue, event.newValue) + } + } + } + } + } } } - - is Upsert -> { - when (oldEvent) { - is Removed -> unsentValues[key] = Upsert(key, null, event.newValue) - is Upsert -> unsentValues[key] = Upsert(key, oldValue, event.newValue) - } - } - } - } + .onFailure { upstreamClosed = true } } } - true } } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt index 7434028..0fb6928 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt @@ -128,14 +128,14 @@ fun Flow>.runningFoldToSet( is Removed -> { set = oldSet.remove(setEvent.value) val changed = oldSet !== set - if (relaxed && !changed) error("Received $setEvent but this did not exist") + if (!relaxed && !changed) error("Received $setEvent but this did not exist") if (changed && (populated || emitPartials)) emit = true } is Insert -> { set = oldSet.add(setEvent.value) val changed = oldSet !== set - if (relaxed && !changed) error("Received $setEvent but this already existed") + if (!relaxed && !changed) error("Received $setEvent but this already existed") if (changed && (populated || emitPartials)) emit = true } diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/BufferKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/BufferKtTest.kt index 7b7a28e..3e6b8f0 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/BufferKtTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/BufferKtTest.kt @@ -5,14 +5,17 @@ package com.caplin.integration.datasourcex.util.flow import app.cash.turbine.test import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.equals.shouldBeEqual +import io.kotest.matchers.types.shouldBeInstanceOf +import java.time.Duration import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.delay import kotlinx.coroutines.flow.consumeAsFlow +import kotlinx.coroutines.flow.receiveAsFlow class BufferKtTest : FunSpec({ - test("Buffered debounce") { + test("Buffered debounce with millis") { val channel = Channel(Channel.BUFFERED) channel.consumeAsFlow().bufferingDebounce(10).test { delay(1) @@ -34,4 +37,38 @@ class BufferKtTest : awaitComplete() } } + + test("Buffered debounce with Duration") { + val channel = Channel(Channel.BUFFERED) + channel.consumeAsFlow().bufferingDebounce(Duration.ofMillis(10)).test { + delay(1) + channel.send("A") + delay(11) + awaitItem() shouldBeEqual listOf("A") + channel.close() + awaitComplete() + } + } + + test("Buffered debounce upstream error propagation") { + val channel = Channel(Channel.BUFFERED) + channel.receiveAsFlow().bufferingDebounce(10).test { + channel.send("A") + delay(50) // Ensure "A" is processed into bufferedItems + channel.close(IllegalArgumentException("test error")) + awaitItem() shouldBeEqual listOf("A") + awaitError().shouldBeInstanceOf() + } + } + + test("Buffered debounce immediate emit on completion") { + val channel = Channel(Channel.BUFFERED) + channel.consumeAsFlow().bufferingDebounce(1000).test { + channel.send("A") + channel.send("B") + channel.close() + awaitItem() shouldBeEqual listOf("A", "B") + awaitComplete() + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CastKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CastKtTest.kt new file mode 100644 index 0000000..333fd90 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CastKtTest.kt @@ -0,0 +1,31 @@ +package com.caplin.integration.datasourcex.util.flow + +import app.cash.turbine.test +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.equals.shouldBeEqual +import kotlinx.coroutines.flow.flowOf + +class CastKtTest : + FunSpec({ + test("cast successfully casts items") { + val flow = flowOf(1, 2, 3) + flow.cast().test { + awaitItem() shouldBeEqual 1 + awaitItem() shouldBeEqual 2 + awaitItem() shouldBeEqual 3 + awaitComplete() + } + } + + test("cast throws ClassCastException on failure when accessed") { + val flow = flowOf(1, "string") + val castFlow = flow.cast() + shouldThrow { + castFlow.collect { + // The cast happens here because 'it' is typed as Int + @Suppress("UNUSED_VARIABLE") val i: Int = it + } + } + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CommonKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CommonKtTest.kt new file mode 100644 index 0000000..92f7b63 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CommonKtTest.kt @@ -0,0 +1,7 @@ +package com.caplin.integration.datasourcex.util.flow + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.nulls.shouldNotBeNull + +class CommonKtTest : + FunSpec({ test("UNSET is a valid non-null instance") { UNSET.shouldNotBeNull() } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CompletingSharedFlowTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CompletingSharedFlowTest.kt index 9520e5d..573bbbc 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CompletingSharedFlowTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/CompletingSharedFlowTest.kt @@ -3,181 +3,187 @@ package com.caplin.integration.datasourcex.util.flow import app.cash.turbine.test +import app.cash.turbine.turbineScope import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion.Completion import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion.Value +import io.kotest.core.coroutines.backgroundScope import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.equals.shouldBeEqual +import io.kotest.matchers.types.shouldBeInstanceOf import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.Job import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.delay import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.receiveAsFlow -import kotlinx.coroutines.plus class CompletingSharedFlowTest : FunSpec({ - test("Share in completing") { - val upstream = Channel>() - + test("Completing shared flow - Value propagation") { + val upstream = Channel>(Channel.BUFFERED) val sharedFlow = upstream .receiveAsFlow() .dematerialize() - .shareInCompleting( - scope = this + Job(), - started = - SharingStarted.WhileSubscribed( - stopTimeoutMillis = 100, - replayExpirationMillis = 0, - ), - replay = 2, - ) + .shareInCompleting(backgroundScope, SharingStarted.Eagerly) - sharedFlow - .onSubscription { emit("INITIAL") } - .test { - val first = this - first.awaitItem() shouldBeEqual "INITIAL" - upstream.send(Value("A")) - first.awaitItem() shouldBeEqual "A" - - upstream.send(Value("B")) - first.awaitItem() shouldBeEqual "B" - - sharedFlow.test { - val second = this - second.awaitItem() shouldBeEqual "A" - second.awaitItem() shouldBeEqual "B" - - upstream.send(Value("C")) - first.awaitItem() shouldBeEqual "C" - second.awaitItem() shouldBeEqual "C" - } - } + sharedFlow.test { + upstream.send(Value("A")) + awaitItem() shouldBeEqual "A" - // The replay buffer and upstream subscription still exists as no upstream complete was - // fired, - // and we are - // within the refCount cooldown + upstream.send(Value("B")) + awaitItem() shouldBeEqual "B" + } + } - sharedFlow - .onSubscription { emit("INITIAL") } - .test { - awaitItem() shouldBeEqual "INITIAL" - awaitItem() shouldBeEqual "B" - awaitItem() shouldBeEqual "C" + test("Completing shared flow - Replay") { + val upstream = Channel>(Channel.BUFFERED) + val sharedFlow = + upstream + .receiveAsFlow() + .dematerialize() + .shareInCompleting(backgroundScope, SharingStarted.Eagerly, 1) - upstream.send(Value("D")) - awaitItem() shouldBeEqual "D" - } + upstream.send(Value("A")) - delay(101) + sharedFlow.test { awaitItem() shouldBeEqual "A" } + } - // The replay buffer was reset and the upstream subscription cancelled as we had no - // subscribers - // for longer than - // the refCount cooldown + test("Completing shared flow - Completion propagation") { + val upstream = Channel>(Channel.BUFFERED) + val sharedFlow = + upstream + .receiveAsFlow() + .dematerialize() + .shareInCompleting(backgroundScope, SharingStarted.Eagerly) sharedFlow.test { - expectNoEvents() - - upstream.send(Value("E")) - awaitItem() shouldBeEqual "E" - upstream.send(Completion()) awaitComplete() } + } - // The upstream complete immediately reset the replay buffer and terminated the upstream - // subscription - // despite the refCount cooldown, and we've now initiated a new upstream subscription + test("Completing shared flow - Error propagation") { + val upstream = Channel>(Channel.BUFFERED) + val sharedFlow = + upstream + .receiveAsFlow() + .dematerialize() + .shareInCompleting(backgroundScope, SharingStarted.Eagerly) sharedFlow.test { - expectNoEvents() + upstream.send(Completion(IllegalArgumentException())) + awaitError().shouldBeInstanceOf() + } + } - upstream.send(Value("X")) - awaitItem() shouldBeEqual "X" + test("Completing shared flow - Upstream restart") { + val upstream = Channel>(Channel.BUFFERED) + val sharedFlow = + upstream + .receiveAsFlow() + .dematerialize() + .shareInCompleting(backgroundScope, SharingStarted.WhileSubscribed()) + + sharedFlow.test { + upstream.send(Value("A")) + awaitItem() shouldBeEqual "A" upstream.send(Completion()) awaitComplete() } + + sharedFlow.test { + upstream.send(Value("B")) + awaitItem() shouldBeEqual "B" + } } test("Shared flow cache") { - val upstream = Channel>() - + val upstream = Channel>(Channel.BUFFERED) val sharedFlowCache = CompletingSharedFlowCache( - scope = this + Job(), - started = SharingStarted.WhileSubscribed(100, 0), - replay = 2, - ) - .loading { upstream.receiveAsFlow().dematerialize() } + scope = backgroundScope, + started = SharingStarted.WhileSubscribed(), + replay = 1, + ) - sharedFlowCache["A"] - .onSubscription { emit("INITIAL") } + sharedFlowCache + .get("A") { upstream.receiveAsFlow().dematerialize() } .test { - upstream.send(Value("A")) - val first = this - first.awaitItem() shouldBeEqual "INITIAL" - first.awaitItem() shouldBeEqual "A" - - upstream.send(Value("B")) - first.awaitItem() shouldBeEqual "B" - - sharedFlowCache["A"].test { - val second = this - second.awaitItem() shouldBeEqual "A" - second.awaitItem() shouldBeEqual "B" - - upstream.send(Value("C")) - first.awaitItem() shouldBeEqual "C" - second.awaitItem() shouldBeEqual "C" - } + expectNoEvents() + + upstream.send(Value("X")) + awaitItem() shouldBeEqual "X" + + upstream.send(Completion()) + awaitComplete() } + } - // The replay buffer and upstream subscription still exists as no upstream complete was - // fired, - // and we are - // within the refCount cooldown + test("Shared flow cache with multiple keys") { + val aUpstream = Channel>(Channel.BUFFERED) + val bUpstream = Channel>(Channel.BUFFERED) - sharedFlowCache["A"].test { - awaitItem() shouldBeEqual "B" - awaitItem() shouldBeEqual "C" + val sharedFlowCache = + CompletingSharedFlowCache( + scope = backgroundScope, + started = SharingStarted.WhileSubscribed(), + replay = 1, + ) + .loading { key -> + when (key) { + "A" -> aUpstream.receiveAsFlow().dematerialize() + "B" -> bUpstream.receiveAsFlow().dematerialize() + else -> error("Invalid key") + } + } - upstream.send(Value("D")) - awaitItem() shouldBeEqual "D" - } + turbineScope { + val aTurbine = sharedFlowCache["A"].testIn(this) + val bTurbine = sharedFlowCache["B"].testIn(this) - delay(101) + aUpstream.send(Value("A1")) + aTurbine.awaitItem() shouldBeEqual "A1" - // The replay buffer was reset and the upstream subscription cancelled as we had no - // subscribers - // for longer than - // the refCount cooldown + bUpstream.send(Value("B1")) + bTurbine.awaitItem() shouldBeEqual "B1" - sharedFlowCache["A"].test { - expectNoEvents() + aUpstream.send(Value("A2")) + aTurbine.awaitItem() shouldBeEqual "A2" - upstream.send(Value("E")) - awaitItem() shouldBeEqual "E" + bUpstream.send(Value("B2")) + bTurbine.awaitItem() shouldBeEqual "B2" - upstream.send(Completion()) - awaitComplete() + aTurbine.expectNoEvents() + bTurbine.expectNoEvents() + + aUpstream.close() + bUpstream.close() + + aTurbine.awaitComplete() + bTurbine.awaitComplete() } + } - // The upstream complete immediately reset the replay buffer and terminated the upstream - // subscription - // despite the refCount cooldown, and we've now initiated a new upstream subscription + test("MutableCompletingSharedFlow basic operations") { + val mutableFlow = MutableCompletingSharedFlow(replay = 1) - sharedFlowCache["A"].test { - expectNoEvents() + mutableFlow.test { + mutableFlow.tryEmit("A") shouldBeEqual true + awaitItem() shouldBeEqual "A" - upstream.send(Value("X")) - awaitItem() shouldBeEqual "X" + mutableFlow.emit("B") + awaitItem() shouldBeEqual "B" - upstream.send(Completion()) + mutableFlow.complete(IllegalArgumentException("Test error")) + awaitError().shouldBeInstanceOf() + } + + // Test normal completion + val mutableFlow2 = MutableCompletingSharedFlow(replay = 1) + mutableFlow2.test { + mutableFlow2.emit("A") + awaitItem() shouldBeEqual "A" + mutableFlow2.complete() awaitComplete() } } diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/DemultiplexKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/DemultiplexKtTest.kt new file mode 100644 index 0000000..fbc7897 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/DemultiplexKtTest.kt @@ -0,0 +1,50 @@ +@file:OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class) + +package com.caplin.integration.datasourcex.util.flow + +import app.cash.turbine.test +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.equals.shouldBeEqual +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.receiveAsFlow + +class DemultiplexKtTest : + FunSpec({ + test("demultiplexBy routes elements by key") { + val channel = Channel(Channel.UNLIMITED) + channel + .receiveAsFlow() + .demultiplexBy( + keySelector = { it.first().toString() }, + flowProducer = { key, flow -> flow.map { "$key-$it" }.collect { emit(it) } }, + ) + .test { + channel.send("Apple") + awaitItem() shouldBeEqual "A-Apple" + channel.send("Banana") + awaitItem() shouldBeEqual "B-Banana" + channel.send("Avocado") + awaitItem() shouldBeEqual "A-Avocado" + cancelAndIgnoreRemainingEvents() + } + } + + test("demultiplexBy ignores null keys") { + val channel = Channel(Channel.UNLIMITED) + channel + .receiveAsFlow() + .demultiplexBy( + keySelector = { if (it.startsWith("A")) "A" else null }, + flowProducer = { key, flow -> flow.map { "$key-$it" }.collect { emit(it) } }, + ) + .test { + channel.send("Apple") + awaitItem() shouldBeEqual "A-Apple" + channel.send("Banana") // ignored + channel.send("Avocado") + awaitItem() shouldBeEqual "A-Avocado" + cancelAndIgnoreRemainingEvents() + } + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt index 538eb50..a998984 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt @@ -186,4 +186,22 @@ class FlowMapTest : awaitItem() shouldBeEqual Upsert("2", "Ax", "Axx") } } + + test("FlowMap putAll and clear") { + val map = mutableFlowMapOf("1" to "A") + + map.asFlow().test { + awaitItem() shouldBeEqual Upsert("1", null, "A") + awaitItem() shouldBeEqual Populated + + map.putAll(mapOf("2" to "B", "3" to "C")) + awaitItem() shouldBeEqual Upsert("2", null, "B") + awaitItem() shouldBeEqual Upsert("3", null, "C") + + map.clear() + awaitItem() shouldBeEqual Removed("1", "A") + awaitItem() shouldBeEqual Removed("2", "B") + awaitItem() shouldBeEqual Removed("3", "C") + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/MapEventKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/MapEventKtTest.kt new file mode 100644 index 0000000..db3da5e --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/MapEventKtTest.kt @@ -0,0 +1,72 @@ +package com.caplin.integration.datasourcex.util.flow + +import app.cash.turbine.test +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Removed +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Upsert +import com.caplin.integration.datasourcex.util.flow.MapEvent.Populated +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.equals.shouldBeEqual +import io.kotest.matchers.maps.shouldContainExactly +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.receiveAsFlow + +class MapEventKtTest : + FunSpec({ + test("runningFoldToMap emits map state") { + flowOf(Upsert("K1", null, "V1"), Upsert("K2", null, "V2"), Populated, Removed("K1", "V1")) + .runningFoldToMap() + .test { + awaitItem() shouldContainExactly mapOf("K1" to "V1", "K2" to "V2") + awaitItem() shouldContainExactly mapOf("K2" to "V2") + awaitComplete() + } + } + + test("runningFoldToMap with partials emits intermediate map states") { + flowOf(Upsert("K1", null, "V1"), Upsert("K2", null, "V2"), Populated, Removed("K1", "V1")) + .runningFoldToMap(emitPartials = true) + .test { + awaitItem() shouldContainExactly emptyMap() + awaitItem() shouldContainExactly mapOf("K1" to "V1") + awaitItem() shouldContainExactly mapOf("K1" to "V1", "K2" to "V2") + awaitItem() shouldContainExactly mapOf("K2" to "V2") + awaitComplete() + } + } + + test("conflateKeys collapses events for same key") { + val channel = Channel>(Channel.UNLIMITED) + channel.trySend(Upsert("K1", null, "V1")) + channel.trySend(Upsert("K2", null, "V2")) + channel.trySend(Upsert("K1", "V1", "V1_NEW")) + channel.trySend(Removed("K2", "V2")) + channel.trySend(Upsert("K3", null, "V3")) + channel.trySend(Populated) + channel.close() + + // We buffer(0) to ensure the conflateKeys actor has a chance to process all items + // from the unlimited channel before the collector starts taking them. + channel.receiveAsFlow().conflateKeys().buffer(0).test { + awaitItem() shouldBeEqual Upsert("K1", null, "V1_NEW") + awaitItem() shouldBeEqual Upsert("K3", null, "V3") + awaitItem() shouldBeEqual Populated + awaitComplete() + } + } + + test("conflateKeys handles Removed correctly") { + val channel = Channel>(Channel.UNLIMITED) + channel.trySend(Upsert("K1", "V0", "V1")) + channel.trySend(Removed("K1", "V1")) + channel.trySend(Populated) + channel.close() + + channel.receiveAsFlow().conflateKeys().buffer(0).test { + awaitItem() shouldBeEqual Removed("K1", "V0") + awaitItem() shouldBeEqual Populated + awaitComplete() + } + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/RetryKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/RetryKtTest.kt index 7571370..2a798ad 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/RetryKtTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/RetryKtTest.kt @@ -7,6 +7,7 @@ import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion.Completion import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion.Value import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.equals.shouldBeEqual +import io.kotest.matchers.shouldBe import java.util.concurrent.atomic.AtomicInteger import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.ExperimentalCoroutinesApi @@ -85,4 +86,33 @@ class RetryKtTest : startCount.get() shouldBeEqual 6 } } + + test("Retry stops when onRetry returns false") { + val startCount = AtomicInteger() + val channel = Channel>(Channel.BUFFERED) + channel + .receiveAsFlow() + .dematerialize() + .onStart { startCount.getAndIncrement() } + .retryWithExponentialBackoff(minMillis = 10, maxMillis = 100) { _, _ -> + startCount.get() < 3 + } + .test { + channel.send(Completion(RuntimeException("fail"))) + // 1st failure: onRetry(1 < 3) is true. Retries. + delay(15) + startCount.get() shouldBeEqual 2 + + channel.send(Completion(RuntimeException("fail"))) + // 2nd failure: onRetry(2 < 3) is true. Retries. + delay(35) + startCount.get() shouldBeEqual 3 + + channel.send(Completion(RuntimeException("fail"))) + // 3rd failure: onRetry(3 < 3) is false. Stops. + + awaitError().message shouldBe "fail" + startCount.get() shouldBeEqual 3 + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/SetEventKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/SetEventKtTest.kt index 7ec8560..76b35c4 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/SetEventKtTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/SetEventKtTest.kt @@ -5,8 +5,10 @@ package com.caplin.integration.datasourcex.util.flow import app.cash.turbine.test import com.caplin.integration.datasourcex.util.flow.SetEvent.EntryEvent.Insert import com.caplin.integration.datasourcex.util.flow.SetEvent.EntryEvent.Removed +import com.caplin.integration.datasourcex.util.flow.SetEvent.Populated import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.equals.shouldBeEqual +import io.kotest.matchers.types.shouldBeInstanceOf import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.consumeAsFlow @@ -15,6 +17,50 @@ import kotlinx.coroutines.flow.flowOf class SetEventKtTest : FunSpec({ + test("toEvents converts sets to events") { + flowOf(setOf("A", "B"), setOf("B", "C")).toEvents().test { + awaitItem() shouldBeEqual Insert("A") + awaitItem() shouldBeEqual Insert("B") + awaitItem() shouldBeEqual Populated + awaitItem() shouldBeEqual Removed("A") + awaitItem() shouldBeEqual Insert("C") + awaitComplete() + } + } + + test("runningFoldToSet folds events back to sets") { + flowOf(Insert("A"), Insert("B"), Populated, Removed("A"), Insert("C")) + .runningFoldToSet() + .test { + awaitItem().toSet() shouldBeEqual setOf("A", "B") + awaitItem().toSet() shouldBeEqual setOf("B") + awaitItem().toSet() shouldBeEqual setOf("B", "C") + awaitComplete() + } + } + + test("runningFoldToSet with emitPartials") { + flowOf(Insert("A"), Insert("B"), Populated).runningFoldToSet(emitPartials = true).test { + awaitItem().toSet() shouldBeEqual emptySet() + awaitItem().toSet() shouldBeEqual setOf("A") + awaitItem().toSet() shouldBeEqual setOf("A", "B") + awaitComplete() + } + } + + test("runningFoldToSet with relaxed = false throws on bad events") { + flowOf( + Insert("A"), + Populated, + Insert("A"), // relaxed = false should throw because it already exists + ) + .runningFoldToSet(relaxed = false) + .test { + awaitItem().toSet() shouldBeEqual setOf("A") + awaitError().shouldBeInstanceOf() + } + } + test("flatMapLatestAndMerge - When upstream completes first, await inner complete") { val setFlow = Channel>() val aFlow = Channel() diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/TimeoutKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/TimeoutKtTest.kt index 09d6eb1..f9b75d9 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/TimeoutKtTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/TimeoutKtTest.kt @@ -6,6 +6,7 @@ import app.cash.turbine.test import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.shouldBe import io.kotest.matchers.types.shouldBeInstanceOf +import java.time.Duration import java.util.concurrent.TimeoutException import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel @@ -24,6 +25,14 @@ internal class TimeoutKtTest : } } + test("Timeout first Duration - Triggers") { + val channel = Channel() + channel.consumeAsFlow().timeoutFirst(Duration.ofMillis(10)).test { + delay(10) + awaitError().shouldBeInstanceOf() + } + } + test("Timeout first - Doesn't trigger") { val channel = Channel() channel.consumeAsFlow().timeoutFirst(10).test { @@ -54,6 +63,16 @@ internal class TimeoutKtTest : } } + test("Timeout first or null Duration - Triggers") { + val channel = Channel() + channel.consumeAsFlow().timeoutFirstOrNull(Duration.ofMillis(10)).test { + delay(10) + expectMostRecentItem() shouldBe null + channel.close() + awaitComplete() + } + } + test("Timeout first or null - Doesn't trigger") { val channel = Channel() channel.consumeAsFlow().timeoutFirstOrNull(10).test { @@ -85,6 +104,29 @@ internal class TimeoutKtTest : } } + test("Timeout first or default Duration - Triggers") { + val channel = Channel() + channel + .consumeAsFlow() + .timeoutFirstOrDefault(Duration.ofMillis(10)) { "X" } + .test { + delay(10) + expectMostRecentItem() shouldBe "X" + channel.close() + awaitComplete() + } + } + + test("Timeout first or default value Duration - Triggers") { + val channel = Channel() + channel.consumeAsFlow().timeoutFirstOrDefault(Duration.ofMillis(10), "X").test { + delay(10) + expectMostRecentItem() shouldBe "X" + channel.close() + awaitComplete() + } + } + test("Timeout first or default - Exception handling") { val channel = Channel() channel From 491111d9f0e2a67d13298276c50f7e1f10222d7e Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Tue, 3 Mar 2026 21:51:57 +0000 Subject: [PATCH 04/14] More performance improvements to FlowMap --- .../datasourcex/util/flow/FlowMap.kt | 86 ++++++++++++------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt index d3ffdd4..c29643c 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt @@ -16,7 +16,6 @@ import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.conflate import kotlinx.coroutines.flow.distinctUntilChanged -import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.map @@ -133,11 +132,13 @@ private class FlowMapImpl(initialMap: PersistentMap) : version = it.state.version emit(it) if (held != null) { - var i = 1 - do { - val next = held?.remove(expectedVersion + i++) - if (next != null) emit(next) - } while (next != null && held?.isNotEmpty() == true) + var nextVersion = version + 1 + while (held?.containsKey(nextVersion) == true) { + val next = held!!.remove(nextVersion)!! + emit(next) + version = next.state.version + nextVersion = version + 1 + } if (held?.isEmpty() == true) held = null } } else if (it.state.version > expectedVersion) { @@ -152,36 +153,57 @@ private class FlowMapImpl(initialMap: PersistentMap) : override fun asFlow(predicate: ((K, V) -> Boolean)?): Flow> = flow { val emittedKeys = if (predicate != null) mutableSetOf() else null - suspend fun processEvents(mapEvents: List>) { - mapEvents.forEach { mapEvent -> - if (emittedKeys == null || predicate == null) emit(mapEvent) - else - when (mapEvent) { - is Removed -> if (emittedKeys.remove(mapEvent.key)) emit(mapEvent) - is Upsert -> - if (predicate(mapEvent.key, mapEvent.newValue)) { - val newValue = - if (emittedKeys.add(mapEvent.key)) - Upsert(mapEvent.key, null, mapEvent.newValue) - else mapEvent - emit(newValue) - } else if (emittedKeys.remove(mapEvent.key)) - emit(Removed(mapEvent.key, mapEvent.oldValue!!)) - - else -> {} - } - } - } - var first = true - orderedSignal.filterNotNull().collect { flowMapEvent -> + orderedSignal.collect { flowMapEvent -> if (first) { - flowMapEvent.state.map.entries - .map { Upsert(it.key, null, it.value) } - .let { processEvents(it) } + val map = flowMapEvent.state.map + if (predicate == null) { + for (entry in map.entries) { + emit(Upsert(entry.key, null, entry.value)) + } + } else { + for (entry in map.entries) { + val key = entry.key + val value = entry.value + if (predicate(key, value)) { + emittedKeys!!.add(key) + emit(Upsert(key, null, value)) + } + } + } emit(Populated) first = false - } else processEvents(flowMapEvent.events) + } else { + val events = flowMapEvent.events + if (predicate == null) { + for (event in events) { + emit(event) + } + } else { + for (event in events) { + when (event) { + is Removed -> { + if (emittedKeys!!.remove(event.key)) emit(event) + } + + is Upsert -> { + val key = event.key + val newValue = event.newValue + if (predicate(key, newValue)) { + val wasEmitted = !emittedKeys!!.add(key) + if (wasEmitted) { + emit(event) + } else { + emit(Upsert(key, null, newValue)) + } + } else if (emittedKeys!!.remove(key)) { + emit(Removed(key, event.oldValue!!)) + } + } + } + } + } + } } } From f6124250343f530c28680e595c02f358663cb445 Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Tue, 3 Mar 2026 21:56:27 +0000 Subject: [PATCH 05/14] More performance improvements to FlowMap --- .../datasourcex/util/flow/FlowMap.kt | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt index c29643c..7eefc82 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt @@ -133,11 +133,12 @@ private class FlowMapImpl(initialMap: PersistentMap) : emit(it) if (held != null) { var nextVersion = version + 1 - while (held?.containsKey(nextVersion) == true) { - val next = held!!.remove(nextVersion)!! + var next: FlowMapEvent? = held?.remove(nextVersion) + while (next != null) { emit(next) version = next.state.version nextVersion = version + 1 + next = held?.remove(nextVersion) } if (held?.isEmpty() == true) held = null } @@ -158,16 +159,16 @@ private class FlowMapImpl(initialMap: PersistentMap) : if (first) { val map = flowMapEvent.state.map if (predicate == null) { - for (entry in map.entries) { + for (entry in map) { emit(Upsert(entry.key, null, entry.value)) } } else { - for (entry in map.entries) { - val key = entry.key - val value = entry.value - if (predicate(key, value)) { - emittedKeys!!.add(key) - emit(Upsert(key, null, value)) + for (entry in map) { + val k = entry.key + val v = entry.value + if (predicate(k, v)) { + emittedKeys!!.add(k) + emit(Upsert(k, null, v)) } } } From 4bfdfb92d87e0eb4022dc702a8aac628742a7d45 Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Wed, 4 Mar 2026 13:42:16 +0000 Subject: [PATCH 06/14] Drop native java serialization support, add fory and jackson serializers, and add benchmarks --- .idea/compiler.xml | 3 + .idea/modules.xml | 8 + gradle/libs.versions.toml | 10 +- util/build.gradle.kts | 15 +- .../datasourcex/util/flow/FlowMapBenchmark.kt | 164 ++++++++++++++++++ .../util/SerializablePersistentMap.kt | 71 -------- .../util/SerializablePersistentSet.kt | 61 ------- .../util/SimpleDataSourceFactory.kt | 2 + .../util/flow/DataSourceJacksonModule.kt | 145 ++++++++++++++++ .../datasourcex/util/flow/FlowMap.kt | 46 +++++ .../util/flow/FlowMapForySupport.kt | 30 ++++ .../datasourcex/util/flow/MapEvent.kt | 14 +- .../datasourcex/util/flow/SetEvent.kt | 10 +- .../datasourcex/util/flow/SimpleMapEvent.kt | 12 +- .../util/flow/ValueOrCompletion.kt | 9 +- .../util/SerializablePersistentMapTest.kt | 37 ---- .../util/SerializablePersistentSetTest.kt | 36 ---- .../util/flow/FlowMapSerializationTest.kt | 92 ++++++++++ .../datasourcex/util/flow/FlowMapTest.kt | 21 +++ 19 files changed, 550 insertions(+), 236 deletions(-) create mode 100644 .idea/modules.xml create mode 100644 util/src/jmh/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapBenchmark.kt delete mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentMap.kt delete mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentSet.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/DataSourceJacksonModule.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt delete mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentMapTest.kt delete mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentSetTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt diff --git a/.idea/compiler.xml b/.idea/compiler.xml index b6f546c..766cef2 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -1,6 +1,9 @@ + + + diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..d8d871f --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 915e5b1..8460ff1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,8 +3,11 @@ springBoot = "3.5.10" kotlin = "1.9.25" # Keep in sync with Spring - https://docs.spring.io/spring-boot/docs/current/reference/html/dependency-versions.html kotlinCollectionsImmutable = "0.3.8" jsonPatch = "1.13" +jmh = "1.37" +jmh-plugin = "0.7.3" kotlinpoet = "2.2.0" ktfmt = "0.61" +fory = "0.15.0" # Caplin Dependencies datasource = "8.0.10-1695-5ddb3798" @@ -28,6 +31,10 @@ vanniktech-maven-publish-plugin = "0.31.0" kotlin-collections-immutable = { module = "org.jetbrains.kotlinx:kotlinx-collections-immutable", version.ref = "kotlinCollectionsImmutable" } json-patch = { module = "com.github.java-json-tools:json-patch", version.ref = "jsonPatch" } kotlinpoet = { module = "com.squareup:kotlinpoet", version.ref = "kotlinpoet" } +fory-core = { module = "org.apache.fory:fory-core", version.ref = "fory" } +fory-kotlin = { module = "org.apache.fory:fory-kotlin", version.ref = "fory" } +jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } +jmh-generator = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" } # Caplin Dependencies datasource = { module = "com.caplin.platform.integration.java:datasource", version.ref = "datasource" } @@ -54,4 +61,5 @@ spring-boot-dependencies = { module = "org.springframework.boot:spring-boot-depe spring-boot-configuration-processor = { module = "org.springframework.boot:spring-boot-configuration-processor", version.ref = "springBoot" } [plugins] -spring-boot = { id = "org.springframework.boot", version.ref = "springBoot" } \ No newline at end of file +spring-boot = { id = "org.springframework.boot", version.ref = "springBoot" } +jmh = { id = "me.champeau.jmh", version.ref = "jmh-plugin" } \ No newline at end of file diff --git a/util/build.gradle.kts b/util/build.gradle.kts index 9fadde4..a40c86e 100644 --- a/util/build.gradle.kts +++ b/util/build.gradle.kts @@ -1,4 +1,7 @@ -plugins { `common-library` } +plugins { + `common-library` + alias(libs.plugins.jmh) +} description = "Utility classes for DataSource extensions" @@ -15,12 +18,22 @@ dependencies { implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") implementation(libs.kotlin.collections.immutable) + compileOnly(libs.fory.core) + compileOnly(libs.fory.kotlin) + testRuntimeOnly("org.slf4j:slf4j-simple") testImplementation("org.springframework:spring-core") // For testing the RegexPathMatcher testImplementation(libs.turbine) testImplementation(libs.kotest.assertions) testImplementation(libs.kotest.runner) + testImplementation(libs.fory.core) + testImplementation(libs.fory.kotlin) + + jmh(libs.jmh.core) + jmh(libs.jmh.generator) } +jmh { duplicateClassesStrategy.set(DuplicatesStrategy.EXCLUDE) } + dokka { dokkaSourceSets.configureEach { includes.from("README.md") } } diff --git a/util/src/jmh/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapBenchmark.kt b/util/src/jmh/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapBenchmark.kt new file mode 100644 index 0000000..81725e5 --- /dev/null +++ b/util/src/jmh/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapBenchmark.kt @@ -0,0 +1,164 @@ +package com.caplin.integration.datasourcex.util.flow + +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Upsert +import com.caplin.integration.datasourcex.util.flow.MapEvent.Populated +import java.util.concurrent.TimeUnit +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.take +import kotlinx.coroutines.flow.takeWhile +import kotlinx.coroutines.runBlocking +import org.openjdk.jmh.annotations.* + +/** + * Benchmarks for [FlowMap] implementation, focusing on mutation performance, lookup efficiency, and + * Flow-based state reconstruction. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +open class FlowMapBenchmark { + + private lateinit var flowMap: MutableFlowMap + + @Setup + fun setup() { + flowMap = mutableFlowMapOf() + } + + /** + * Measures the throughput of single [MutableFlowMap.put] operations on an initially empty map. + */ + @Benchmark + fun putSingle() { + flowMap.put("key", "value") + } + + /** + * Measures the throughput of a [MutableFlowMap.put] followed by a [MutableFlowMap.remove] on the + * same key, exercising the event emission and state update logic. + */ + @Benchmark + fun putAndRemove() { + flowMap.put("key", "value") + flowMap.remove("key") + } + + /** + * Measures the throughput of [MutableFlowMap.putAll] with a small map, which triggers multiple + * events in a single state update. + */ + @Benchmark + fun putAllSmall() { + flowMap.putAll(mapOf("1" to "A", "2" to "B", "3" to "C")) + } + + /** State holder for a [FlowMap] pre-populated with 1,000 entries. */ + @State(Scope.Benchmark) + open class PopulatedFlowMapState { + lateinit var flowMap: MutableFlowMap + + @Setup + fun setup() { + flowMap = mutableFlowMapOf() + repeat(1000) { flowMap.put("key$it", it) } + } + } + + /** State holder for measuring mutation throughput with multiple active subscribers. */ + @State(Scope.Benchmark) + open class ActiveSubscriberState { + @Param("1", "10", "100") var subscriberCount: Int = 0 + + lateinit var flowMap: MutableFlowMap + lateinit var scope: CoroutineScope + + @Setup + fun setup() { + flowMap = mutableFlowMapOf() + // Using Dispatchers.Default for subscribers to simulate real-world processing + scope = CoroutineScope(Dispatchers.Default) + repeat(subscriberCount) { flowMap.asFlow().launchIn(scope) } + } + + @TearDown + fun tearDown() { + scope.cancel() + } + } + + /** + * Measures the throughput of [MutableFlowMap.put] when there are multiple active subscribers + * collecting from the map. This identifies contention or overhead in the event dispatching logic. + */ + @Benchmark + fun putWithSubscribers(state: ActiveSubscriberState) { + state.flowMap.put("key", "value") + } + + /** Measures the throughput of retrieving a value from a large, pre-populated [FlowMap]. */ + @Benchmark + fun getFromLargeMap(state: PopulatedFlowMapState): Int? { + return state.flowMap["key500"] + } + + /** + * Measures the time taken to collect the initial state of a large [FlowMap] via [FlowMap.asFlow]. + */ + @Benchmark + fun asFlowCollection(state: PopulatedFlowMapState) = runBlocking { + state.flowMap + .asFlow() + .takeWhile { it != Populated } + .collect { + // just collect + } + } + + /** + * Measures the time taken to collect the initial state of a large [FlowMap] via + * [FlowMap.asFlowWithState]. This avoids emitting individual Upsert events, making it much + * faster. + */ + @Benchmark + fun asFlowWithStateCollection(state: PopulatedFlowMapState) = runBlocking { + state.flowMap.asFlowWithState().take(1).collect { + // just collect + } + } + + /** + * Measures the time taken to collect the initial state of a large [FlowMap] via [FlowMap.asFlow] + * when a predicate is applied, exercising the filtering logic within the flow. + */ + @Benchmark + fun asFlowWithPredicateCollection(state: PopulatedFlowMapState) = runBlocking { + state.flowMap + .asFlow { _, value -> value % 2 == 0 } + .takeWhile { it != Populated } + .collect { + // just collect + } + } + + /** + * Measures the overhead of reconstructing a [FlowMap] from a stream of events using + * [toFlowMapIn]. + */ + @Benchmark + fun toFlowMapInBenchmark() = runBlocking { + val events = flow { + repeat(100) { emit(Upsert("key$it", null, it)) } + emit(Populated) + } + val scope = CoroutineScope(Dispatchers.Default) + events.toFlowMapIn(scope) + scope.cancel() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentMap.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentMap.kt deleted file mode 100644 index 6ff6c93..0000000 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentMap.kt +++ /dev/null @@ -1,71 +0,0 @@ -package com.caplin.integration.datasourcex.util - -import java.io.ObjectInputStream -import java.io.ObjectOutputStream -import java.io.Serializable -import kotlinx.collections.immutable.ImmutableCollection -import kotlinx.collections.immutable.ImmutableSet -import kotlinx.collections.immutable.PersistentMap -import kotlinx.collections.immutable.mutate -import kotlinx.collections.immutable.persistentMapOf - -fun PersistentMap.serializable(): PersistentMap = SerializablePersistentMap(this) - -internal class SerializablePersistentMap(@Transient private var map: PersistentMap) : - PersistentMap, Serializable { - - private companion object { - private const val serialVersionUID: Long = 5296450722117811345L - } - - override val entries: ImmutableSet> - get() = map.entries - - override val keys: ImmutableSet - get() = map.keys - - override val size: Int - get() = map.size - - override val values: ImmutableCollection - get() = map.values - - override fun builder(): PersistentMap.Builder = map.builder() - - override fun clear(): PersistentMap = map.clear() - - override fun isEmpty(): Boolean = map.isEmpty() - - override fun remove(key: K, value: V): PersistentMap = map.remove(key, value) - - override fun remove(key: K): PersistentMap = map.remove(key) - - override fun putAll(m: Map): PersistentMap = map.putAll(m) - - override fun put(key: K, value: V): PersistentMap = map.put(key, value) - - override fun get(key: K): V? = map[key] - - override fun containsValue(value: V): Boolean = map.containsValue(value) - - override fun containsKey(key: K): Boolean = map.containsKey(key) - - override fun toString(): String = map.toString() - - @Suppress("unused") - private fun writeObject(out: ObjectOutputStream) { - out.writeInt(map.size) - map.forEach { - out.writeObject(it.key) - out.writeObject(it.value) - } - } - - @Suppress("UNCHECKED_CAST") - private fun readObject(input: ObjectInputStream) { - map = - persistentMapOf().mutate { - repeat(input.readInt()) { _ -> it[input.readObject() as K] = input.readObject() as V } - } - } -} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentSet.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentSet.kt deleted file mode 100644 index d5203d8..0000000 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentSet.kt +++ /dev/null @@ -1,61 +0,0 @@ -package com.caplin.integration.datasourcex.util - -import java.io.ObjectInputStream -import java.io.ObjectOutputStream -import java.io.Serializable -import kotlinx.collections.immutable.PersistentSet -import kotlinx.collections.immutable.mutate -import kotlinx.collections.immutable.persistentSetOf - -fun PersistentSet.serializable(): PersistentSet = SerializablePersistentSet(this) - -internal class SerializablePersistentSet(@Transient private var set: PersistentSet) : - PersistentSet, Serializable { - - companion object { - private const val serialVersionUID: Long = 1203740006547290238L - } - - override val size: Int - get() = set.size - - override fun builder(): PersistentSet.Builder = set.builder() - - override fun clear(): PersistentSet = set.clear() - - override fun addAll(elements: Collection): PersistentSet = set.addAll(elements) - - override fun add(element: E): PersistentSet = set.add(element) - - override fun isEmpty(): Boolean = set.isEmpty() - - override fun iterator(): Iterator = set.iterator() - - override fun retainAll(elements: Collection): PersistentSet = set.retainAll(elements) - - override fun removeAll(elements: Collection): PersistentSet = set.removeAll(elements) - - override fun removeAll(predicate: (E) -> Boolean): PersistentSet = set.removeAll(predicate) - - override fun remove(element: E): PersistentSet = set.remove(element) - - override fun containsAll(elements: Collection): Boolean = set.containsAll(elements) - - override fun contains(element: E): Boolean = set.contains(element) - - override fun toString(): String = set.toString() - - @Suppress("unused") - private fun writeObject(out: ObjectOutputStream) { - out.writeInt(set.size) - set.forEach { out.writeObject(it) } - } - - @Suppress("UNCHECKED_CAST") - private fun readObject(input: ObjectInputStream) { - set = - persistentSetOf().mutate { - repeat(input.readInt()) { _ -> it.add(input.readObject() as E) } - } - } -} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt index 85b6641..403867e 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt @@ -2,6 +2,7 @@ package com.caplin.integration.datasourcex.util import com.caplin.datasource.DataSource import com.caplin.datasource.messaging.json.JacksonJsonHandler +import com.caplin.integration.datasourcex.util.flow.registerDataSourceModule import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule @@ -19,6 +20,7 @@ object SimpleDataSourceFactory { jacksonObjectMapper() .configure(WRITE_DATES_AS_TIMESTAMPS, false) .registerModule(JavaTimeModule()) + .registerDataSourceModule() /** * Creates a data source based on the given simple configuration. diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/DataSourceJacksonModule.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/DataSourceJacksonModule.kt new file mode 100644 index 0000000..8b09a5b --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/DataSourceJacksonModule.kt @@ -0,0 +1,145 @@ +package com.caplin.integration.datasourcex.util.flow + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.Module +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.deser.std.StdDeserializer +import com.fasterxml.jackson.databind.module.SimpleModule +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.ser.std.StdSerializer +import kotlinx.collections.immutable.toPersistentMap + +fun ObjectMapper.registerDataSourceModule(): ObjectMapper = registerModule(DataSourceJacksonModule) + +/** + * A Jackson [Module] that provides support for serializing and deserializing [FlowMapStreamEvent] + * and [MapEvent] without requiring annotations on the classes themselves. + */ +object DataSourceJacksonModule : SimpleModule() { + private fun readResolve(): Any = DataSourceJacksonModule + + init { + addSerializer(FlowMapStreamEvent::class.java, FlowMapStreamEventSerializer()) + addDeserializer(FlowMapStreamEvent::class.java, FlowMapStreamEventDeserializer()) + + addSerializer(MapEvent::class.java, MapEventSerializer()) + addDeserializer(MapEvent::class.java, MapEventDeserializer()) + } +} + +private class FlowMapStreamEventSerializer : + StdSerializer>(FlowMapStreamEvent::class.java) { + override fun serialize( + value: FlowMapStreamEvent<*, *>, + gen: JsonGenerator, + provider: SerializerProvider, + ) { + gen.writeStartObject() + when (value) { + is FlowMapStreamEvent.InitialState -> { + gen.writeStringField("type", "initial") + gen.writeFieldName("map") + provider.defaultSerializeValue(value.map, gen) + } + is FlowMapStreamEvent.EventUpdate -> { + gen.writeStringField("type", "update") + gen.writeFieldName("event") + provider.defaultSerializeValue(value.event, gen) + } + } + gen.writeEndObject() + } +} + +private class FlowMapStreamEventDeserializer : + StdDeserializer>(FlowMapStreamEvent::class.java) { + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): FlowMapStreamEvent<*, *> { + val node = p.codec.readTree(p) + val type = + node.get("type")?.asText() + ?: throw JsonMappingException.from(p, "Missing type field for FlowMapStreamEvent") + return when (type) { + "initial" -> { + val mapNode = + node.get("map") ?: throw JsonMappingException.from(p, "Missing map field for initial") + val map = p.codec.treeToValue(mapNode, Map::class.java) as Map + FlowMapStreamEvent.InitialState(map.toPersistentMap()) + } + "update" -> { + val eventNode = + node.get("event") + ?: throw JsonMappingException.from(p, "Missing event field for update") + val event = p.codec.treeToValue(eventNode, MapEvent::class.java) as MapEvent + FlowMapStreamEvent.EventUpdate(event) + } + else -> throw JsonMappingException.from(p, "Unknown type: $type") + } + } +} + +private class MapEventSerializer : StdSerializer>(MapEvent::class.java) { + override fun serialize(value: MapEvent<*, *>, gen: JsonGenerator, provider: SerializerProvider) { + gen.writeStartObject() + when (value) { + is MapEvent.Populated -> { + gen.writeStringField("type", "populated") + } + is MapEvent.EntryEvent.Upsert -> { + gen.writeStringField("type", "upsert") + gen.writeFieldName("key") + provider.defaultSerializeValue(value.key, gen) + gen.writeFieldName("oldValue") + provider.defaultSerializeValue(value.oldValue, gen) + gen.writeFieldName("newValue") + provider.defaultSerializeValue(value.newValue, gen) + } + is MapEvent.EntryEvent.Removed -> { + gen.writeStringField("type", "removed") + gen.writeFieldName("key") + provider.defaultSerializeValue(value.key, gen) + gen.writeFieldName("oldValue") + provider.defaultSerializeValue(value.oldValue, gen) + } + } + gen.writeEndObject() + } +} + +private class MapEventDeserializer : StdDeserializer>(MapEvent::class.java) { + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): MapEvent<*, *> { + val node = p.codec.readTree(p) + val type = + node.get("type")?.asText() + ?: throw JsonMappingException.from(p, "Missing type field for MapEvent") + return when (type) { + "populated" -> MapEvent.Populated + "upsert" -> { + val key = + node.get("key")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing key field for upsert") + val oldValueNode = node.get("oldValue") + val oldValue = + if (oldValueNode == null || oldValueNode.isNull) null + else p.codec.treeToValue(oldValueNode, Any::class.java) + val newValue = + node.get("newValue")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing newValue field for upsert") + MapEvent.EntryEvent.Upsert(key, oldValue, newValue) + } + "removed" -> { + val key = + node.get("key")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing key field for removed") + val oldValue = + node.get("oldValue")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing oldValue field for removed") + MapEvent.EntryEvent.Removed(key, oldValue) + } + else -> throw JsonMappingException.from(p, "Unknown MapEvent type: $type") + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt index 7eefc82..72002c2 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt @@ -86,6 +86,17 @@ interface FlowMap : MapFlow, Map { fun asMap(): PersistentMap } +/** An event emitted by [MapFlow.asFlowWithState]. */ +sealed interface FlowMapStreamEvent { + /** Emitted on initial collection, containing the entire initial [map] state. */ + @JvmInline + value class InitialState(val map: PersistentMap) : FlowMapStreamEvent + + /** Emitted for subsequent updates, containing only the delta ([event]). */ + @JvmInline + value class EventUpdate(val event: MapEvent) : FlowMapStreamEvent +} + interface MapFlow { /** * A [Flow] of events that can be used to reconstitute the current and future state of the @@ -96,9 +107,29 @@ interface MapFlow { * A [Removed] will be sent if an entry used to match, but no longer does. * * Events can be conflated with [conflateKeys]. + * + * **Note:** This method is useful when individual entry tracking or filtering is required. + * However, for maps with a large initial state, it has higher overhead because it emits an + * individual [Upsert] event for every existing entry before emitting [Populated]. If you only + * need the current map state and subsequent updates, consider using [asFlowWithState] for better + * performance. */ fun asFlow(predicate: ((K, V) -> Boolean)? = null): Flow> + /** + * A [Flow] that emits a [FlowMapStreamEvent] to represent the state and its mutations over time. + * + * On initial collection, it emits a single [FlowMapStreamEvent.InitialState] with the full + * initial map state. Later events are emitted as [FlowMapStreamEvent.EventUpdate] containing the + * [Upsert] or [Removed] deltas. + * + * **Note:** This is the preferred method for performance-sensitive subscribers who need the + * current state immediately, as it avoids the overhead of processing individual events to + * reconstruct the initial map, while also avoiding the serialization cost of sending the full map + * on every subsequent update. + */ + fun asFlowWithState(): Flow> + /** * A [Flow] of the latest value for the provided [key] or `null` if no value is present. * @@ -208,6 +239,21 @@ private class FlowMapImpl(initialMap: PersistentMap) : } } + override fun asFlowWithState(): Flow> = flow { + var first = true + orderedSignal.collect { flowMapEvent -> + if (first) { + emit(FlowMapStreamEvent.InitialState(flowMapEvent.state.map)) + first = false + } else { + val events = flowMapEvent.events + for (event in events) { + emit(FlowMapStreamEvent.EventUpdate(event)) + } + } + } + } + override fun valueFlow(key: K): Flow = state.map { it.map[key] }.distinctUntilChanged() override fun put(key: K, value: V): V? { diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt new file mode 100644 index 0000000..076c954 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt @@ -0,0 +1,30 @@ +package com.caplin.integration.datasourcex.util.flow + +import kotlinx.collections.immutable.PersistentMap +import kotlinx.collections.immutable.toPersistentMap +import org.apache.fory.Fory +import org.apache.fory.serializer.collection.MapSerializer + +/** + * Registers serializers for [PersistentMap] and other internal types with the provided [Fory] instance. + */ +fun Fory.registerDataSourceSerializers(): Fory = apply { + registerSerializer(PersistentMap::class.java, PersistentMapSerializer::class.java) + // Kotlin immutable collections implementations often have internal names + val hashMapClass = + runCatching { Class.forName("kotlinx.collections.immutable.implementations.immutableMap.PersistentHashMap") }.getOrNull() + if (hashMapClass != null) { + registerSerializer(hashMapClass, PersistentMapSerializer::class.java) + } +} + +/** + * A Fory [MapSerializer] for [PersistentMap]. + */ +class PersistentMapSerializer(fory: Fory, type: Class>) : MapSerializer>(fory, type) { + + @Suppress("UNCHECKED_CAST") + override fun onMapRead(map: Map<*, *>): PersistentMap<*, *> { + return (map as Map).toPersistentMap() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/MapEvent.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/MapEvent.kt index 5639281..09154e7 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/MapEvent.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/MapEvent.kt @@ -6,8 +6,6 @@ import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Removed import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Upsert import com.caplin.integration.datasourcex.util.flow.MapEvent.Populated -import com.caplin.integration.datasourcex.util.serializable -import java.io.Serializable import kotlinx.collections.immutable.persistentMapOf import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.onFailure @@ -26,7 +24,7 @@ import kotlinx.coroutines.selects.select * This does not support maps with `null` values or keys, consider using [java.util.Optional] if * this is required. */ -sealed interface MapEvent : Serializable { +sealed interface MapEvent { /** * Indicates that a consistent view of the map has been emitted and only updates will be seen from @@ -36,8 +34,6 @@ sealed interface MapEvent : Serializable { * event. */ object Populated : MapEvent { - private fun readResolve(): Any = Populated - override fun toString(): String { return "Populated()" } @@ -140,7 +136,7 @@ fun Flow>.runningFoldToMap( var populated = false var map = persistentMapOf() - if (emitPartials) emit(map.serializable()) + if (emitPartials) emit(map) collect { mapEvent -> var emit = false @@ -165,7 +161,7 @@ fun Flow>.runningFoldToMap( } if (emit) { emitted = true - emit(map.serializable()) + emit(map) } } } @@ -182,12 +178,12 @@ fun Flow>.runningFoldToMap(): Flow map.remove(mapEvent.key).also { newMap -> check(newMap !== map) { "Attempted to remove non existent key ${mapEvent.key}" } } - emit(map.serializable()) + emit(map) } is Upsert -> { map = map.put(mapEvent.key, mapEvent.newValue) - emit(map.serializable()) + emit(map) } } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt index 0fb6928..67b6e59 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt @@ -4,8 +4,6 @@ import com.caplin.integration.datasourcex.util.flow.SetEvent.EntryEvent import com.caplin.integration.datasourcex.util.flow.SetEvent.EntryEvent.Insert import com.caplin.integration.datasourcex.util.flow.SetEvent.EntryEvent.Removed import com.caplin.integration.datasourcex.util.flow.SetEvent.Populated -import com.caplin.integration.datasourcex.util.serializable -import java.io.Serializable import java.util.concurrent.ConcurrentHashMap import kotlinx.collections.immutable.persistentSetOf import kotlinx.coroutines.Job @@ -17,11 +15,9 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onEach -sealed interface SetEvent : Serializable { +sealed interface SetEvent { object Populated : SetEvent { - private fun readResolve(): Any = Populated - override fun toString(): String { return "Populated()" } @@ -119,7 +115,7 @@ fun Flow>.runningFoldToSet( var populated = false var set = persistentSetOf() - if (emitPartials) emit(set.serializable()) + if (emitPartials) emit(set) collect { setEvent -> var emit = false @@ -147,7 +143,7 @@ fun Flow>.runningFoldToSet( } if (emit) { emitted = true - emit(set.serializable()) + emit(set) } } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEvent.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEvent.kt index 424b772..7f30974 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEvent.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEvent.kt @@ -4,8 +4,6 @@ import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent.EntryEvent import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent.EntryEvent.Removed import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent.EntryEvent.Upsert import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent.Populated -import com.caplin.integration.datasourcex.util.serializable -import java.io.Serializable import kotlinx.collections.immutable.persistentMapOf import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow @@ -22,7 +20,7 @@ import kotlinx.coroutines.flow.flow * This does not support maps with `null` values or keys, consider using [java.util.Optional] if * this is required. */ -sealed interface SimpleMapEvent : Serializable { +sealed interface SimpleMapEvent { /** * Indicates that a consistent view of the map has been emitted and only updates will be seen from @@ -32,8 +30,6 @@ sealed interface SimpleMapEvent : Serializable { * event. */ object Populated : SimpleMapEvent { - private fun readResolve(): Any = Populated - override fun toString(): String { return "Populated()" } @@ -140,7 +136,7 @@ fun Flow>.runningFoldToMap( } if (emit) { emitted = true - emit(map.serializable()) + emit(map) } } } @@ -157,12 +153,12 @@ fun Flow>.runningFoldToMap(): Flow map.remove(mapEvent.key).also { newMap -> check(newMap !== map) { "Attempted to remove non existent key ${mapEvent.key}" } } - emit(map.serializable()) + emit(map) } is Upsert -> { map = map.put(mapEvent.key, mapEvent.newValue) - emit(map.serializable()) + emit(map) } } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/ValueOrCompletion.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/ValueOrCompletion.kt index 15ddeab..6ce4b56 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/ValueOrCompletion.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/ValueOrCompletion.kt @@ -2,7 +2,6 @@ package com.caplin.integration.datasourcex.util.flow import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion.Completion import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion.Value -import java.io.Serializable import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.ensureActive @@ -23,17 +22,17 @@ import kotlinx.coroutines.flow.transformWhile * @see materializeUnboxed * @see dematerializeUnboxed */ -sealed interface ValueOrCompletion : Serializable { +sealed interface ValueOrCompletion { @Suppress("UNCHECKED_CAST") - suspend fun map(block: suspend (T) -> R): ValueOrCompletion = + suspend fun map(block: suspend (T) -> R): ValueOrCompletion = this as ValueOrCompletion - class Value(val value: T) : ValueOrCompletion { + class Value(val value: T) : ValueOrCompletion { operator fun component1(): T = value @Suppress("UNCHECKED_CAST") - override suspend fun map(block: suspend (T) -> R): ValueOrCompletion { + override suspend fun map(block: suspend (T) -> R): ValueOrCompletion { val result = block(value) return when { result === value -> this as ValueOrCompletion diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentMapTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentMapTest.kt deleted file mode 100644 index 12abdd9..0000000 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentMapTest.kt +++ /dev/null @@ -1,37 +0,0 @@ -@file:Suppress("UNCHECKED_CAST") - -package com.caplin.integration.datasourcex.util - -import io.kotest.core.spec.style.FunSpec -import io.kotest.matchers.equals.shouldBeEqual -import java.io.ByteArrayInputStream -import java.io.ByteArrayOutputStream -import java.io.ObjectInputStream -import java.io.ObjectOutputStream -import kotlinx.collections.immutable.PersistentMap -import kotlinx.collections.immutable.persistentMapOf - -class SerializablePersistentMapTest : - FunSpec({ - test("serialize persistent map") { - val map = persistentMapOf("a" to 1, "b" to 3, "c" to "z").serializable() - - val fileOutputStream = ByteArrayOutputStream() - val objectOutputStream = ObjectOutputStream(fileOutputStream) - objectOutputStream.writeObject(map) - objectOutputStream.flush() - objectOutputStream.close() - - val fileInputStream = ByteArrayInputStream(fileOutputStream.toByteArray()) - val objectInputStream = ObjectInputStream(fileInputStream) - val map2 = objectInputStream.readObject() as PersistentMap - objectInputStream.close() - - println(map2) - - map2.entries.zip(map.entries).forEach { (copy, original) -> - copy.key shouldBeEqual original.key - copy.value shouldBeEqual original.value - } - } - }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentSetTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentSetTest.kt deleted file mode 100644 index 214a701..0000000 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/SerializablePersistentSetTest.kt +++ /dev/null @@ -1,36 +0,0 @@ -@file:Suppress("UNCHECKED_CAST") - -package com.caplin.integration.datasourcex.util - -import io.kotest.common.ExperimentalKotest -import io.kotest.core.spec.style.FunSpec -import io.kotest.engine.test.logging.info -import io.kotest.matchers.equals.shouldBeEqual -import java.io.ByteArrayInputStream -import java.io.ByteArrayOutputStream -import java.io.ObjectInputStream -import java.io.ObjectOutputStream -import kotlinx.collections.immutable.persistentSetOf - -@OptIn(ExperimentalKotest::class) -class SerializablePersistentSetTest : - FunSpec({ - test("serialize persistent set") { - val set = persistentSetOf("a", "b", "c").serializable() - - val fileOutputStream = ByteArrayOutputStream() - val objectOutputStream = ObjectOutputStream(fileOutputStream) - objectOutputStream.writeObject(set) - objectOutputStream.flush() - objectOutputStream.close() - - val fileInputStream = ByteArrayInputStream(fileOutputStream.toByteArray()) - val objectInputStream = ObjectInputStream(fileInputStream) - val set2 = objectInputStream.readObject() as Set - objectInputStream.close() - - info { set2 } - - set2.zip(set).forEach { (copy, original) -> copy shouldBeEqual original } - } - }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt new file mode 100644 index 0000000..ac863d0 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt @@ -0,0 +1,92 @@ +package com.caplin.integration.datasourcex.util.flow + +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Upsert +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.maps.shouldContainExactly +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeInstanceOf +import kotlinx.collections.immutable.persistentMapOf +import org.apache.fory.Fory +import org.apache.fory.config.Language + +class FlowMapSerializationTest : + FunSpec({ + + context("Jackson Serialization") { + val mapper: ObjectMapper = jacksonObjectMapper() + .registerDataSourceModule() + + test("serialize and deserialize InitialState") { + val map = mutableFlowMapOf("1" to "A", "2" to "B") + val initialState: FlowMapStreamEvent = + FlowMapStreamEvent.InitialState(map.asMap()) + + val json = mapper.writeValueAsString(initialState) + val deserialized: FlowMapStreamEvent = + mapper.readValue(json, object : TypeReference>() {}) + + deserialized.shouldBeInstanceOf>() + deserialized.map shouldContainExactly mapOf("1" to "A", "2" to "B") + } + + test("serialize and deserialize EventUpdate") { + val event: FlowMapStreamEvent = + FlowMapStreamEvent.EventUpdate(Upsert("3", "B", "C")) + + val json = mapper.writeValueAsString(event) + val deserialized: FlowMapStreamEvent = + mapper.readValue(json, object : TypeReference>() {}) + + deserialized.shouldBeInstanceOf>() + deserialized.event.shouldBeInstanceOf>() + val upsert = deserialized.event as Upsert + upsert.key shouldBe "3" + upsert.oldValue shouldBe "B" + upsert.newValue shouldBe "C" + } + + test("serialize and deserialize MapEvent.Populated") { + val event: MapEvent = MapEvent.Populated + + val json = mapper.writeValueAsString(event) + val deserialized: MapEvent = + mapper.readValue(json, object : TypeReference>() {}) + + deserialized shouldBe MapEvent.Populated + } + } + + context("Apache Fory Serialization") { + val fory = + Fory.builder().withLanguage(Language.JAVA).requireClassRegistration(false).build() + .registerDataSourceSerializers() + + test("serialize and deserialize InitialState (raw PersistentMap)") { + val map = persistentMapOf("1" to "A", "2" to "B") + val initialState = FlowMapStreamEvent.InitialState(map) + + val bytes = fory.serialize(initialState) + val deserialized = fory.deserialize(bytes) + + deserialized.shouldBeInstanceOf>() + deserialized.map shouldContainExactly mapOf("1" to "A", "2" to "B") + } + + test("serialize and deserialize EventUpdate") { + val event = FlowMapStreamEvent.EventUpdate(Upsert("3", "B", "C")) + + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + + deserialized.shouldBeInstanceOf>() + deserialized.event.shouldBeInstanceOf>() + val upsert = deserialized.event as Upsert<*, *> + upsert.key shouldBe "3" + upsert.oldValue shouldBe "B" + upsert.newValue shouldBe "C" + } + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt index a998984..2f17993 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt @@ -14,6 +14,7 @@ import io.kotest.matchers.maps.shouldContainExactly import io.kotest.matchers.nulls.shouldBeNull import io.kotest.matchers.nulls.shouldNotBeNull import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeInstanceOf import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.sync.Mutex @@ -40,6 +41,26 @@ class FlowMapTest : } } + test("FlowMap asFlowWithState") { + val map = mutableFlowMapOf("1" to "A", "2" to "B") + + map.asFlowWithState().test { + val initial = awaitItem() + initial.shouldBeInstanceOf>() + initial.map shouldContainExactly mapOf("1" to "A", "2" to "B") + + map.put("3", "C") + val afterPut = awaitItem() + afterPut.shouldBeInstanceOf>() + afterPut.event shouldBeEqual Upsert("3", null, "C") + + map.remove("1") + val afterRemove = awaitItem() + afterRemove.shouldBeInstanceOf>() + afterRemove.event shouldBeEqual Removed("1", "A") + } + } + test("FlowMap asFlow with predicate") { val map = mutableFlowMapOf("1" to "AL", "2" to "B") From 6dcdb52368af57818b06ec795c215b2cf602af8c Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Wed, 4 Mar 2026 13:45:14 +0000 Subject: [PATCH 07/14] Drop native java serialization support, add fory and jackson serializers, and add benchmarks --- .../datasourcex/util/flow/FlowMap.kt | 3 ++- .../util/flow/FlowMapForySupport.kt | 17 +++++++++++------ .../util/flow/ValueOrCompletion.kt | 3 +-- .../util/flow/FlowMapSerializationTest.kt | 19 +++++++++++++------ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt index 72002c2..28e292f 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt @@ -90,7 +90,8 @@ interface FlowMap : MapFlow, Map { sealed interface FlowMapStreamEvent { /** Emitted on initial collection, containing the entire initial [map] state. */ @JvmInline - value class InitialState(val map: PersistentMap) : FlowMapStreamEvent + value class InitialState(val map: PersistentMap) : + FlowMapStreamEvent /** Emitted for subsequent updates, containing only the delta ([event]). */ @JvmInline diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt index 076c954..c0c6f75 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt @@ -6,22 +6,27 @@ import org.apache.fory.Fory import org.apache.fory.serializer.collection.MapSerializer /** - * Registers serializers for [PersistentMap] and other internal types with the provided [Fory] instance. + * Registers serializers for [PersistentMap] and other internal types with the provided [Fory] + * instance. */ fun Fory.registerDataSourceSerializers(): Fory = apply { registerSerializer(PersistentMap::class.java, PersistentMapSerializer::class.java) // Kotlin immutable collections implementations often have internal names val hashMapClass = - runCatching { Class.forName("kotlinx.collections.immutable.implementations.immutableMap.PersistentHashMap") }.getOrNull() + runCatching { + Class.forName( + "kotlinx.collections.immutable.implementations.immutableMap.PersistentHashMap" + ) + } + .getOrNull() if (hashMapClass != null) { registerSerializer(hashMapClass, PersistentMapSerializer::class.java) } } -/** - * A Fory [MapSerializer] for [PersistentMap]. - */ -class PersistentMapSerializer(fory: Fory, type: Class>) : MapSerializer>(fory, type) { +/** A Fory [MapSerializer] for [PersistentMap]. */ +class PersistentMapSerializer(fory: Fory, type: Class>) : + MapSerializer>(fory, type) { @Suppress("UNCHECKED_CAST") override fun onMapRead(map: Map<*, *>): PersistentMap<*, *> { diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/ValueOrCompletion.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/ValueOrCompletion.kt index 6ce4b56..bb3ce9a 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/ValueOrCompletion.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/ValueOrCompletion.kt @@ -25,8 +25,7 @@ import kotlinx.coroutines.flow.transformWhile sealed interface ValueOrCompletion { @Suppress("UNCHECKED_CAST") - suspend fun map(block: suspend (T) -> R): ValueOrCompletion = - this as ValueOrCompletion + suspend fun map(block: suspend (T) -> R): ValueOrCompletion = this as ValueOrCompletion class Value(val value: T) : ValueOrCompletion { operator fun component1(): T = value diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt index ac863d0..39fdd6c 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt @@ -14,10 +14,8 @@ import org.apache.fory.config.Language class FlowMapSerializationTest : FunSpec({ - context("Jackson Serialization") { - val mapper: ObjectMapper = jacksonObjectMapper() - .registerDataSourceModule() + val mapper: ObjectMapper = jacksonObjectMapper().registerDataSourceModule() test("serialize and deserialize InitialState") { val map = mutableFlowMapOf("1" to "A", "2" to "B") @@ -26,7 +24,10 @@ class FlowMapSerializationTest : val json = mapper.writeValueAsString(initialState) val deserialized: FlowMapStreamEvent = - mapper.readValue(json, object : TypeReference>() {}) + mapper.readValue( + json, + object : TypeReference>() {}, + ) deserialized.shouldBeInstanceOf>() deserialized.map shouldContainExactly mapOf("1" to "A", "2" to "B") @@ -38,7 +39,10 @@ class FlowMapSerializationTest : val json = mapper.writeValueAsString(event) val deserialized: FlowMapStreamEvent = - mapper.readValue(json, object : TypeReference>() {}) + mapper.readValue( + json, + object : TypeReference>() {}, + ) deserialized.shouldBeInstanceOf>() deserialized.event.shouldBeInstanceOf>() @@ -61,7 +65,10 @@ class FlowMapSerializationTest : context("Apache Fory Serialization") { val fory = - Fory.builder().withLanguage(Language.JAVA).requireClassRegistration(false).build() + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .build() .registerDataSourceSerializers() test("serialize and deserialize InitialState (raw PersistentMap)") { From 5bff7a74f8e4c4d2d4dd53450946c4042faf28df Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Wed, 4 Mar 2026 17:55:26 +0000 Subject: [PATCH 08/14] Add serializers --- .gitattributes | 1 + util/api/datasourcex-util.api | 61 ++++++-- .../util/SimpleDataSourceFactory.kt | 2 +- .../util/flow/DataSourceJacksonModule.kt | 145 ------------------ .../datasourcex/util/flow/FlowMap.kt | 40 ++++- .../util/flow/FlowMapForySupport.kt | 35 ----- .../datasourcex/util/flow/Retry.kt | 2 +- .../serialization/fory/DataSourceModule.kt | 62 ++++++++ .../fory/EventUpdateSerializer.kt | 24 +++ .../fory/FlowMapStreamEventSerializer.kt | 42 +++++ .../fory/InitialStateSerializer.kt | 23 +++ .../serialization/fory/MapEventSerializer.kt | 52 +++++++ .../fory/PersistentHashMapSerializer.kt | 31 ++++ .../fory/PersistentHashSetSerializer.kt | 31 ++++ .../fory/PersistentOrderedMapSerializer.kt | 32 ++++ .../fory/PersistentOrderedSetSerializer.kt | 31 ++++ .../serialization/fory/SetEventSerializer.kt | 46 ++++++ .../fory/SimpleMapEventSerializer.kt | 48 ++++++ .../fory/ValueOrCompletionSerializer.kt | 42 +++++ .../serialization/jackson/DataSourceModule.kt | 37 +++++ .../jackson/FlowMapStreamEventDeserializer.kt | 37 +++++ .../jackson/FlowMapStreamEventSerializer.kt | 30 ++++ .../jackson/MapEventDeserializer.kt | 43 ++++++ .../jackson/MapEventSerializer.kt | 34 ++++ .../jackson/SetEventDeserializer.kt | 33 ++++ .../jackson/SetEventSerializer.kt | 32 ++++ .../jackson/SimpleMapEventDeserializer.kt | 37 +++++ .../jackson/SimpleMapEventSerializer.kt | 35 +++++ .../jackson/ValueOrCompletionDeserializer.kt | 31 ++++ .../jackson/ValueOrCompletionSerializer.kt | 33 ++++ .../util/flow/FlowMapSerializationTest.kt | 99 ------------ .../datasourcex/util/flow/FlowMapTest.kt | 14 ++ .../FlowMapStreamEventSerializationTest.kt | 58 +++++++ .../fory/MapEventSerializationTest.kt | 40 +++++ .../PersistentCollectionSerializationTest.kt | 51 ++++++ .../fory/SetEventSerializationTest.kt | 38 +++++ .../fory/SimpleMapEventSerializationTest.kt | 38 +++++ .../ValueOrCompletionSerializationTest.kt | 31 ++++ .../FlowMapStreamEventSerializationTest.kt | 52 +++++++ .../jackson/MapEventSerializationTest.kt | 39 +++++ .../jackson/SetEventSerializationTest.kt | 34 ++++ .../SimpleMapEventSerializationTest.kt | 37 +++++ .../ValueOrCompletionSerializationTest.kt | 41 +++++ 43 files changed, 1408 insertions(+), 296 deletions(-) delete mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/DataSourceJacksonModule.kt delete mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/DataSourceModule.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/EventUpdateSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/InitialStateSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashMapSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashSetSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedMapSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedSetSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventDeserializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventDeserializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventDeserializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventDeserializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionDeserializer.kt create mode 100644 util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializer.kt delete mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentCollectionSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializationTest.kt create mode 100644 util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializationTest.kt diff --git a/.gitattributes b/.gitattributes index 80f7947..7d7e5fe 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,6 +3,7 @@ *.sh text eol=lf *.conf text eol=lf gradlew text eol=lf +*.api text eol=lf # These files are text and should be normalized (Convert crlf <=> lf) *.kt text diff --git a/util/api/datasourcex-util.api b/util/api/datasourcex-util.api index 8e25079..d686a63 100644 --- a/util/api/datasourcex-util.api +++ b/util/api/datasourcex-util.api @@ -77,14 +77,6 @@ public final class com/caplin/integration/datasourcex/util/ReadWriteLock { public final fun writeUnlock ()V } -public final class com/caplin/integration/datasourcex/util/SerializablePersistentMapKt { - public static final fun serializable (Lkotlinx/collections/immutable/PersistentMap;)Lkotlinx/collections/immutable/PersistentMap; -} - -public final class com/caplin/integration/datasourcex/util/SerializablePersistentSetKt { - public static final fun serializable (Lkotlinx/collections/immutable/PersistentSet;)Lkotlinx/collections/immutable/PersistentSet; -} - public abstract interface class com/caplin/integration/datasourcex/util/SimpleDataSourceConfig { public abstract fun getExtraConfig ()Ljava/lang/String; public abstract fun getLocalLabel ()Ljava/lang/String; @@ -190,16 +182,48 @@ public abstract interface class com/caplin/integration/datasourcex/util/flow/Flo public final class com/caplin/integration/datasourcex/util/flow/FlowMapKt { public static final fun mutableFlowMapOf ()Lcom/caplin/integration/datasourcex/util/flow/MutableFlowMap; public static final fun mutableFlowMapOf ([Lkotlin/Pair;)Lcom/caplin/integration/datasourcex/util/flow/MutableFlowMap; + public static final fun runningFoldToMapFlowMapStreamEvent (Lkotlinx/coroutines/flow/Flow;)Lkotlinx/coroutines/flow/Flow; public static final fun simpleToFlowMapIn (Lkotlinx/coroutines/flow/Flow;Lkotlinx/coroutines/CoroutineScope;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static final fun toFlowMapIn (Lkotlinx/coroutines/flow/Flow;Lkotlinx/coroutines/CoroutineScope;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static final fun toMutableFlowMap (Ljava/util/Map;)Lcom/caplin/integration/datasourcex/util/flow/MutableFlowMap; } +public abstract interface class com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent { +} + +public final class com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent$EventUpdate : com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent { + public static final synthetic fun box-impl (Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent;)Lcom/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent$EventUpdate; + public static fun constructor-impl (Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent;)Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent; + public fun equals (Ljava/lang/Object;)Z + public static fun equals-impl (Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent;Ljava/lang/Object;)Z + public static final fun equals-impl0 (Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent;Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent;)Z + public final fun getEvent ()Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent; + public fun hashCode ()I + public static fun hashCode-impl (Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent;)I + public fun toString ()Ljava/lang/String; + public static fun toString-impl (Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent;)Ljava/lang/String; + public final synthetic fun unbox-impl ()Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent; +} + +public final class com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent$InitialState : com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent { + public static final synthetic fun box-impl (Ljava/util/Map;)Lcom/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent$InitialState; + public static fun constructor-impl (Ljava/util/Map;)Ljava/util/Map; + public fun equals (Ljava/lang/Object;)Z + public static fun equals-impl (Ljava/util/Map;Ljava/lang/Object;)Z + public static final fun equals-impl0 (Ljava/util/Map;Ljava/util/Map;)Z + public final fun getMap ()Ljava/util/Map; + public fun hashCode ()I + public static fun hashCode-impl (Ljava/util/Map;)I + public fun toString ()Ljava/lang/String; + public static fun toString-impl (Ljava/util/Map;)Ljava/lang/String; + public final synthetic fun unbox-impl ()Ljava/util/Map; +} + public abstract interface class com/caplin/integration/datasourcex/util/flow/LoadingCompletingSharedFlowCache { public abstract fun get (Ljava/lang/Object;)Lcom/caplin/integration/datasourcex/util/flow/CompletingSharedFlow; } -public abstract interface class com/caplin/integration/datasourcex/util/flow/MapEvent : java/io/Serializable { +public abstract interface class com/caplin/integration/datasourcex/util/flow/MapEvent { } public abstract interface class com/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent : com/caplin/integration/datasourcex/util/flow/MapEvent { @@ -257,6 +281,7 @@ public final class com/caplin/integration/datasourcex/util/flow/MapEventKt { public abstract interface class com/caplin/integration/datasourcex/util/flow/MapFlow { public abstract fun asFlow (Lkotlin/jvm/functions/Function2;)Lkotlinx/coroutines/flow/Flow; + public abstract fun asFlowWithState ()Lkotlinx/coroutines/flow/Flow; public abstract fun valueFlow (Ljava/lang/Object;)Lkotlinx/coroutines/flow/Flow; } @@ -294,7 +319,7 @@ public final class com/caplin/integration/datasourcex/util/flow/RetryKt { public static synthetic fun retryWithExponentialBackoff$default (Lkotlinx/coroutines/flow/Flow;JJLkotlin/jvm/functions/Function3;ILjava/lang/Object;)Lkotlinx/coroutines/flow/Flow; } -public abstract interface class com/caplin/integration/datasourcex/util/flow/SetEvent : java/io/Serializable { +public abstract interface class com/caplin/integration/datasourcex/util/flow/SetEvent { } public abstract interface class com/caplin/integration/datasourcex/util/flow/SetEvent$EntryEvent : com/caplin/integration/datasourcex/util/flow/SetEvent { @@ -337,7 +362,7 @@ public final class com/caplin/integration/datasourcex/util/flow/SetEventKt { public static final fun toEvents (Lkotlinx/coroutines/flow/Flow;)Lkotlinx/coroutines/flow/Flow; } -public abstract interface class com/caplin/integration/datasourcex/util/flow/SimpleMapEvent : java/io/Serializable { +public abstract interface class com/caplin/integration/datasourcex/util/flow/SimpleMapEvent { } public abstract interface class com/caplin/integration/datasourcex/util/flow/SimpleMapEvent$EntryEvent : com/caplin/integration/datasourcex/util/flow/SimpleMapEvent { @@ -393,7 +418,7 @@ public final class com/caplin/integration/datasourcex/util/flow/TimeoutKt { public static final fun timeoutFirstOrNull (Lkotlinx/coroutines/flow/Flow;Ljava/time/Duration;)Lkotlinx/coroutines/flow/Flow; } -public abstract interface class com/caplin/integration/datasourcex/util/flow/ValueOrCompletion : java/io/Serializable { +public abstract interface class com/caplin/integration/datasourcex/util/flow/ValueOrCompletion { public abstract fun map (Lkotlin/jvm/functions/Function2;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; } @@ -430,3 +455,15 @@ public final class com/caplin/integration/datasourcex/util/flow/ValueOrCompletio public static final fun materializeUnboxed (Lkotlinx/coroutines/flow/Flow;)Lkotlinx/coroutines/flow/Flow; } +public final class com/caplin/integration/datasourcex/util/serialization/fory/DataSourceModuleKt { + public static final fun registerDataSourceSerializers (Lorg/apache/fory/Fory;)Lorg/apache/fory/Fory; +} + +public final class com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule : com/fasterxml/jackson/databind/module/SimpleModule { + public static final field INSTANCE Lcom/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule; +} + +public final class com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModuleKt { + public static final fun registerDataSourceModule (Lcom/fasterxml/jackson/databind/ObjectMapper;)Lcom/fasterxml/jackson/databind/ObjectMapper; +} + diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt index 403867e..3fa96b1 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt @@ -2,7 +2,7 @@ package com.caplin.integration.datasourcex.util import com.caplin.datasource.DataSource import com.caplin.datasource.messaging.json.JacksonJsonHandler -import com.caplin.integration.datasourcex.util.flow.registerDataSourceModule +import com.caplin.integration.datasourcex.util.serialization.jackson.registerDataSourceModule import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/DataSourceJacksonModule.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/DataSourceJacksonModule.kt deleted file mode 100644 index 8b09a5b..0000000 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/DataSourceJacksonModule.kt +++ /dev/null @@ -1,145 +0,0 @@ -package com.caplin.integration.datasourcex.util.flow - -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.core.JsonParser -import com.fasterxml.jackson.databind.DeserializationContext -import com.fasterxml.jackson.databind.JsonMappingException -import com.fasterxml.jackson.databind.Module -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.SerializerProvider -import com.fasterxml.jackson.databind.deser.std.StdDeserializer -import com.fasterxml.jackson.databind.module.SimpleModule -import com.fasterxml.jackson.databind.node.ObjectNode -import com.fasterxml.jackson.databind.ser.std.StdSerializer -import kotlinx.collections.immutable.toPersistentMap - -fun ObjectMapper.registerDataSourceModule(): ObjectMapper = registerModule(DataSourceJacksonModule) - -/** - * A Jackson [Module] that provides support for serializing and deserializing [FlowMapStreamEvent] - * and [MapEvent] without requiring annotations on the classes themselves. - */ -object DataSourceJacksonModule : SimpleModule() { - private fun readResolve(): Any = DataSourceJacksonModule - - init { - addSerializer(FlowMapStreamEvent::class.java, FlowMapStreamEventSerializer()) - addDeserializer(FlowMapStreamEvent::class.java, FlowMapStreamEventDeserializer()) - - addSerializer(MapEvent::class.java, MapEventSerializer()) - addDeserializer(MapEvent::class.java, MapEventDeserializer()) - } -} - -private class FlowMapStreamEventSerializer : - StdSerializer>(FlowMapStreamEvent::class.java) { - override fun serialize( - value: FlowMapStreamEvent<*, *>, - gen: JsonGenerator, - provider: SerializerProvider, - ) { - gen.writeStartObject() - when (value) { - is FlowMapStreamEvent.InitialState -> { - gen.writeStringField("type", "initial") - gen.writeFieldName("map") - provider.defaultSerializeValue(value.map, gen) - } - is FlowMapStreamEvent.EventUpdate -> { - gen.writeStringField("type", "update") - gen.writeFieldName("event") - provider.defaultSerializeValue(value.event, gen) - } - } - gen.writeEndObject() - } -} - -private class FlowMapStreamEventDeserializer : - StdDeserializer>(FlowMapStreamEvent::class.java) { - override fun deserialize(p: JsonParser, ctxt: DeserializationContext): FlowMapStreamEvent<*, *> { - val node = p.codec.readTree(p) - val type = - node.get("type")?.asText() - ?: throw JsonMappingException.from(p, "Missing type field for FlowMapStreamEvent") - return when (type) { - "initial" -> { - val mapNode = - node.get("map") ?: throw JsonMappingException.from(p, "Missing map field for initial") - val map = p.codec.treeToValue(mapNode, Map::class.java) as Map - FlowMapStreamEvent.InitialState(map.toPersistentMap()) - } - "update" -> { - val eventNode = - node.get("event") - ?: throw JsonMappingException.from(p, "Missing event field for update") - val event = p.codec.treeToValue(eventNode, MapEvent::class.java) as MapEvent - FlowMapStreamEvent.EventUpdate(event) - } - else -> throw JsonMappingException.from(p, "Unknown type: $type") - } - } -} - -private class MapEventSerializer : StdSerializer>(MapEvent::class.java) { - override fun serialize(value: MapEvent<*, *>, gen: JsonGenerator, provider: SerializerProvider) { - gen.writeStartObject() - when (value) { - is MapEvent.Populated -> { - gen.writeStringField("type", "populated") - } - is MapEvent.EntryEvent.Upsert -> { - gen.writeStringField("type", "upsert") - gen.writeFieldName("key") - provider.defaultSerializeValue(value.key, gen) - gen.writeFieldName("oldValue") - provider.defaultSerializeValue(value.oldValue, gen) - gen.writeFieldName("newValue") - provider.defaultSerializeValue(value.newValue, gen) - } - is MapEvent.EntryEvent.Removed -> { - gen.writeStringField("type", "removed") - gen.writeFieldName("key") - provider.defaultSerializeValue(value.key, gen) - gen.writeFieldName("oldValue") - provider.defaultSerializeValue(value.oldValue, gen) - } - } - gen.writeEndObject() - } -} - -private class MapEventDeserializer : StdDeserializer>(MapEvent::class.java) { - override fun deserialize(p: JsonParser, ctxt: DeserializationContext): MapEvent<*, *> { - val node = p.codec.readTree(p) - val type = - node.get("type")?.asText() - ?: throw JsonMappingException.from(p, "Missing type field for MapEvent") - return when (type) { - "populated" -> MapEvent.Populated - "upsert" -> { - val key = - node.get("key")?.let { p.codec.treeToValue(it, Any::class.java) } - ?: throw JsonMappingException.from(p, "Missing key field for upsert") - val oldValueNode = node.get("oldValue") - val oldValue = - if (oldValueNode == null || oldValueNode.isNull) null - else p.codec.treeToValue(oldValueNode, Any::class.java) - val newValue = - node.get("newValue")?.let { p.codec.treeToValue(it, Any::class.java) } - ?: throw JsonMappingException.from(p, "Missing newValue field for upsert") - MapEvent.EntryEvent.Upsert(key, oldValue, newValue) - } - "removed" -> { - val key = - node.get("key")?.let { p.codec.treeToValue(it, Any::class.java) } - ?: throw JsonMappingException.from(p, "Missing key field for removed") - val oldValue = - node.get("oldValue")?.let { p.codec.treeToValue(it, Any::class.java) } - ?: throw JsonMappingException.from(p, "Missing oldValue field for removed") - MapEvent.EntryEvent.Removed(key, oldValue) - } - else -> throw JsonMappingException.from(p, "Unknown MapEvent type: $type") - } - } -} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt index 28e292f..e8d8232 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt @@ -90,12 +90,11 @@ interface FlowMap : MapFlow, Map { sealed interface FlowMapStreamEvent { /** Emitted on initial collection, containing the entire initial [map] state. */ @JvmInline - value class InitialState(val map: PersistentMap) : - FlowMapStreamEvent + value class InitialState(val map: Map) : FlowMapStreamEvent /** Emitted for subsequent updates, containing only the delta ([event]). */ @JvmInline - value class EventUpdate(val event: MapEvent) : FlowMapStreamEvent + value class EventUpdate(val event: EntryEvent) : FlowMapStreamEvent } interface MapFlow { @@ -139,6 +138,41 @@ interface MapFlow { fun valueFlow(key: K): Flow } +/** Folds a flow of [FlowMapStreamEvent]s into a flow of [Map]. */ +@JvmName("runningFoldToMapFlowMapStreamEvent") +fun Flow>.runningFoldToMap(): Flow> = flow { + var map: PersistentMap? = null + + collect { streamEvent -> + when (streamEvent) { + is FlowMapStreamEvent.InitialState -> { + map = streamEvent.map.toPersistentMap() + emit(map!!) + } + + is FlowMapStreamEvent.EventUpdate -> { + val currentMap = map ?: error("InitialState must be received before EventUpdate") + when (val mapEvent = streamEvent.event) { + is Removed -> { + map = + currentMap.remove(mapEvent.key).also { newMap -> + check(newMap !== currentMap) { + "Attempted to remove non existent key ${mapEvent.key}" + } + } + emit(map!!) + } + + is Upsert -> { + map = currentMap.put(mapEvent.key, mapEvent.newValue) + emit(map!!) + } + } + } + } + } +} + private class FlowMapImpl(initialMap: PersistentMap) : MutableFlowMap { private data class State(val version: Long, val map: PersistentMap) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt deleted file mode 100644 index c0c6f75..0000000 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapForySupport.kt +++ /dev/null @@ -1,35 +0,0 @@ -package com.caplin.integration.datasourcex.util.flow - -import kotlinx.collections.immutable.PersistentMap -import kotlinx.collections.immutable.toPersistentMap -import org.apache.fory.Fory -import org.apache.fory.serializer.collection.MapSerializer - -/** - * Registers serializers for [PersistentMap] and other internal types with the provided [Fory] - * instance. - */ -fun Fory.registerDataSourceSerializers(): Fory = apply { - registerSerializer(PersistentMap::class.java, PersistentMapSerializer::class.java) - // Kotlin immutable collections implementations often have internal names - val hashMapClass = - runCatching { - Class.forName( - "kotlinx.collections.immutable.implementations.immutableMap.PersistentHashMap" - ) - } - .getOrNull() - if (hashMapClass != null) { - registerSerializer(hashMapClass, PersistentMapSerializer::class.java) - } -} - -/** A Fory [MapSerializer] for [PersistentMap]. */ -class PersistentMapSerializer(fory: Fory, type: Class>) : - MapSerializer>(fory, type) { - - @Suppress("UNCHECKED_CAST") - override fun onMapRead(map: Map<*, *>): PersistentMap<*, *> { - return (map as Map).toPersistentMap() - } -} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Retry.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Retry.kt index dce33ec..8600190 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Retry.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Retry.kt @@ -12,7 +12,7 @@ import kotlinx.coroutines.flow.retry * be modified by proving [onRetry] - if a call to this returns `false` it will stop retrying and * propagate the error downstream. */ -fun Flow.retryWithExponentialBackoff( +fun Flow.retryWithExponentialBackoff( minMillis: Long = 100L, maxMillis: Long = 60000L, onRetry: (suspend (Throwable, Long) -> Boolean) = { _, _ -> true }, diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/DataSourceModule.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/DataSourceModule.kt new file mode 100644 index 0000000..0fb287e --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/DataSourceModule.kt @@ -0,0 +1,62 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.FlowMapStreamEvent +import com.caplin.integration.datasourcex.util.flow.MapEvent +import com.caplin.integration.datasourcex.util.flow.SetEvent +import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent +import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion +import org.apache.fory.Fory + +/** Registers serializers for internal types with the provided [Fory] instance. */ +fun Fory.registerDataSourceSerializers(): Fory = apply { + // Register serializers for FlowMapStreamEvent value classes + registerSerializer(FlowMapStreamEvent::class.java, FlowMapStreamEventSerializer::class.java) + registerSerializer( + FlowMapStreamEvent.InitialState::class.java, + InitialStateSerializer::class.java, + ) + registerSerializer( + FlowMapStreamEvent.EventUpdate::class.java, + EventUpdateSerializer::class.java, + ) + + // Register serializers for sealed interfaces + registerSerializer(MapEvent::class.java, MapEventSerializer::class.java) + registerSerializer(SimpleMapEvent::class.java, SimpleMapEventSerializer::class.java) + registerSerializer(SetEvent::class.java, SetEventSerializer::class.java) + registerSerializer(ValueOrCompletion::class.java, ValueOrCompletionSerializer::class.java) + + // Register concrete serializers for the internal PersistentMap implementations + runCatching { + Class.forName( + "kotlinx.collections.immutable.implementations.immutableMap.PersistentHashMap" + ) + } + .getOrNull() + ?.let { registerSerializer(it, PersistentHashMapSerializer::class.java) } + + runCatching { + Class.forName( + "kotlinx.collections.immutable.implementations.persistentOrderedMap.PersistentOrderedMap" + ) + } + .getOrNull() + ?.let { registerSerializer(it, PersistentOrderedMapSerializer::class.java) } + + // Register concrete serializers for the internal PersistentSet implementations + runCatching { + Class.forName( + "kotlinx.collections.immutable.implementations.immutableSet.PersistentHashSet" + ) + } + .getOrNull() + ?.let { registerSerializer(it, PersistentHashSetSerializer::class.java) } + + runCatching { + Class.forName( + "kotlinx.collections.immutable.implementations.persistentOrderedSet.PersistentOrderedSet" + ) + } + .getOrNull() + ?.let { registerSerializer(it, PersistentOrderedSetSerializer::class.java) } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/EventUpdateSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/EventUpdateSerializer.kt new file mode 100644 index 0000000..daff1e6 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/EventUpdateSerializer.kt @@ -0,0 +1,24 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.FlowMapStreamEvent +import com.caplin.integration.datasourcex.util.flow.MapEvent +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.Serializer + +/** A Fory [Serializer] for [FlowMapStreamEvent.EventUpdate]. */ +internal class EventUpdateSerializer( + fory: Fory, + type: Class>, +) : Serializer>(fory, type) { + + override fun write(buffer: MemoryBuffer, value: FlowMapStreamEvent.EventUpdate<*, *>) { + fory.writeRef(buffer, value.event) + } + + @Suppress("UNCHECKED_CAST") + override fun read(buffer: MemoryBuffer): FlowMapStreamEvent.EventUpdate<*, *> { + val event = fory.readRef(buffer) as MapEvent.EntryEvent + return FlowMapStreamEvent.EventUpdate(event) + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializer.kt new file mode 100644 index 0000000..58c7bc7 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializer.kt @@ -0,0 +1,42 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.FlowMapStreamEvent +import com.caplin.integration.datasourcex.util.flow.MapEvent +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.Serializer + +internal class FlowMapStreamEventSerializer(fory: Fory, type: Class>) : + Serializer>(fory, type) { + + private enum class Type { + INITIAL_STATE, + EVENT_UPDATE, + } + + override fun write(buffer: MemoryBuffer, value: FlowMapStreamEvent<*, *>) { + when (value) { + is FlowMapStreamEvent.InitialState -> { + buffer.writeByte(Type.INITIAL_STATE.ordinal.toByte()) + fory.writeRef(buffer, value.map) + } + is FlowMapStreamEvent.EventUpdate -> { + buffer.writeByte(Type.EVENT_UPDATE.ordinal.toByte()) + fory.writeRef(buffer, value.event) + } + } + } + + override fun read(buffer: MemoryBuffer): FlowMapStreamEvent<*, *> { + return when (Type.entries[buffer.readByte().toInt()]) { + Type.INITIAL_STATE -> { + val map = fory.readRef(buffer) as Map + FlowMapStreamEvent.InitialState(map) + } + Type.EVENT_UPDATE -> { + val event = fory.readRef(buffer) as MapEvent.EntryEvent + FlowMapStreamEvent.EventUpdate(event) + } + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/InitialStateSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/InitialStateSerializer.kt new file mode 100644 index 0000000..24a4f1b --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/InitialStateSerializer.kt @@ -0,0 +1,23 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.FlowMapStreamEvent +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.Serializer + +/** A Fory [Serializer] for [FlowMapStreamEvent.InitialState]. */ +internal class InitialStateSerializer( + fory: Fory, + type: Class>, +) : Serializer>(fory, type) { + + override fun write(buffer: MemoryBuffer, value: FlowMapStreamEvent.InitialState<*, *>) { + fory.writeRef(buffer, value.map) + } + + @Suppress("UNCHECKED_CAST") + override fun read(buffer: MemoryBuffer): FlowMapStreamEvent.InitialState<*, *> { + val map = fory.readRef(buffer) as Map + return FlowMapStreamEvent.InitialState(map) + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializer.kt new file mode 100644 index 0000000..3fbf879 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializer.kt @@ -0,0 +1,52 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.MapEvent +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.Serializer + +internal class MapEventSerializer(fory: Fory, type: Class>) : + Serializer>(fory, type) { + + private enum class Type { + POPULATED, + UPSERT, + REMOVED, + } + + override fun write(buffer: MemoryBuffer, value: MapEvent<*, *>) { + when (value) { + is MapEvent.Populated -> { + buffer.writeByte(Type.POPULATED.ordinal.toByte()) + } + is MapEvent.EntryEvent.Upsert -> { + buffer.writeByte(Type.UPSERT.ordinal.toByte()) + fory.writeRef(buffer, value.key) + fory.writeRef(buffer, value.oldValue) + fory.writeRef(buffer, value.newValue) + } + is MapEvent.EntryEvent.Removed -> { + buffer.writeByte(Type.REMOVED.ordinal.toByte()) + fory.writeRef(buffer, value.key) + fory.writeRef(buffer, value.oldValue) + } + } + } + + override fun read(buffer: MemoryBuffer): MapEvent<*, *> { + return when (Type.values()[buffer.readByte().toInt()]) { + Type.POPULATED -> MapEvent.Populated + Type.UPSERT -> { + val key = fory.readRef(buffer) as Any + val oldValue = fory.readRef(buffer) + val newValue = fory.readRef(buffer) as Any + MapEvent.EntryEvent.Upsert(key, oldValue, newValue) + } + Type.REMOVED -> { + val key = fory.readRef(buffer) as Any + val oldValue = fory.readRef(buffer) as Any + MapEvent.EntryEvent.Removed(key, oldValue) + } + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashMapSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashMapSerializer.kt new file mode 100644 index 0000000..8c68d5e --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashMapSerializer.kt @@ -0,0 +1,31 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import kotlinx.collections.immutable.PersistentMap +import kotlinx.collections.immutable.toPersistentHashMap +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.collection.MapSerializer + +/** A Fory [MapSerializer] for [PersistentMap] (Hash implementation). */ +internal class PersistentHashMapSerializer(fory: Fory, type: Class>) : + MapSerializer>(fory, type) { + + override fun write(buffer: MemoryBuffer, value: PersistentMap<*, *>) { + fory.writeRef(buffer, HashMap(value)) + } + + @Suppress("UNCHECKED_CAST") + override fun read(buffer: MemoryBuffer): PersistentMap<*, *> { + val map = fory.readRef(buffer) as Map + return map.toPersistentHashMap() + } + + override fun newMap(buffer: MemoryBuffer): MutableMap<*, *> { + return HashMap() + } + + @Suppress("UNCHECKED_CAST") + override fun onMapRead(map: Map<*, *>): PersistentMap<*, *> { + return (map as Map).toPersistentHashMap() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashSetSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashSetSerializer.kt new file mode 100644 index 0000000..a0ec381 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashSetSerializer.kt @@ -0,0 +1,31 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import kotlinx.collections.immutable.PersistentSet +import kotlinx.collections.immutable.toPersistentHashSet +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.collection.CollectionSerializer + +/** A Fory [CollectionSerializer] for [PersistentSet] (Hash implementation). */ +internal class PersistentHashSetSerializer(fory: Fory, type: Class>) : + CollectionSerializer>(fory, type) { + + override fun write(buffer: MemoryBuffer, value: PersistentSet<*>) { + fory.writeRef(buffer, HashSet(value)) + } + + @Suppress("UNCHECKED_CAST") + override fun read(buffer: MemoryBuffer): PersistentSet<*> { + val collection = fory.readRef(buffer) as Collection + return collection.toPersistentHashSet() + } + + override fun newCollection(buffer: MemoryBuffer): MutableCollection<*> { + return HashSet() + } + + @Suppress("UNCHECKED_CAST") + override fun onCollectionRead(collection: Collection<*>): PersistentSet<*> { + return (collection as Collection).toPersistentHashSet() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedMapSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedMapSerializer.kt new file mode 100644 index 0000000..f7f98ba --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedMapSerializer.kt @@ -0,0 +1,32 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import kotlinx.collections.immutable.PersistentMap +import kotlinx.collections.immutable.toPersistentMap +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.collection.MapSerializer + +/** A Fory [MapSerializer] for [PersistentMap] (Ordered implementation). */ +internal class PersistentOrderedMapSerializer(fory: Fory, type: Class>) : + MapSerializer>(fory, type) { + + override fun write(buffer: MemoryBuffer, value: PersistentMap<*, *>) { + fory.writeRef(buffer, LinkedHashMap(value)) + } + + @Suppress("UNCHECKED_CAST") + override fun read(buffer: MemoryBuffer): PersistentMap<*, *> { + val map = fory.readRef(buffer) as Map + return map.toPersistentMap() + } + + override fun newMap(buffer: MemoryBuffer): MutableMap<*, *> { + return LinkedHashMap() + } + + @Suppress("UNCHECKED_CAST") + override fun onMapRead(map: Map<*, *>): PersistentMap<*, *> { + // toPersistentMap() on a LinkedHashMap preserves order in kotlinx-collections-immutable + return (map as Map).toPersistentMap() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedSetSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedSetSerializer.kt new file mode 100644 index 0000000..25d7892 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedSetSerializer.kt @@ -0,0 +1,31 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import kotlinx.collections.immutable.PersistentSet +import kotlinx.collections.immutable.toPersistentSet +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.collection.CollectionSerializer + +/** A Fory [CollectionSerializer] for [PersistentSet] (Ordered implementation). */ +internal class PersistentOrderedSetSerializer(fory: Fory, type: Class>) : + CollectionSerializer>(fory, type) { + + override fun write(buffer: MemoryBuffer, value: PersistentSet<*>) { + fory.writeRef(buffer, LinkedHashSet(value)) + } + + @Suppress("UNCHECKED_CAST") + override fun read(buffer: MemoryBuffer): PersistentSet<*> { + val collection = fory.readRef(buffer) as Collection + return collection.toPersistentSet() + } + + override fun newCollection(buffer: MemoryBuffer): MutableCollection<*> { + return LinkedHashSet() + } + + @Suppress("UNCHECKED_CAST") + override fun onCollectionRead(collection: Collection<*>): PersistentSet<*> { + return (collection as Collection).toPersistentSet() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializer.kt new file mode 100644 index 0000000..d917445 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializer.kt @@ -0,0 +1,46 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.SetEvent +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.Serializer + +internal class SetEventSerializer(fory: Fory, type: Class>) : + Serializer>(fory, type) { + + private enum class Type { + POPULATED, + INSERT, + REMOVED, + } + + override fun write(buffer: MemoryBuffer, value: SetEvent<*>) { + when (value) { + is SetEvent.Populated -> { + buffer.writeByte(Type.POPULATED.ordinal.toByte()) + } + is SetEvent.EntryEvent.Insert -> { + buffer.writeByte(Type.INSERT.ordinal.toByte()) + fory.writeRef(buffer, value.value) + } + is SetEvent.EntryEvent.Removed -> { + buffer.writeByte(Type.REMOVED.ordinal.toByte()) + fory.writeRef(buffer, value.value) + } + } + } + + override fun read(buffer: MemoryBuffer): SetEvent<*> { + return when (Type.values()[buffer.readByte().toInt()]) { + Type.POPULATED -> SetEvent.Populated + Type.INSERT -> { + val value = fory.readRef(buffer) as Any + SetEvent.EntryEvent.Insert(value) + } + Type.REMOVED -> { + val value = fory.readRef(buffer) as Any + SetEvent.EntryEvent.Removed(value) + } + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializer.kt new file mode 100644 index 0000000..741007c --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializer.kt @@ -0,0 +1,48 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.Serializer + +internal class SimpleMapEventSerializer(fory: Fory, type: Class>) : + Serializer>(fory, type) { + + private enum class Type { + POPULATED, + UPSERT, + REMOVED, + } + + override fun write(buffer: MemoryBuffer, value: SimpleMapEvent<*, *>) { + when (value) { + is SimpleMapEvent.Populated -> { + buffer.writeByte(Type.POPULATED.ordinal.toByte()) + } + is SimpleMapEvent.EntryEvent.Upsert -> { + buffer.writeByte(Type.UPSERT.ordinal.toByte()) + fory.writeRef(buffer, value.key) + fory.writeRef(buffer, value.newValue) + } + is SimpleMapEvent.EntryEvent.Removed -> { + buffer.writeByte(Type.REMOVED.ordinal.toByte()) + fory.writeRef(buffer, value.key) + } + } + } + + override fun read(buffer: MemoryBuffer): SimpleMapEvent<*, *> { + return when (Type.values()[buffer.readByte().toInt()]) { + Type.POPULATED -> SimpleMapEvent.Populated + Type.UPSERT -> { + val key = fory.readRef(buffer) as Any + val newValue = fory.readRef(buffer) as Any + SimpleMapEvent.EntryEvent.Upsert(key, newValue) + } + Type.REMOVED -> { + val key = fory.readRef(buffer) as Any + SimpleMapEvent.EntryEvent.Removed(key) + } + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializer.kt new file mode 100644 index 0000000..e4e4adb --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializer.kt @@ -0,0 +1,42 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion +import org.apache.fory.Fory +import org.apache.fory.memory.MemoryBuffer +import org.apache.fory.serializer.Serializer + +internal class ValueOrCompletionSerializer(fory: Fory, type: Class>) : + Serializer>(fory, type) { + + private enum class Type { + VALUE, + COMPLETION, + } + + override fun write(buffer: MemoryBuffer, value: ValueOrCompletion<*>) { + when (value) { + is ValueOrCompletion.Value -> { + buffer.writeByte(Type.VALUE.ordinal.toByte()) + fory.writeRef(buffer, value.value) + } + is ValueOrCompletion.Completion -> { + buffer.writeByte(Type.COMPLETION.ordinal.toByte()) + val message = value.throwable?.message ?: value.throwable?.toString() + fory.writeRef(buffer, message) + } + } + } + + override fun read(buffer: MemoryBuffer): ValueOrCompletion<*> { + return when (Type.values()[buffer.readByte().toInt()]) { + Type.VALUE -> { + val value = fory.readRef(buffer) as Any + ValueOrCompletion.Value(value) + } + Type.COMPLETION -> { + val message = fory.readRef(buffer) as String? + ValueOrCompletion.Completion(message?.let { RuntimeException(it) }) + } + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt new file mode 100644 index 0000000..dd5b286 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt @@ -0,0 +1,37 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.FlowMapStreamEvent +import com.caplin.integration.datasourcex.util.flow.MapEvent +import com.caplin.integration.datasourcex.util.flow.SetEvent +import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent +import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion +import com.fasterxml.jackson.databind.Module +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.module.SimpleModule + +fun ObjectMapper.registerDataSourceModule(): ObjectMapper = registerModule(DataSourceModule) + +/** + * A Jackson [Module] that provides support for serializing and deserializing DataSource types + * without requiring annotations on the classes themselves. + */ +object DataSourceModule : SimpleModule() { + private fun readResolve(): Any = DataSourceModule + + init { + addSerializer(FlowMapStreamEvent::class.java, FlowMapStreamEventSerializer()) + addDeserializer(FlowMapStreamEvent::class.java, FlowMapStreamEventDeserializer()) + + addSerializer(MapEvent::class.java, MapEventSerializer()) + addDeserializer(MapEvent::class.java, MapEventDeserializer()) + + addSerializer(SimpleMapEvent::class.java, SimpleMapEventSerializer()) + addDeserializer(SimpleMapEvent::class.java, SimpleMapEventDeserializer()) + + addSerializer(SetEvent::class.java, SetEventSerializer()) + addDeserializer(SetEvent::class.java, SetEventDeserializer()) + + addSerializer(ValueOrCompletion::class.java, ValueOrCompletionSerializer()) + addDeserializer(ValueOrCompletion::class.java, ValueOrCompletionDeserializer()) + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventDeserializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventDeserializer.kt new file mode 100644 index 0000000..d7609d8 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventDeserializer.kt @@ -0,0 +1,37 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.FlowMapStreamEvent +import com.caplin.integration.datasourcex.util.flow.MapEvent +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.deser.std.StdDeserializer +import com.fasterxml.jackson.databind.node.ObjectNode + +internal class FlowMapStreamEventDeserializer : + StdDeserializer>(FlowMapStreamEvent::class.java) { + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): FlowMapStreamEvent<*, *> { + val node = p.codec.readTree(p) + val type = + node.get("type")?.asText() + ?: throw JsonMappingException.from(p, "Missing type field for FlowMapStreamEvent") + return when (type) { + "initial" -> { + val mapNode = + node.get("map") ?: throw JsonMappingException.from(p, "Missing map field for initial") + val map = p.codec.treeToValue(mapNode, Map::class.java) as Map + FlowMapStreamEvent.InitialState(map) + } + "update" -> { + val eventNode = + node.get("event") + ?: throw JsonMappingException.from(p, "Missing event field for update") + val event = + p.codec.treeToValue(eventNode, MapEvent.EntryEvent::class.java) + as MapEvent.EntryEvent + FlowMapStreamEvent.EventUpdate(event) + } + else -> throw JsonMappingException.from(p, "Unknown type: $type") + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializer.kt new file mode 100644 index 0000000..83896bf --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializer.kt @@ -0,0 +1,30 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.FlowMapStreamEvent +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer + +internal class FlowMapStreamEventSerializer : + StdSerializer>(FlowMapStreamEvent::class.java) { + override fun serialize( + value: FlowMapStreamEvent<*, *>, + gen: JsonGenerator, + provider: SerializerProvider, + ) { + gen.writeStartObject() + when (value) { + is FlowMapStreamEvent.InitialState -> { + gen.writeStringField("type", "initial") + gen.writeFieldName("map") + provider.defaultSerializeValue(value.map, gen) + } + is FlowMapStreamEvent.EventUpdate -> { + gen.writeStringField("type", "update") + gen.writeFieldName("event") + provider.defaultSerializeValue(value.event, gen) + } + } + gen.writeEndObject() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventDeserializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventDeserializer.kt new file mode 100644 index 0000000..4116821 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventDeserializer.kt @@ -0,0 +1,43 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.MapEvent +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.deser.std.StdDeserializer +import com.fasterxml.jackson.databind.node.ObjectNode + +internal class MapEventDeserializer : StdDeserializer>(MapEvent::class.java) { + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): MapEvent<*, *> { + val node = p.codec.readTree(p) + val type = + node.get("type")?.asText() + ?: throw JsonMappingException.from(p, "Missing type field for MapEvent") + return when (type) { + "populated" -> MapEvent.Populated + "upsert" -> { + val key = + node.get("key")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing key field for upsert") + val oldValueNode = node.get("oldValue") + val oldValue = + if (oldValueNode == null || oldValueNode.isNull) null + else p.codec.treeToValue(oldValueNode, Any::class.java) + val newValue = + node.get("newValue")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing newValue field for upsert") + MapEvent.EntryEvent.Upsert(key, oldValue, newValue) + } + "removed" -> { + val key = + node.get("key")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing key field for removed") + val oldValue = + node.get("oldValue")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing oldValue field for removed") + MapEvent.EntryEvent.Removed(key, oldValue) + } + else -> throw JsonMappingException.from(p, "Unknown MapEvent type: $type") + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializer.kt new file mode 100644 index 0000000..c4aab5d --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializer.kt @@ -0,0 +1,34 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.MapEvent +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer + +internal class MapEventSerializer : StdSerializer>(MapEvent::class.java) { + override fun serialize(value: MapEvent<*, *>, gen: JsonGenerator, provider: SerializerProvider) { + gen.writeStartObject() + when (value) { + is MapEvent.Populated -> { + gen.writeStringField("type", "populated") + } + is MapEvent.EntryEvent.Upsert -> { + gen.writeStringField("type", "upsert") + gen.writeFieldName("key") + provider.defaultSerializeValue(value.key, gen) + gen.writeFieldName("oldValue") + provider.defaultSerializeValue(value.oldValue, gen) + gen.writeFieldName("newValue") + provider.defaultSerializeValue(value.newValue, gen) + } + is MapEvent.EntryEvent.Removed -> { + gen.writeStringField("type", "removed") + gen.writeFieldName("key") + provider.defaultSerializeValue(value.key, gen) + gen.writeFieldName("oldValue") + provider.defaultSerializeValue(value.oldValue, gen) + } + } + gen.writeEndObject() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventDeserializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventDeserializer.kt new file mode 100644 index 0000000..41c90f4 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventDeserializer.kt @@ -0,0 +1,33 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.SetEvent +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.deser.std.StdDeserializer +import com.fasterxml.jackson.databind.node.ObjectNode + +internal class SetEventDeserializer : StdDeserializer>(SetEvent::class.java) { + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): SetEvent<*> { + val node = p.codec.readTree(p) + val type = + node.get("type")?.asText() + ?: throw JsonMappingException.from(p, "Missing type field for SetEvent") + return when (type) { + "populated" -> SetEvent.Populated + "insert" -> { + val value = + node.get("value")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing value field for insert") + SetEvent.EntryEvent.Insert(value) + } + "removed" -> { + val value = + node.get("value")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing value field for removed") + SetEvent.EntryEvent.Removed(value) + } + else -> throw JsonMappingException.from(p, "Unknown SetEvent type: $type") + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializer.kt new file mode 100644 index 0000000..9c2d490 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializer.kt @@ -0,0 +1,32 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.SetEvent +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer + +internal class SetEventSerializer : StdSerializer>(SetEvent::class.java) { + override fun serialize( + value: SetEvent<*>, + gen: JsonGenerator, + provider: SerializerProvider, + ) { + gen.writeStartObject() + when (value) { + is SetEvent.Populated -> { + gen.writeStringField("type", "populated") + } + is SetEvent.EntryEvent.Insert -> { + gen.writeStringField("type", "insert") + gen.writeFieldName("value") + provider.defaultSerializeValue(value.value, gen) + } + is SetEvent.EntryEvent.Removed -> { + gen.writeStringField("type", "removed") + gen.writeFieldName("value") + provider.defaultSerializeValue(value.value, gen) + } + } + gen.writeEndObject() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventDeserializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventDeserializer.kt new file mode 100644 index 0000000..03e0147 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventDeserializer.kt @@ -0,0 +1,37 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.deser.std.StdDeserializer +import com.fasterxml.jackson.databind.node.ObjectNode + +internal class SimpleMapEventDeserializer : + StdDeserializer>(SimpleMapEvent::class.java) { + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): SimpleMapEvent<*, *> { + val node = p.codec.readTree(p) + val type = + node.get("type")?.asText() + ?: throw JsonMappingException.from(p, "Missing type field for SimpleMapEvent") + return when (type) { + "populated" -> SimpleMapEvent.Populated + "upsert" -> { + val key = + node.get("key")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing key field for upsert") + val newValue = + node.get("newValue")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing newValue field for upsert") + SimpleMapEvent.EntryEvent.Upsert(key, newValue) + } + "removed" -> { + val key = + node.get("key")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing key field for removed") + SimpleMapEvent.EntryEvent.Removed(key) + } + else -> throw JsonMappingException.from(p, "Unknown SimpleMapEvent type: $type") + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializer.kt new file mode 100644 index 0000000..9f036b5 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializer.kt @@ -0,0 +1,35 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer + +internal class SimpleMapEventSerializer : + StdSerializer>(SimpleMapEvent::class.java) { + override fun serialize( + value: SimpleMapEvent<*, *>, + gen: JsonGenerator, + provider: SerializerProvider, + ) { + gen.writeStartObject() + when (value) { + is SimpleMapEvent.Populated -> { + gen.writeStringField("type", "populated") + } + is SimpleMapEvent.EntryEvent.Upsert -> { + gen.writeStringField("type", "upsert") + gen.writeFieldName("key") + provider.defaultSerializeValue(value.key, gen) + gen.writeFieldName("newValue") + provider.defaultSerializeValue(value.newValue, gen) + } + is SimpleMapEvent.EntryEvent.Removed -> { + gen.writeStringField("type", "removed") + gen.writeFieldName("key") + provider.defaultSerializeValue(value.key, gen) + } + } + gen.writeEndObject() + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionDeserializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionDeserializer.kt new file mode 100644 index 0000000..ad79c9b --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionDeserializer.kt @@ -0,0 +1,31 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.deser.std.StdDeserializer +import com.fasterxml.jackson.databind.node.ObjectNode + +internal class ValueOrCompletionDeserializer : + StdDeserializer>(ValueOrCompletion::class.java) { + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): ValueOrCompletion<*> { + val node = p.codec.readTree(p) + val type = + node.get("type")?.asText() + ?: throw JsonMappingException.from(p, "Missing type field for ValueOrCompletion") + return when (type) { + "value" -> { + val value = + node.get("value")?.let { p.codec.treeToValue(it, Any::class.java) } + ?: throw JsonMappingException.from(p, "Missing value field for value type") + ValueOrCompletion.Value(value) + } + "completion" -> { + val error = node.get("error")?.let { if (it.isNull) null else it.asText() } + ValueOrCompletion.Completion(error?.let { RuntimeException(it) }) + } + else -> throw JsonMappingException.from(p, "Unknown ValueOrCompletion type: $type") + } + } +} diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializer.kt new file mode 100644 index 0000000..23fe993 --- /dev/null +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializer.kt @@ -0,0 +1,33 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer + +internal class ValueOrCompletionSerializer : + StdSerializer>(ValueOrCompletion::class.java) { + override fun serialize( + value: ValueOrCompletion<*>, + gen: JsonGenerator, + provider: SerializerProvider, + ) { + gen.writeStartObject() + when (value) { + is ValueOrCompletion.Value -> { + gen.writeStringField("type", "value") + gen.writeFieldName("value") + provider.defaultSerializeValue(value.value, gen) + } + is ValueOrCompletion.Completion -> { + gen.writeStringField("type", "completion") + if (value.throwable != null) { + gen.writeStringField("error", value.throwable.message ?: value.throwable.toString()) + } else { + gen.writeNullField("error") + } + } + } + gen.writeEndObject() + } +} diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt deleted file mode 100644 index 39fdd6c..0000000 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapSerializationTest.kt +++ /dev/null @@ -1,99 +0,0 @@ -package com.caplin.integration.datasourcex.util.flow - -import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Upsert -import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import io.kotest.core.spec.style.FunSpec -import io.kotest.matchers.maps.shouldContainExactly -import io.kotest.matchers.shouldBe -import io.kotest.matchers.types.shouldBeInstanceOf -import kotlinx.collections.immutable.persistentMapOf -import org.apache.fory.Fory -import org.apache.fory.config.Language - -class FlowMapSerializationTest : - FunSpec({ - context("Jackson Serialization") { - val mapper: ObjectMapper = jacksonObjectMapper().registerDataSourceModule() - - test("serialize and deserialize InitialState") { - val map = mutableFlowMapOf("1" to "A", "2" to "B") - val initialState: FlowMapStreamEvent = - FlowMapStreamEvent.InitialState(map.asMap()) - - val json = mapper.writeValueAsString(initialState) - val deserialized: FlowMapStreamEvent = - mapper.readValue( - json, - object : TypeReference>() {}, - ) - - deserialized.shouldBeInstanceOf>() - deserialized.map shouldContainExactly mapOf("1" to "A", "2" to "B") - } - - test("serialize and deserialize EventUpdate") { - val event: FlowMapStreamEvent = - FlowMapStreamEvent.EventUpdate(Upsert("3", "B", "C")) - - val json = mapper.writeValueAsString(event) - val deserialized: FlowMapStreamEvent = - mapper.readValue( - json, - object : TypeReference>() {}, - ) - - deserialized.shouldBeInstanceOf>() - deserialized.event.shouldBeInstanceOf>() - val upsert = deserialized.event as Upsert - upsert.key shouldBe "3" - upsert.oldValue shouldBe "B" - upsert.newValue shouldBe "C" - } - - test("serialize and deserialize MapEvent.Populated") { - val event: MapEvent = MapEvent.Populated - - val json = mapper.writeValueAsString(event) - val deserialized: MapEvent = - mapper.readValue(json, object : TypeReference>() {}) - - deserialized shouldBe MapEvent.Populated - } - } - - context("Apache Fory Serialization") { - val fory = - Fory.builder() - .withLanguage(Language.JAVA) - .requireClassRegistration(false) - .build() - .registerDataSourceSerializers() - - test("serialize and deserialize InitialState (raw PersistentMap)") { - val map = persistentMapOf("1" to "A", "2" to "B") - val initialState = FlowMapStreamEvent.InitialState(map) - - val bytes = fory.serialize(initialState) - val deserialized = fory.deserialize(bytes) - - deserialized.shouldBeInstanceOf>() - deserialized.map shouldContainExactly mapOf("1" to "A", "2" to "B") - } - - test("serialize and deserialize EventUpdate") { - val event = FlowMapStreamEvent.EventUpdate(Upsert("3", "B", "C")) - - val bytes = fory.serialize(event) - val deserialized = fory.deserialize(bytes) - - deserialized.shouldBeInstanceOf>() - deserialized.event.shouldBeInstanceOf>() - val upsert = deserialized.event as Upsert<*, *> - upsert.key shouldBe "3" - upsert.oldValue shouldBe "B" - upsert.newValue shouldBe "C" - } - } - }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt index 2f17993..9996491 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMapTest.kt @@ -61,6 +61,20 @@ class FlowMapTest : } } + test("FlowMap asFlowWithState running fold to map") { + val map = mutableFlowMapOf("1" to "A", "2" to "B") + + map.asFlowWithState().runningFoldToMap().test { + awaitItem() shouldContainExactly mapOf("1" to "A", "2" to "B") + + map.put("3", "C") + awaitItem() shouldContainExactly mapOf("1" to "A", "2" to "B", "3" to "C") + + map.remove("1") + awaitItem() shouldContainExactly mapOf("2" to "B", "3" to "C") + } + } + test("FlowMap asFlow with predicate") { val map = mutableFlowMapOf("1" to "AL", "2" to "B") diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializationTest.kt new file mode 100644 index 0000000..ad77824 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializationTest.kt @@ -0,0 +1,58 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.FlowMapStreamEvent +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Upsert +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.maps.shouldContainExactly +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeInstanceOf +import kotlinx.collections.immutable.persistentMapOf +import org.apache.fory.Fory +import org.apache.fory.config.Language + +class FlowMapStreamEventSerializationTest : + FunSpec({ + val fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .build() + .registerDataSourceSerializers() + + test("InitialState (raw PersistentMap)") { + val map = persistentMapOf("1" to "A", "2" to "B") + val initialState = FlowMapStreamEvent.InitialState(map) + + val bytes = fory.serialize(initialState) + val deserialized = fory.deserialize(bytes) + + deserialized.shouldBeInstanceOf>() + deserialized.map shouldContainExactly mapOf("1" to "A", "2" to "B") + } + + test("EventUpdate") { + val event = FlowMapStreamEvent.EventUpdate(Upsert("3", "B", "C")) + + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + + deserialized.shouldBeInstanceOf>() + deserialized.event.shouldBeInstanceOf>() + val upsert = deserialized.event as Upsert<*, *> + upsert.key shouldBe "3" + upsert.oldValue shouldBe "B" + upsert.newValue shouldBe "C" + } + + test("InitialState with PersistentOrderedMap") { + val map = persistentMapOf("Z" to 1, "A" to 2, "M" to 3) + val initialState = FlowMapStreamEvent.InitialState(map) + + val bytes = fory.serialize(initialState) + val deserialized = fory.deserialize(bytes) + + deserialized.shouldBeInstanceOf>() + deserialized.map shouldContainExactly mapOf("Z" to 1, "A" to 2, "M" to 3) + deserialized.map.keys.toList() shouldBe listOf("Z", "A", "M") + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializationTest.kt new file mode 100644 index 0000000..7b8882f --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializationTest.kt @@ -0,0 +1,40 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.MapEvent +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Removed +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Upsert +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import org.apache.fory.Fory +import org.apache.fory.config.Language + +class MapEventSerializationTest : + FunSpec({ + val fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .build() + .registerDataSourceSerializers() + + test("Populated") { + val event = MapEvent.Populated + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe MapEvent.Populated + } + + test("Upsert") { + val event = Upsert("key", "old", "new") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Removed") { + val event = Removed("key", "old") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentCollectionSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentCollectionSerializationTest.kt new file mode 100644 index 0000000..79c33e8 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentCollectionSerializationTest.kt @@ -0,0 +1,51 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.maps.shouldContainExactly +import kotlinx.collections.immutable.persistentHashMapOf +import kotlinx.collections.immutable.persistentHashSetOf +import kotlinx.collections.immutable.persistentMapOf +import kotlinx.collections.immutable.persistentSetOf +import org.apache.fory.Fory +import org.apache.fory.config.Language + +class PersistentCollectionSerializationTest : + FunSpec({ + val fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .build() + .registerDataSourceSerializers() + + test("PersistentHashMap") { + val map = persistentHashMapOf("1" to "A", "2" to "B") + val bytes = fory.serialize(map) + val deserialized = fory.deserialize(bytes) as Map + deserialized shouldContainExactly mapOf("1" to "A", "2" to "B") + } + + test("PersistentOrderedMap") { + val map = persistentMapOf("Z" to 1, "A" to 2, "M" to 3) + val bytes = fory.serialize(map) + val deserialized = fory.deserialize(bytes) as Map + deserialized shouldContainExactly mapOf("Z" to 1, "A" to 2, "M" to 3) + deserialized.keys.toList() shouldContainExactly listOf("Z", "A", "M") + } + + test("PersistentHashSet") { + val set = persistentHashSetOf("1", "2") + val bytes = fory.serialize(set) + val deserialized = fory.deserialize(bytes) as Set + deserialized shouldContainExactly setOf("1", "2") + } + + test("PersistentOrderedSet") { + val set = persistentSetOf("Z", "A", "M") + val bytes = fory.serialize(set) + val deserialized = fory.deserialize(bytes) as Set + deserialized shouldContainExactly setOf("Z", "A", "M") + deserialized.toList() shouldContainExactly listOf("Z", "A", "M") + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializationTest.kt new file mode 100644 index 0000000..b6420eb --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializationTest.kt @@ -0,0 +1,38 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.SetEvent +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import org.apache.fory.Fory +import org.apache.fory.config.Language + +class SetEventSerializationTest : + FunSpec({ + val fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .build() + .registerDataSourceSerializers() + + test("Populated") { + val event = SetEvent.Populated + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Insert") { + val event = SetEvent.EntryEvent.Insert("value") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Removed") { + val event = SetEvent.EntryEvent.Removed("value") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializationTest.kt new file mode 100644 index 0000000..88b50b9 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializationTest.kt @@ -0,0 +1,38 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import org.apache.fory.Fory +import org.apache.fory.config.Language + +class SimpleMapEventSerializationTest : + FunSpec({ + val fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .build() + .registerDataSourceSerializers() + + test("Populated") { + val event = SimpleMapEvent.Populated + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Upsert") { + val event = SimpleMapEvent.EntryEvent.Upsert("key", "value") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Removed") { + val event = SimpleMapEvent.EntryEvent.Removed("key") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializationTest.kt new file mode 100644 index 0000000..afb9569 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializationTest.kt @@ -0,0 +1,31 @@ +package com.caplin.integration.datasourcex.util.serialization.fory + +import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import org.apache.fory.Fory +import org.apache.fory.config.Language + +class ValueOrCompletionSerializationTest : + FunSpec({ + val fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .build() + .registerDataSourceSerializers() + + test("Value") { + val event = ValueOrCompletion.Value("value") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Completion") { + val event = ValueOrCompletion.Completion(null) + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializationTest.kt new file mode 100644 index 0000000..be280c3 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializationTest.kt @@ -0,0 +1,52 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.FlowMapStreamEvent +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Upsert +import com.caplin.integration.datasourcex.util.flow.mutableFlowMapOf +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.maps.shouldContainExactly +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeInstanceOf + +class FlowMapStreamEventSerializationTest : + FunSpec({ + val mapper: ObjectMapper = jacksonObjectMapper().registerDataSourceModule() + + test("InitialState") { + val map = mutableFlowMapOf("1" to "A", "2" to "B") + val initialState: FlowMapStreamEvent = + FlowMapStreamEvent.InitialState(map.asMap()) + + val json = mapper.writeValueAsString(initialState) + val deserialized: FlowMapStreamEvent = + mapper.readValue( + json, + object : TypeReference>() {}, + ) + + deserialized.shouldBeInstanceOf>() + deserialized.map shouldContainExactly mapOf("1" to "A", "2" to "B") + } + + test("EventUpdate") { + val event: FlowMapStreamEvent = + FlowMapStreamEvent.EventUpdate(Upsert("3", "B", "C")) + + val json = mapper.writeValueAsString(event) + val deserialized: FlowMapStreamEvent = + mapper.readValue( + json, + object : TypeReference>() {}, + ) + + deserialized.shouldBeInstanceOf>() + deserialized.event.shouldBeInstanceOf>() + val upsert = deserialized.event + upsert.key shouldBe "3" + upsert.oldValue shouldBe "B" + upsert.newValue shouldBe "C" + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializationTest.kt new file mode 100644 index 0000000..cf74b48 --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializationTest.kt @@ -0,0 +1,39 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.MapEvent +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Removed +import com.caplin.integration.datasourcex.util.flow.MapEvent.EntryEvent.Upsert +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe + +class MapEventSerializationTest : + FunSpec({ + val mapper: ObjectMapper = jacksonObjectMapper().registerDataSourceModule() + + test("Populated") { + val event: MapEvent = MapEvent.Populated + val json = mapper.writeValueAsString(event) + val deserialized: MapEvent = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe MapEvent.Populated + } + + test("Upsert") { + val event: MapEvent = Upsert("key", "old", "new") + val json = mapper.writeValueAsString(event) + val deserialized: MapEvent = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + + test("Removed") { + val event: MapEvent = Removed("key", "old") + val json = mapper.writeValueAsString(event) + val deserialized: MapEvent = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializationTest.kt new file mode 100644 index 0000000..79f8bcd --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializationTest.kt @@ -0,0 +1,34 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.SetEvent +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe + +class SetEventSerializationTest : + FunSpec({ + val mapper: ObjectMapper = jacksonObjectMapper().registerDataSourceModule() + + test("Populated") { + val event: SetEvent = SetEvent.Populated + val json = mapper.writeValueAsString(event) + val deserialized = mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe SetEvent.Populated + } + + test("Insert") { + val event: SetEvent = SetEvent.EntryEvent.Insert("value") + val json = mapper.writeValueAsString(event) + val deserialized = mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + + test("Removed") { + val event: SetEvent = SetEvent.EntryEvent.Removed("value") + val json = mapper.writeValueAsString(event) + val deserialized = mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializationTest.kt new file mode 100644 index 0000000..9a8da5a --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializationTest.kt @@ -0,0 +1,37 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe + +class SimpleMapEventSerializationTest : + FunSpec({ + val mapper: ObjectMapper = jacksonObjectMapper().registerDataSourceModule() + + test("Populated") { + val event: SimpleMapEvent = SimpleMapEvent.Populated + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe SimpleMapEvent.Populated + } + + test("Upsert") { + val event: SimpleMapEvent = SimpleMapEvent.EntryEvent.Upsert("key", "value") + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + + test("Removed") { + val event: SimpleMapEvent = SimpleMapEvent.EntryEvent.Removed("key") + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializationTest.kt new file mode 100644 index 0000000..47d08fd --- /dev/null +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializationTest.kt @@ -0,0 +1,41 @@ +package com.caplin.integration.datasourcex.util.serialization.jackson + +import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeInstanceOf + +class ValueOrCompletionSerializationTest : + FunSpec({ + val mapper: ObjectMapper = jacksonObjectMapper().registerDataSourceModule() + + test("Value") { + val event: ValueOrCompletion = ValueOrCompletion.Value("value") + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + + test("Completion (null)") { + val event: ValueOrCompletion = ValueOrCompletion.Completion(null) + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference>() {}) + deserialized.shouldBeInstanceOf() + deserialized.throwable shouldBe null + } + + test("Completion (error)") { + val event: ValueOrCompletion = + ValueOrCompletion.Completion(RuntimeException("error")) + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference>() {}) + deserialized.shouldBeInstanceOf() + deserialized.throwable?.message shouldBe "error" + } + }) From 63ee6d70f47f67a96255de3ca5fb3f68c7ca554d Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Wed, 4 Mar 2026 18:12:38 +0000 Subject: [PATCH 09/14] Specify serializers for all subtypes --- .../serialization/fory/MapEventSerializer.kt | 2 +- .../serialization/fory/SetEventSerializer.kt | 2 +- .../fory/SimpleMapEventSerializer.kt | 2 +- .../fory/ValueOrCompletionSerializer.kt | 2 +- .../serialization/jackson/DataSourceModule.kt | 94 +++++++++++++++++-- 5 files changed, 89 insertions(+), 13 deletions(-) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializer.kt index 3fbf879..79244a1 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializer.kt @@ -34,7 +34,7 @@ internal class MapEventSerializer(fory: Fory, type: Class>) : } override fun read(buffer: MemoryBuffer): MapEvent<*, *> { - return when (Type.values()[buffer.readByte().toInt()]) { + return when (Type.entries[buffer.readByte().toInt()]) { Type.POPULATED -> MapEvent.Populated Type.UPSERT -> { val key = fory.readRef(buffer) as Any diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializer.kt index d917445..3a2a020 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializer.kt @@ -31,7 +31,7 @@ internal class SetEventSerializer(fory: Fory, type: Class>) : } override fun read(buffer: MemoryBuffer): SetEvent<*> { - return when (Type.values()[buffer.readByte().toInt()]) { + return when (Type.entries[buffer.readByte().toInt()]) { Type.POPULATED -> SetEvent.Populated Type.INSERT -> { val value = fory.readRef(buffer) as Any diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializer.kt index 741007c..f1cee8e 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializer.kt @@ -32,7 +32,7 @@ internal class SimpleMapEventSerializer(fory: Fory, type: Class { - return when (Type.values()[buffer.readByte().toInt()]) { + return when (Type.entries[buffer.readByte().toInt()]) { Type.POPULATED -> SimpleMapEvent.Populated Type.UPSERT -> { val key = fory.readRef(buffer) as Any diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializer.kt index e4e4adb..d28c0e0 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializer.kt @@ -28,7 +28,7 @@ internal class ValueOrCompletionSerializer(fory: Fory, type: Class { - return when (Type.values()[buffer.readByte().toInt()]) { + return when (Type.entries[buffer.readByte().toInt()]) { Type.VALUE -> { val value = fory.readRef(buffer) as Any ValueOrCompletion.Value(value) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt index dd5b286..64f0e75 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt @@ -16,22 +16,98 @@ fun ObjectMapper.registerDataSourceModule(): ObjectMapper = registerModule(DataS * without requiring annotations on the classes themselves. */ object DataSourceModule : SimpleModule() { - private fun readResolve(): Any = DataSourceModule + @Suppress("unused") private fun readResolve(): Any = DataSourceModule init { addSerializer(FlowMapStreamEvent::class.java, FlowMapStreamEventSerializer()) addDeserializer(FlowMapStreamEvent::class.java, FlowMapStreamEventDeserializer()) - addSerializer(MapEvent::class.java, MapEventSerializer()) - addDeserializer(MapEvent::class.java, MapEventDeserializer()) + val mapEventSerializer = MapEventSerializer() + val mapEventDeserializer = MapEventDeserializer() - addSerializer(SimpleMapEvent::class.java, SimpleMapEventSerializer()) - addDeserializer(SimpleMapEvent::class.java, SimpleMapEventDeserializer()) + addSerializer(MapEvent::class.java, mapEventSerializer) + addDeserializer(MapEvent::class.java, mapEventDeserializer) - addSerializer(SetEvent::class.java, SetEventSerializer()) - addDeserializer(SetEvent::class.java, SetEventDeserializer()) + @Suppress("UNCHECKED_CAST") + addSerializer( + MapEvent.EntryEvent::class.java, + mapEventSerializer + as com.fasterxml.jackson.databind.JsonSerializer>, + ) + @Suppress("UNCHECKED_CAST") + addDeserializer( + MapEvent.EntryEvent::class.java, + mapEventDeserializer + as com.fasterxml.jackson.databind.JsonDeserializer>, + ) - addSerializer(ValueOrCompletion::class.java, ValueOrCompletionSerializer()) - addDeserializer(ValueOrCompletion::class.java, ValueOrCompletionDeserializer()) + val simpleMapEventSerializer = SimpleMapEventSerializer() + val simpleMapEventDeserializer = SimpleMapEventDeserializer() + + addSerializer(SimpleMapEvent::class.java, simpleMapEventSerializer) + addDeserializer(SimpleMapEvent::class.java, simpleMapEventDeserializer) + + @Suppress("UNCHECKED_CAST") + addSerializer( + SimpleMapEvent.EntryEvent::class.java, + simpleMapEventSerializer + as com.fasterxml.jackson.databind.JsonSerializer>, + ) + @Suppress("UNCHECKED_CAST") + addDeserializer( + SimpleMapEvent.EntryEvent::class.java, + simpleMapEventDeserializer + as com.fasterxml.jackson.databind.JsonDeserializer>, + ) + + val setEventSerializer = SetEventSerializer() + val setEventDeserializer = SetEventDeserializer() + + addSerializer(SetEvent::class.java, setEventSerializer) + addDeserializer(SetEvent::class.java, setEventDeserializer) + + @Suppress("UNCHECKED_CAST") + addSerializer( + SetEvent.EntryEvent::class.java, + setEventSerializer as com.fasterxml.jackson.databind.JsonSerializer>, + ) + @Suppress("UNCHECKED_CAST") + addDeserializer( + SetEvent.EntryEvent::class.java, + setEventDeserializer + as com.fasterxml.jackson.databind.JsonDeserializer>, + ) + + val valueOrCompletionSerializer = ValueOrCompletionSerializer() + val valueOrCompletionDeserializer = ValueOrCompletionDeserializer() + + addSerializer(ValueOrCompletion::class.java, valueOrCompletionSerializer) + addDeserializer(ValueOrCompletion::class.java, valueOrCompletionDeserializer) + + @Suppress("UNCHECKED_CAST") + addSerializer( + ValueOrCompletion.Value::class.java, + valueOrCompletionSerializer + as com.fasterxml.jackson.databind.JsonSerializer>, + ) + @Suppress("UNCHECKED_CAST") + addDeserializer( + ValueOrCompletion.Value::class.java, + valueOrCompletionDeserializer + as com.fasterxml.jackson.databind.JsonDeserializer>, + ) + + @Suppress("UNCHECKED_CAST") + addSerializer( + ValueOrCompletion.Completion::class.java, + valueOrCompletionSerializer + as com.fasterxml.jackson.databind.JsonSerializer, + ) + @Suppress("UNCHECKED_CAST") + addDeserializer( + ValueOrCompletion.Completion::class.java, + valueOrCompletionDeserializer + as com.fasterxml.jackson.databind.JsonDeserializer, + ) } } From b84223c2ff3218a92a5ed5c06f9ddfe5159b9716 Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Wed, 4 Mar 2026 18:16:25 +0000 Subject: [PATCH 10/14] More serialization tests --- .../serialization/jackson/DataSourceModule.kt | 31 +++++++------------ .../fory/MapEventSerializationTest.kt | 16 ++++++++++ .../fory/SetEventSerializationTest.kt | 16 ++++++++++ .../fory/SimpleMapEventSerializationTest.kt | 18 +++++++++++ .../ValueOrCompletionSerializationTest.kt | 16 ++++++++++ .../jackson/MapEventSerializationTest.kt | 24 ++++++++++++++ .../jackson/SetEventSerializationTest.kt | 18 +++++++++++ .../SimpleMapEventSerializationTest.kt | 26 ++++++++++++++++ .../ValueOrCompletionSerializationTest.kt | 29 +++++++++++++++++ 9 files changed, 175 insertions(+), 19 deletions(-) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt index 64f0e75..44c09bb 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/DataSourceModule.kt @@ -5,6 +5,8 @@ import com.caplin.integration.datasourcex.util.flow.MapEvent import com.caplin.integration.datasourcex.util.flow.SetEvent import com.caplin.integration.datasourcex.util.flow.SimpleMapEvent import com.caplin.integration.datasourcex.util.flow.ValueOrCompletion +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonSerializer import com.fasterxml.jackson.databind.Module import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.module.SimpleModule @@ -31,14 +33,12 @@ object DataSourceModule : SimpleModule() { @Suppress("UNCHECKED_CAST") addSerializer( MapEvent.EntryEvent::class.java, - mapEventSerializer - as com.fasterxml.jackson.databind.JsonSerializer>, + mapEventSerializer as JsonSerializer>, ) @Suppress("UNCHECKED_CAST") addDeserializer( MapEvent.EntryEvent::class.java, - mapEventDeserializer - as com.fasterxml.jackson.databind.JsonDeserializer>, + mapEventDeserializer as JsonDeserializer>, ) val simpleMapEventSerializer = SimpleMapEventSerializer() @@ -50,14 +50,12 @@ object DataSourceModule : SimpleModule() { @Suppress("UNCHECKED_CAST") addSerializer( SimpleMapEvent.EntryEvent::class.java, - simpleMapEventSerializer - as com.fasterxml.jackson.databind.JsonSerializer>, + simpleMapEventSerializer as JsonSerializer>, ) @Suppress("UNCHECKED_CAST") addDeserializer( SimpleMapEvent.EntryEvent::class.java, - simpleMapEventDeserializer - as com.fasterxml.jackson.databind.JsonDeserializer>, + simpleMapEventDeserializer as JsonDeserializer>, ) val setEventSerializer = SetEventSerializer() @@ -69,13 +67,12 @@ object DataSourceModule : SimpleModule() { @Suppress("UNCHECKED_CAST") addSerializer( SetEvent.EntryEvent::class.java, - setEventSerializer as com.fasterxml.jackson.databind.JsonSerializer>, + setEventSerializer as JsonSerializer>, ) @Suppress("UNCHECKED_CAST") addDeserializer( SetEvent.EntryEvent::class.java, - setEventDeserializer - as com.fasterxml.jackson.databind.JsonDeserializer>, + setEventDeserializer as JsonDeserializer>, ) val valueOrCompletionSerializer = ValueOrCompletionSerializer() @@ -87,27 +84,23 @@ object DataSourceModule : SimpleModule() { @Suppress("UNCHECKED_CAST") addSerializer( ValueOrCompletion.Value::class.java, - valueOrCompletionSerializer - as com.fasterxml.jackson.databind.JsonSerializer>, + valueOrCompletionSerializer as JsonSerializer>, ) @Suppress("UNCHECKED_CAST") addDeserializer( ValueOrCompletion.Value::class.java, - valueOrCompletionDeserializer - as com.fasterxml.jackson.databind.JsonDeserializer>, + valueOrCompletionDeserializer as JsonDeserializer>, ) @Suppress("UNCHECKED_CAST") addSerializer( ValueOrCompletion.Completion::class.java, - valueOrCompletionSerializer - as com.fasterxml.jackson.databind.JsonSerializer, + valueOrCompletionSerializer as JsonSerializer, ) @Suppress("UNCHECKED_CAST") addDeserializer( ValueOrCompletion.Completion::class.java, - valueOrCompletionDeserializer - as com.fasterxml.jackson.databind.JsonDeserializer, + valueOrCompletionDeserializer as JsonDeserializer, ) } } diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializationTest.kt index 7b8882f..612fdf3 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializationTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/MapEventSerializationTest.kt @@ -37,4 +37,20 @@ class MapEventSerializationTest : val deserialized = fory.deserialize(bytes) deserialized shouldBe event } + + context("EntryEvent specifically") { + test("Upsert") { + val event: MapEvent.EntryEvent = Upsert("key", "old", "new") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Removed") { + val event: MapEvent.EntryEvent = Removed("key", "old") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializationTest.kt index b6420eb..e5c2123 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializationTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SetEventSerializationTest.kt @@ -35,4 +35,20 @@ class SetEventSerializationTest : val deserialized = fory.deserialize(bytes) deserialized shouldBe event } + + context("EntryEvent specifically") { + test("Insert") { + val event: SetEvent.EntryEvent = SetEvent.EntryEvent.Insert("value") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Removed") { + val event: SetEvent.EntryEvent = SetEvent.EntryEvent.Removed("value") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializationTest.kt index 88b50b9..950befc 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializationTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/SimpleMapEventSerializationTest.kt @@ -35,4 +35,22 @@ class SimpleMapEventSerializationTest : val deserialized = fory.deserialize(bytes) deserialized shouldBe event } + + context("EntryEvent specifically") { + test("Upsert") { + val event: SimpleMapEvent.EntryEvent = + SimpleMapEvent.EntryEvent.Upsert("key", "value") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Removed") { + val event: SimpleMapEvent.EntryEvent = + SimpleMapEvent.EntryEvent.Removed("key") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializationTest.kt index afb9569..1e11243 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializationTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/ValueOrCompletionSerializationTest.kt @@ -28,4 +28,20 @@ class ValueOrCompletionSerializationTest : val deserialized = fory.deserialize(bytes) deserialized shouldBe event } + + context("Value and Completion specifically") { + test("Value") { + val event: ValueOrCompletion.Value = ValueOrCompletion.Value("value") + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + + test("Completion") { + val event: ValueOrCompletion.Completion = ValueOrCompletion.Completion(null) + val bytes = fory.serialize(event) + val deserialized = fory.deserialize(bytes) + deserialized shouldBe event + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializationTest.kt index cf74b48..2626ac8 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializationTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/MapEventSerializationTest.kt @@ -36,4 +36,28 @@ class MapEventSerializationTest : mapper.readValue(json, object : TypeReference>() {}) deserialized shouldBe event } + + context("EntryEvent specifically") { + test("Upsert") { + val event: MapEvent.EntryEvent = Upsert("key", "old", "new") + val json = mapper.writeValueAsString(event) + val deserialized: MapEvent.EntryEvent = + mapper.readValue( + json, + object : TypeReference>() {}, + ) + deserialized shouldBe event + } + + test("Removed") { + val event: MapEvent.EntryEvent = Removed("key", "old") + val json = mapper.writeValueAsString(event) + val deserialized: MapEvent.EntryEvent = + mapper.readValue( + json, + object : TypeReference>() {}, + ) + deserialized shouldBe event + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializationTest.kt index 79f8bcd..757f218 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializationTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SetEventSerializationTest.kt @@ -31,4 +31,22 @@ class SetEventSerializationTest : val deserialized = mapper.readValue(json, object : TypeReference>() {}) deserialized shouldBe event } + + context("EntryEvent specifically") { + test("Insert") { + val event: SetEvent.EntryEvent = SetEvent.EntryEvent.Insert("value") + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + + test("Removed") { + val event: SetEvent.EntryEvent = SetEvent.EntryEvent.Removed("value") + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializationTest.kt index 9a8da5a..e3a0a91 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializationTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/SimpleMapEventSerializationTest.kt @@ -34,4 +34,30 @@ class SimpleMapEventSerializationTest : mapper.readValue(json, object : TypeReference>() {}) deserialized shouldBe event } + + context("EntryEvent specifically") { + test("Upsert") { + val event: SimpleMapEvent.EntryEvent = + SimpleMapEvent.EntryEvent.Upsert("key", "value") + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue( + json, + object : TypeReference>() {}, + ) + deserialized shouldBe event + } + + test("Removed") { + val event: SimpleMapEvent.EntryEvent = + SimpleMapEvent.EntryEvent.Removed("key") + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue( + json, + object : TypeReference>() {}, + ) + deserialized shouldBe event + } + } }) diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializationTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializationTest.kt index 47d08fd..c6b7b36 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializationTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/ValueOrCompletionSerializationTest.kt @@ -38,4 +38,33 @@ class ValueOrCompletionSerializationTest : deserialized.shouldBeInstanceOf() deserialized.throwable?.message shouldBe "error" } + + context("Value and Completion specifically") { + test("Value") { + val event: ValueOrCompletion.Value = ValueOrCompletion.Value("value") + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference>() {}) + deserialized shouldBe event + } + + test("Completion (null)") { + val event: ValueOrCompletion.Completion = ValueOrCompletion.Completion(null) + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference() {}) + deserialized.shouldBeInstanceOf() + deserialized.throwable shouldBe null + } + + test("Completion (error)") { + val event: ValueOrCompletion.Completion = + ValueOrCompletion.Completion(RuntimeException("error")) + val json = mapper.writeValueAsString(event) + val deserialized = + mapper.readValue(json, object : TypeReference() {}) + deserialized.shouldBeInstanceOf() + deserialized.throwable?.message shouldBe "error" + } + } }) From 8fd297505f4a75a1b140232a10647523e8e74711 Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Wed, 4 Mar 2026 18:41:24 +0000 Subject: [PATCH 11/14] Review fixes --- .../datasourcex/util/flow/FlowMap.kt | 45 ++++++++++++++----- .../datasourcex/util/flow/SetEvent.kt | 8 +++- .../datasourcex/util/flow/SimpleMapEvent.kt | 5 ++- .../datasourcex/util/flow/Timeout.kt | 2 +- .../fory/FlowMapStreamEventSerializer.kt | 5 +++ .../jackson/FlowMapStreamEventDeserializer.kt | 1 + .../jackson/FlowMapStreamEventSerializer.kt | 3 ++ .../util/flow/SimpleMapEventKtTest.kt | 1 + 8 files changed, 55 insertions(+), 15 deletions(-) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt index e8d8232..5ac6f3c 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/FlowMap.kt @@ -95,6 +95,11 @@ sealed interface FlowMapStreamEvent { /** Emitted for subsequent updates, containing only the delta ([event]). */ @JvmInline value class EventUpdate(val event: EntryEvent) : FlowMapStreamEvent + + /** Emitted when the map is cleared. */ + object Cleared : FlowMapStreamEvent { + override fun toString(): String = "Cleared" + } } interface MapFlow { @@ -150,6 +155,11 @@ fun Flow>.runningFoldToMap(): Flow { + map = persistentMapOf() + emit(map!!) + } + is FlowMapStreamEvent.EventUpdate -> { val currentMap = map ?: error("InitialState must be received before EventUpdate") when (val mapEvent = streamEvent.event) { @@ -180,6 +190,7 @@ private class FlowMapImpl(initialMap: PersistentMap) : private data class FlowMapEvent( val state: State, val events: List>, + val isClear: Boolean = false, ) private val state = MutableStateFlow(State(0L, initialMap)) @@ -281,9 +292,13 @@ private class FlowMapImpl(initialMap: PersistentMap) : emit(FlowMapStreamEvent.InitialState(flowMapEvent.state.map)) first = false } else { - val events = flowMapEvent.events - for (event in events) { - emit(FlowMapStreamEvent.EventUpdate(event)) + if (flowMapEvent.isClear) { + emit(FlowMapStreamEvent.Cleared) + } else { + val events = flowMapEvent.events + for (event in events) { + emit(FlowMapStreamEvent.EventUpdate(event)) + } } } } @@ -334,14 +349,20 @@ private class FlowMapImpl(initialMap: PersistentMap) : override fun containsKey(key: K): Boolean = state.value.map.containsKey(key) override fun putAll(from: Map) { - val (prev, next) = state.updateAndGetPrevAndNext { State(it.version + 1, it.map.putAll(from)) } - - val events = - from.mapNotNull { (key, newValue) -> - val oldValue = prev.map[key] - if (newValue != oldValue) Upsert(key, oldValue, newValue) else null + val (prev, next) = + state.updateAndGetPrevAndNext { + val nextMap = it.map.putAll(from) + if (nextMap === it.map) it else State(it.version + 1, nextMap) } - if (events.isNotEmpty()) signal.tryEmit(FlowMapEvent(next, events)) + + if (prev != next) { + val events = + from.mapNotNull { (key, newValue) -> + val oldValue = prev.map[key] + if (newValue != oldValue) Upsert(key, oldValue, newValue) else null + } + if (events.isNotEmpty()) signal.tryEmit(FlowMapEvent(next, events)) + } } override fun clear() { @@ -350,7 +371,9 @@ private class FlowMapImpl(initialMap: PersistentMap) : if (it.map.isEmpty()) it else State(it.version + 1, it.map.clear()) } if (prev.map.isNotEmpty()) - signal.tryEmit(FlowMapEvent(next, prev.map.map { Removed(it.key, it.value) })) + signal.tryEmit( + FlowMapEvent(next, prev.map.map { Removed(it.key, it.value) }, isClear = true) + ) } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt index 67b6e59..5caeee3 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt @@ -159,15 +159,19 @@ fun Flow>.flatMapLatestAndMerge( val jobs = ConcurrentHashMap() collect { setEvent -> when (setEvent) { - is EntryEvent -> { + is Insert -> { jobs[setEvent.value]?.cancelAndJoin() jobs[setEvent.value] = entryEventTransformer(setEvent) .onEach { send(it) } - .onCompletion { throwable -> if (throwable == null) jobs.remove(setEvent.value) } + .onCompletion { jobs.remove(setEvent.value) } .launchIn(this@channelFlow) } + is Removed -> { + jobs.remove(setEvent.value)?.cancelAndJoin() + } + is Populated -> {} } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEvent.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEvent.kt index 7f30974..e02156a 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEvent.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEvent.kt @@ -113,6 +113,8 @@ fun Flow>.runningFoldToMap( var populated = false var map = persistentMapOf() + if (emitPartials) emit(map) + collect { mapEvent -> var emit = false when (mapEvent) { @@ -130,8 +132,9 @@ fun Flow>.runningFoldToMap( } is Populated -> { + if (populated) error("Populated event already received") populated = true - if (!emitted || !emitPartials) emit = true + if (!emitted && !emitPartials) emit = true } } if (emit) { diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Timeout.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Timeout.kt index 10aac92..1e9a6f3 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Timeout.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Timeout.kt @@ -21,7 +21,7 @@ fun Flow.timeoutFirstOrDefault(millis: Long, default: () -> R): Flow send(result as R) } + receiveChannel.onReceiveCatching { result -> result.getOrNull()?.let { send(it as R) } } onTimeout(millis) { send(default()) } } receiveChannel.consumeEach { send(it as R) } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializer.kt index 58c7bc7..4166e58 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/FlowMapStreamEventSerializer.kt @@ -12,6 +12,7 @@ internal class FlowMapStreamEventSerializer(fory: Fory, type: Class) { @@ -24,6 +25,9 @@ internal class FlowMapStreamEventSerializer(fory: Fory, type: Class { + buffer.writeByte(Type.CLEARED.ordinal.toByte()) + } } } @@ -37,6 +41,7 @@ internal class FlowMapStreamEventSerializer(fory: Fory, type: Class FlowMapStreamEvent.EventUpdate(event) } + Type.CLEARED -> FlowMapStreamEvent.Cleared } } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventDeserializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventDeserializer.kt index d7609d8..4986d42 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventDeserializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventDeserializer.kt @@ -31,6 +31,7 @@ internal class FlowMapStreamEventDeserializer : as MapEvent.EntryEvent FlowMapStreamEvent.EventUpdate(event) } + "cleared" -> FlowMapStreamEvent.Cleared else -> throw JsonMappingException.from(p, "Unknown type: $type") } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializer.kt index 83896bf..91269a3 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/jackson/FlowMapStreamEventSerializer.kt @@ -24,6 +24,9 @@ internal class FlowMapStreamEventSerializer : gen.writeFieldName("event") provider.defaultSerializeValue(value.event, gen) } + is FlowMapStreamEvent.Cleared -> { + gen.writeStringField("type", "cleared") + } } gen.writeEndObject() } diff --git a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEventKtTest.kt b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEventKtTest.kt index 8aa170a..360ed43 100644 --- a/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEventKtTest.kt +++ b/util/src/test/kotlin/com/caplin/integration/datasourcex/util/flow/SimpleMapEventKtTest.kt @@ -42,6 +42,7 @@ class SimpleMapEventKtTest : ) .runningFoldToMap(emitPartials = true) .test { + awaitItem() shouldContainExactly emptyMap() awaitItem() shouldContainExactly mapOf("K" to "v1") awaitItem() shouldContainExactly mapOf("K" to "v2") awaitItem() shouldContainExactly mapOf("K" to "v2", "K2" to "v3") From 08c3fcdea26accc9dbfdbd32847bd0b0c7bcc78d Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Wed, 4 Mar 2026 19:13:31 +0000 Subject: [PATCH 12/14] Fix BindTest.kt --- .../caplin/integration/datasourcex/reactive/kotlin/BindTest.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/reactive/kotlin/src/test/kotlin/com/caplin/integration/datasourcex/reactive/kotlin/BindTest.kt b/reactive/kotlin/src/test/kotlin/com/caplin/integration/datasourcex/reactive/kotlin/BindTest.kt index e8d544c..427b554 100644 --- a/reactive/kotlin/src/test/kotlin/com/caplin/integration/datasourcex/reactive/kotlin/BindTest.kt +++ b/reactive/kotlin/src/test/kotlin/com/caplin/integration/datasourcex/reactive/kotlin/BindTest.kt @@ -469,6 +469,7 @@ class BindTest : delay(1) verify { cachedMessageFactory.createContainerMessage("/SUBJECT/1") } + verify { cachedMessageFactory.createGenericMessage("/SUBJECT/1-items/1") } sharedFlow.subscriptionCount.value shouldBeEqual 0 From 5d9868da972f772d84385bca1c50942ae6fa2868 Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Wed, 4 Mar 2026 19:45:19 +0000 Subject: [PATCH 13/14] Avoid another copy on write --- .../datasourcex/util/flow/Timeout.kt | 5 ++-- .../fory/PersistentHashMapSerializer.kt | 23 ++++++++++-------- .../fory/PersistentHashSetSerializer.kt | 23 ++++++++++-------- .../fory/PersistentOrderedMapSerializer.kt | 24 ++++++++++--------- .../fory/PersistentOrderedSetSerializer.kt | 23 ++++++++++-------- 5 files changed, 55 insertions(+), 43 deletions(-) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Timeout.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Timeout.kt index 1e9a6f3..fd65400 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Timeout.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/Timeout.kt @@ -6,6 +6,7 @@ import java.time.Duration import java.util.concurrent.TimeoutException import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.consumeEach +import kotlinx.coroutines.channels.onSuccess import kotlinx.coroutines.channels.produce import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow @@ -21,7 +22,7 @@ fun Flow.timeoutFirstOrDefault(millis: Long, default: () -> R): Flow result.getOrNull()?.let { send(it as R) } } + receiveChannel.onReceiveCatching { result -> result.onSuccess { send(it as R) } } onTimeout(millis) { send(default()) } } receiveChannel.consumeEach { send(it as R) } @@ -45,7 +46,7 @@ fun Flow.timeoutFirstOrDefault(millis: Long, default: R): Flow = * If the upstream emits no first event within [duration] it will emit the event [default] followed * by all later emissions from the upstream. */ -fun Flow.timeoutFirstOrDefault(duration: Duration, default: R): Flow = +fun Flow.timeoutFirstOrDefault(duration: Duration, default: R): Flow = timeoutFirstOrDefault(duration.toMillis(), default) /** diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashMapSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashMapSerializer.kt index 8c68d5e..36cf3c2 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashMapSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashMapSerializer.kt @@ -8,24 +8,27 @@ import org.apache.fory.serializer.collection.MapSerializer /** A Fory [MapSerializer] for [PersistentMap] (Hash implementation). */ internal class PersistentHashMapSerializer(fory: Fory, type: Class>) : - MapSerializer>(fory, type) { + MapSerializer>(fory, type, true) { - override fun write(buffer: MemoryBuffer, value: PersistentMap<*, *>) { - fory.writeRef(buffer, HashMap(value)) + override fun newMap(buffer: MemoryBuffer): MutableMap<*, *> { + val numElements = buffer.readVarUint32Small7() + setNumElements(numElements) + val map = HashMap(numElements) + refResolver.reference(map) + return map } - @Suppress("UNCHECKED_CAST") - override fun read(buffer: MemoryBuffer): PersistentMap<*, *> { - val map = fory.readRef(buffer) as Map - return map.toPersistentHashMap() + override fun newMap(map: Map<*, *>): MutableMap<*, *> { + return HashMap(map.size) } - override fun newMap(buffer: MemoryBuffer): MutableMap<*, *> { - return HashMap() + @Suppress("UNCHECKED_CAST") + override fun onMapRead(map: Map<*, *>): PersistentMap<*, *> { + return (map as Map).toPersistentHashMap() } @Suppress("UNCHECKED_CAST") - override fun onMapRead(map: Map<*, *>): PersistentMap<*, *> { + override fun onMapCopy(map: Map<*, *>): PersistentMap<*, *> { return (map as Map).toPersistentHashMap() } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashSetSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashSetSerializer.kt index a0ec381..fdc7493 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashSetSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentHashSetSerializer.kt @@ -8,24 +8,27 @@ import org.apache.fory.serializer.collection.CollectionSerializer /** A Fory [CollectionSerializer] for [PersistentSet] (Hash implementation). */ internal class PersistentHashSetSerializer(fory: Fory, type: Class>) : - CollectionSerializer>(fory, type) { + CollectionSerializer>(fory, type, true) { - override fun write(buffer: MemoryBuffer, value: PersistentSet<*>) { - fory.writeRef(buffer, HashSet(value)) + override fun newCollection(buffer: MemoryBuffer): MutableCollection<*> { + val numElements = buffer.readVarUint32Small7() + setNumElements(numElements) + val set = HashSet(numElements) + refResolver.reference(set) + return set } - @Suppress("UNCHECKED_CAST") - override fun read(buffer: MemoryBuffer): PersistentSet<*> { - val collection = fory.readRef(buffer) as Collection - return collection.toPersistentHashSet() + override fun newCollection(collection: Collection<*>): MutableCollection<*> { + return HashSet(collection.size) } - override fun newCollection(buffer: MemoryBuffer): MutableCollection<*> { - return HashSet() + @Suppress("UNCHECKED_CAST") + override fun onCollectionRead(collection: Collection<*>): PersistentSet<*> { + return (collection as Collection).toPersistentHashSet() } @Suppress("UNCHECKED_CAST") - override fun onCollectionRead(collection: Collection<*>): PersistentSet<*> { + fun onCollectionCopy(collection: Collection<*>): PersistentSet<*> { return (collection as Collection).toPersistentHashSet() } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedMapSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedMapSerializer.kt index f7f98ba..8a121b9 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedMapSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedMapSerializer.kt @@ -8,25 +8,27 @@ import org.apache.fory.serializer.collection.MapSerializer /** A Fory [MapSerializer] for [PersistentMap] (Ordered implementation). */ internal class PersistentOrderedMapSerializer(fory: Fory, type: Class>) : - MapSerializer>(fory, type) { + MapSerializer>(fory, type, true) { - override fun write(buffer: MemoryBuffer, value: PersistentMap<*, *>) { - fory.writeRef(buffer, LinkedHashMap(value)) + override fun newMap(buffer: MemoryBuffer): MutableMap<*, *> { + val numElements = buffer.readVarUint32Small7() + setNumElements(numElements) + val map = LinkedHashMap(numElements) + refResolver.reference(map) + return map } - @Suppress("UNCHECKED_CAST") - override fun read(buffer: MemoryBuffer): PersistentMap<*, *> { - val map = fory.readRef(buffer) as Map - return map.toPersistentMap() + override fun newMap(map: Map<*, *>): MutableMap<*, *> { + return LinkedHashMap(map.size) } - override fun newMap(buffer: MemoryBuffer): MutableMap<*, *> { - return LinkedHashMap() + @Suppress("UNCHECKED_CAST") + override fun onMapRead(map: Map<*, *>): PersistentMap<*, *> { + return (map as Map).toPersistentMap() } @Suppress("UNCHECKED_CAST") - override fun onMapRead(map: Map<*, *>): PersistentMap<*, *> { - // toPersistentMap() on a LinkedHashMap preserves order in kotlinx-collections-immutable + override fun onMapCopy(map: Map<*, *>): PersistentMap<*, *> { return (map as Map).toPersistentMap() } } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedSetSerializer.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedSetSerializer.kt index 25d7892..e23f46a 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedSetSerializer.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/serialization/fory/PersistentOrderedSetSerializer.kt @@ -8,24 +8,27 @@ import org.apache.fory.serializer.collection.CollectionSerializer /** A Fory [CollectionSerializer] for [PersistentSet] (Ordered implementation). */ internal class PersistentOrderedSetSerializer(fory: Fory, type: Class>) : - CollectionSerializer>(fory, type) { + CollectionSerializer>(fory, type, true) { - override fun write(buffer: MemoryBuffer, value: PersistentSet<*>) { - fory.writeRef(buffer, LinkedHashSet(value)) + override fun newCollection(buffer: MemoryBuffer): MutableCollection<*> { + val numElements = buffer.readVarUint32Small7() + setNumElements(numElements) + val set = LinkedHashSet(numElements) + refResolver.reference(set) + return set } - @Suppress("UNCHECKED_CAST") - override fun read(buffer: MemoryBuffer): PersistentSet<*> { - val collection = fory.readRef(buffer) as Collection - return collection.toPersistentSet() + override fun newCollection(collection: Collection<*>): MutableCollection<*> { + return LinkedHashSet(collection.size) } - override fun newCollection(buffer: MemoryBuffer): MutableCollection<*> { - return LinkedHashSet() + @Suppress("UNCHECKED_CAST") + override fun onCollectionRead(collection: Collection<*>): PersistentSet<*> { + return (collection as Collection).toPersistentSet() } @Suppress("UNCHECKED_CAST") - override fun onCollectionRead(collection: Collection<*>): PersistentSet<*> { + fun onCollectionCopy(collection: Collection<*>): PersistentSet<*> { return (collection as Collection).toPersistentSet() } } From 7253786e86025b9fdb411ff71b603b696118b86a Mon Sep 17 00:00:00 2001 From: Ross Anderson Date: Wed, 4 Mar 2026 20:01:27 +0000 Subject: [PATCH 14/14] Add more kdoc --- util/api/datasourcex-util.api | 5 ++++ .../datasourcex/util/AntPatternNamespace.kt | 15 +++++++++++ .../integration/datasourcex/util/KLogger.kt | 26 +++++++++++++++++++ .../datasourcex/util/ReadWriteLock.kt | 12 +++++++++ .../util/SimpleDataSourceConfig.kt | 15 +++++++++++ .../util/SimpleDataSourceFactory.kt | 8 ++++++ .../datasourcex/util/flow/SetEvent.kt | 19 ++++++++++++++ 7 files changed, 100 insertions(+) diff --git a/util/api/datasourcex-util.api b/util/api/datasourcex-util.api index d686a63..22afb5a 100644 --- a/util/api/datasourcex-util.api +++ b/util/api/datasourcex-util.api @@ -191,6 +191,11 @@ public final class com/caplin/integration/datasourcex/util/flow/FlowMapKt { public abstract interface class com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent { } +public final class com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent$Cleared : com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent { + public static final field INSTANCE Lcom/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent$Cleared; + public fun toString ()Ljava/lang/String; +} + public final class com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent$EventUpdate : com/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent { public static final synthetic fun box-impl (Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent;)Lcom/caplin/integration/datasourcex/util/flow/FlowMapStreamEvent$EventUpdate; public static fun constructor-impl (Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent;)Lcom/caplin/integration/datasourcex/util/flow/MapEvent$EntryEvent; diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/AntPatternNamespace.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/AntPatternNamespace.kt index 943a313..569dfa9 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/AntPatternNamespace.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/AntPatternNamespace.kt @@ -31,6 +31,13 @@ class AntPatternNamespace(pattern: String) : Namespace { } } + /** + * Represents a mapping between a from-pattern and a to-pattern, used for injecting user-specific + * information into subjects requested by Liberator. + * + * @property fromPattern The pattern used to match the incoming subject. + * @property toPattern The pattern used to map to the destination subject. + */ class ObjectMap(val fromPattern: String, val toPattern: String) { operator fun component1(): String = fromPattern @@ -59,12 +66,20 @@ class AntPatternNamespace(pattern: String) : Namespace { } } + /** The Ant-style path pattern used by this namespace. */ val pattern = pattern.removeSuffix("/") private val matcher = AntRegexPathMatcher(pattern) override fun match(subject: String): Boolean = matcher.regex.matchEntire(subject) != null + /** + * Extracts path variables from a matching subject. + * + * @param subject The subject to extract variables from. Must match the [pattern]. + * @return A map of path variable names to their extracted values. + * @throws IllegalStateException If the subject does not match the pattern. + */ fun extractPathVariables(subject: String): Map { val groups = checkNotNull(matcher.regex.matchEntire(subject)) { diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/KLogger.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/KLogger.kt index 7891df0..3369586 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/KLogger.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/KLogger.kt @@ -10,45 +10,71 @@ import org.slf4j.LoggerFactory @JvmInline value class KLogger(private val logger: Logger) { + /** Logs an error message lazily evaluated from [message] if error logging is enabled. */ fun error(message: () -> Any?) { if (logger.isErrorEnabled) logger.error(message().toString()) } + /** Logs a warning message lazily evaluated from [message] if warn logging is enabled. */ fun warn(message: () -> Any?) { if (logger.isWarnEnabled) logger.warn(message().toString()) } + /** Logs an info message lazily evaluated from [message] if info logging is enabled. */ fun info(message: () -> Any?) { if (logger.isInfoEnabled) logger.info(message().toString()) } + /** Logs a debug message lazily evaluated from [message] if debug logging is enabled. */ fun debug(message: () -> Any?) { if (logger.isDebugEnabled) logger.debug(message().toString()) } + /** Logs a trace message lazily evaluated from [message] if trace logging is enabled. */ fun trace(message: () -> Any?) { if (logger.isTraceEnabled) logger.trace(message().toString()) } + /** + * Logs an error message with a [throwable], lazily evaluated from [message], if error logging is + * enabled. + */ fun error(throwable: Throwable?, message: () -> Any?) { if (logger.isErrorEnabled) logger.error(message().toString(), throwable) } + /** + * Logs a warning message with a [throwable], lazily evaluated from [message], if warn logging is + * enabled. + */ fun warn(throwable: Throwable?, message: () -> Any?) { if (logger.isWarnEnabled) logger.warn(message().toString(), throwable) } + /** + * Logs an info message with a [throwable], lazily evaluated from [message], if info logging is + * enabled. + */ fun info(throwable: Throwable?, message: () -> Any?) { if (logger.isInfoEnabled) logger.info(message().toString(), throwable) } + /** + * Logs a debug message with a [throwable], lazily evaluated from [message], if debug logging is + * enabled. + */ fun debug(throwable: Throwable?, message: () -> Any?) { if (logger.isDebugEnabled) logger.debug(message().toString(), throwable) } + /** + * Logs a trace message with a [throwable], lazily evaluated from [message], if trace logging is + * enabled. + */ fun trace(throwable: Throwable?, message: () -> Any?) { if (logger.isTraceEnabled) logger.trace(message().toString(), throwable) } } +/** Returns a [KLogger] instance for the specified class [T]. */ inline fun getLogger(): KLogger = KLogger(LoggerFactory.getLogger(T::class.java)) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/ReadWriteLock.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/ReadWriteLock.kt index 0101cb4..9345406 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/ReadWriteLock.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/ReadWriteLock.kt @@ -11,8 +11,16 @@ class ReadWriteLock { private val readerMutex = Mutex() private val writerMutex = Mutex() + /** + * Executes the given [block] of code within a write lock. Suspends until the write lock can be + * acquired. + */ suspend fun withWriteLock(block: suspend () -> R): R = writerMutex.withLock(null) { block() } + /** + * Executes the given [block] of code within a read lock. Suspends until the read lock can be + * acquired. + */ suspend fun withReadLock(block: suspend () -> R): R = withContext(NonCancellable) { readLock() @@ -23,6 +31,7 @@ class ReadWriteLock { } } + /** Acquires a read lock. Suspends if a write lock is currently held. */ suspend fun readLock() = withContext(NonCancellable) { readerMutex.withLock { @@ -31,6 +40,7 @@ class ReadWriteLock { } } + /** Releases a previously acquired read lock. */ suspend fun readUnlock() = withContext(NonCancellable) { readerMutex.withLock { @@ -39,7 +49,9 @@ class ReadWriteLock { } } + /** Acquires a write lock. Suspends if any read or write locks are currently held. */ suspend fun writeLock() = writerMutex.lock() + /** Releases a previously acquired write lock. */ fun writeUnlock() = writerMutex.unlock() } diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceConfig.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceConfig.kt index 481a364..21619cf 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceConfig.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceConfig.kt @@ -3,14 +3,22 @@ package com.caplin.integration.datasourcex.util import java.nio.file.Path import java.util.UUID +/** Configuration for creating a DataSource via [SimpleDataSourceFactory]. */ sealed interface SimpleDataSourceConfig { + /** The directory where log files will be written. */ val logDirectory: Path? + /** The name of the DataSource. */ val name: String + /** The local label for the DataSource. */ val localLabel: String + /** Any extra configuration to append to the configuration string. */ val extraConfig: String? + /** Configuration for a DataSource that connects to a discovery service. */ class Discovery( + /** The hostname of the discovery service. */ val hostname: String, + /** The cluster name to join. */ val clusterName: String = "caplin", override val name: String, override val logDirectory: Path?, @@ -23,14 +31,19 @@ sealed interface SimpleDataSourceConfig { } } + /** Configuration for a DataSource that connects to specific peers. */ class Peer( override val name: String, override val logDirectory: Path? = null, override val localLabel: String = "$name-${UUID.randomUUID()}", override val extraConfig: String? = null, + /** Optional configuration for accepting incoming connections. */ val incoming: Incoming? = null, + /** List of outgoing peer connections. */ val outgoing: List = emptyList(), + /** List of services required before this DataSource becomes active. */ val requiredServices: List = emptyList(), + /** Whether to override development mode checks. */ val devOverride: Boolean = false, ) : SimpleDataSourceConfig { @@ -40,12 +53,14 @@ sealed interface SimpleDataSourceConfig { } } + /** Configuration for an outgoing peer connection. */ class Outgoing(val hostname: String, val port: Int, val isWebsocket: Boolean) { override fun toString(): String { return "Outgoing(hostname='$hostname', port=$port, isWebsocket=$isWebsocket)" } } + /** Configuration for accepting incoming connections. */ class Incoming(val port: Int, val isWebsocket: Boolean) { override fun toString(): String { return "Incoming(port=$port, isWebsocket=$isWebsocket)" diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt index 3fa96b1..4c41526 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/SimpleDataSourceFactory.kt @@ -10,12 +10,20 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import java.nio.file.Files import java.util.logging.Logger +/** + * A factory for creating [DataSource] instances from simplified configurations. Allows easy setup + * for tests and examples. + */ object SimpleDataSourceFactory { private const val MAX_PATH_LENGTH = 32 private val logger = getLogger() + /** + * The default [ObjectMapper] used for serializing and deserializing JSON payloads. It is + * pre-configured with the JavaTime module and DataSource serialization extensions. + */ val defaultObjectMapper: ObjectMapper = jacksonObjectMapper() .configure(WRITE_DATES_AS_TIMESTAMPS, false) diff --git a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt index 5caeee3..30cd39f 100644 --- a/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt +++ b/util/src/main/kotlin/com/caplin/integration/datasourcex/util/flow/SetEvent.kt @@ -15,19 +15,26 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onEach +/** Events representing a mutation to a [Set]. */ sealed interface SetEvent { + /** + * Indicates that a consistent view of the set has been emitted and only updates will be seen from + * now on. + */ object Populated : SetEvent { override fun toString(): String { return "Populated()" } } + /** Mutation event for a specific entry. */ sealed interface EntryEvent : SetEvent { val value: V operator fun component1(): V = value + /** An event indicating a value has been inserted into the set. */ class Insert(override val value: V) : EntryEvent { override fun equals(other: Any?): Boolean { @@ -48,6 +55,7 @@ sealed interface SetEvent { } } + /** An event indicating a value has been removed from the set. */ class Removed(override val value: V) : EntryEvent { override fun equals(other: Any?): Boolean { @@ -148,11 +156,22 @@ fun Flow>.runningFoldToSet( } } +/** + * Transforms a flow of sets into a merged flow by applying [entryEventTransformer] to each entry + * event (insert or remove). When a value is inserted, a new flow is created and merged. When a + * value is removed, the corresponding flow is cancelled. + */ @JvmName("flatMapLatestAndMergeSet") fun Flow>.flatMapLatestAndMerge( entryEventTransformer: (EntryEvent) -> Flow ): Flow = toEvents().flatMapLatestAndMerge(entryEventTransformer) +/** + * Transforms a flow of [SetEvent] into a merged flow by applying [entryEventTransformer] to each + * entry event. When an [Insert] event is received, a new flow is created and its emissions are + * merged into the resulting flow. When a [Removed] event is received, the previously created flow + * for that value is cancelled. + */ fun Flow>.flatMapLatestAndMerge( entryEventTransformer: (EntryEvent) -> Flow ) = channelFlow {