Skip to content

feat(lapis): add overall statistics to .*OverTime endpoints #1509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
fhennig wants to merge 1 commit into from
Closed

feat(lapis): add overall statistics to .*OverTime endpoints #1509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions lapis-e2e/test/queriesOverTime.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ describe('The /mutationsOverTime endpoint', () => {
],
],
totalCountsByDateRange: [22, 77, 0],
overallStatisticsByQuery: [
{ count: 58, coverage: 95, proportion: 58 / 95 },
{ count: 1, coverage: 95, proportion: 1 / 95 },
],
});
});

Expand Down Expand Up @@ -79,6 +83,14 @@ describe('The /mutationsOverTime endpoint', () => {
{ count: 58, coverage: 62 },
{ count: 14, coverage: 14 },
]);

expect(result.data.overallStatisticsByMutation).to.have.lengthOf(4);
const c241tStats = result.data.overallStatisticsByMutation![3];
expect(c241tStats).to.deep.equal({
count: 92,
coverage: 96,
proportion: 92 / 96,
});
});

it('returns an empty response if no mutations are given', async () => {
Expand Down Expand Up @@ -108,6 +120,7 @@ describe('The /mutationsOverTime endpoint', () => {
expect(result.data.data).to.have.lengthOf(0);
expect(result.data.dateRanges).to.have.lengthOf(3);
expect(result.data.mutations).to.have.lengthOf(0);
expect(result.data.overallStatisticsByMutation).to.have.lengthOf(0);
});

it('returns an empty response if no date ranges are given', async () => {
Expand All @@ -124,6 +137,7 @@ describe('The /mutationsOverTime endpoint', () => {
expect(result.data.data).to.have.lengthOf(0);
expect(result.data.dateRanges).to.have.lengthOf(0);
expect(result.data.mutations).to.have.lengthOf(4);
expect(result.data.overallStatisticsByMutation).to.have.lengthOf(0);
expect(result.info.dataVersion).to.exist;
});

Expand Down
41 changes: 41 additions & 0 deletions lapis/src/main/kotlin/org/genspectrum/lapis/model/mutationsOverTime/QueriesOverTimeModel.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ data class MutationsOverTimeResult(
description = "The list of total sample counts per date range",
)
var totalCountsByDateRange: List<Number>,
@param:Schema(
description = "Aggregated statistics per mutation, summed across all date ranges. " +
"One entry per mutation in the same order as the mutations array.",
)
var overallStatisticsByMutation: List<OverallStatistics>,
)

data class MutationsOverTimeCell(
Expand Down Expand Up @@ -93,6 +98,11 @@ data class QueriesOverTimeResult(
description = "The list of total sample counts per date range",
)
var totalCountsByDateRange: List<Number>,
@param:Schema(
description = "Aggregated statistics per query, summed across all date ranges. " +
"One entry per query in the same order as the queries array.",
)
var overallStatisticsByQuery: List<OverallStatistics>,
) {
fun toMutationsOverTimeResult() =
MutationsOverTimeResult(
Expand All @@ -107,6 +117,7 @@ data class QueriesOverTimeResult(
}
},
totalCountsByDateRange = totalCountsByDateRange,
overallStatisticsByMutation = overallStatisticsByQuery,
)
}

Expand All @@ -121,6 +132,15 @@ data class QueryOverTimeCell(
var coverage: Int,
)

data class OverallStatistics(
@param:Schema(description = "Total count across all date ranges")
var count: Int,
@param:Schema(description = "Total coverage across all date ranges")
var coverage: Int,
@param:Schema(description = "Proportion (count / coverage). Omitted if coverage is 0.")
var proportion: Double?,
)

@Component
class QueriesOverTimeModel(
private val siloClient: SiloClient,
Expand Down Expand Up @@ -264,6 +284,7 @@ class QueriesOverTimeModel(
dateRanges = dateRanges,
data = emptyList(),
totalCountsByDateRange = emptyList(),
overallStatisticsByQuery = emptyList(),
)
}

Expand Down Expand Up @@ -326,11 +347,14 @@ class QueriesOverTimeModel(
}
dataVersion.dataVersion = dataVersions.first()

val overallStats = computeOverallStatistics(dataWithDataVersions.map { it.second })

return QueriesOverTimeResult(
queries = queryItems.map(mutationToStringFn),
dateRanges = dateRanges,
data = dataWithDataVersions.map { it.second },
totalCountsByDateRange = totalCountsByDateRange,
overallStatisticsByQuery = overallStats,
)
}

Expand Down Expand Up @@ -419,6 +443,23 @@ class QueriesOverTimeModel(
}.takeIf { it >= 0 }
}

/**
* Aggregates statistics across all date ranges for each query/mutation.
* For each row in the data (representing a mutation/query), sums up the counts and coverage
* across all columns (representing date ranges).
*/
private fun computeOverallStatistics(data: List<List<QueryOverTimeCell>>): List<OverallStatistics> {
return data.map { row ->
val totalCount = row.sumOf { it.count }
val totalCoverage = row.sumOf { it.coverage }
OverallStatistics(
count = totalCount,
coverage = totalCoverage,
proportion = if (totalCoverage > 0) totalCount.toDouble() / totalCoverage else null,
)
}
}

@PreDestroy
fun shutdownThreadPool() {
threadPool.shutdown()
Expand Down
34 changes: 34 additions & 0 deletions lapis/src/test/kotlin/org/genspectrum/lapis/controller/QueriesOverTimeControllerTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import io.mockk.verify
import org.genspectrum.lapis.model.mutationsOverTime.DateRange
import org.genspectrum.lapis.model.mutationsOverTime.MutationsOverTimeCell
import org.genspectrum.lapis.model.mutationsOverTime.MutationsOverTimeResult
import org.genspectrum.lapis.model.mutationsOverTime.OverallStatistics
import org.genspectrum.lapis.model.mutationsOverTime.QueriesOverTimeModel
import org.genspectrum.lapis.model.mutationsOverTime.QueriesOverTimeResult
import org.genspectrum.lapis.model.mutationsOverTime.QueryOverTimeCell
Expand Down Expand Up @@ -66,6 +67,10 @@ class QueriesOverTimeControllerTest(
listOf(QueryOverTimeCell(count = 5, coverage = 50)),
),
totalCountsByDateRange = listOf(300),
overallStatisticsByQuery = listOf(
OverallStatistics(count = 10, coverage = 100, proportion = 0.1),
OverallStatistics(count = 5, coverage = 50, proportion = 0.1),
),
)

val queriesSlot = slot<List<QueryOverTimeItem>>()
Expand Down Expand Up @@ -111,6 +116,12 @@ class QueriesOverTimeControllerTest(
.andExpect(jsonPath("$.data.data[0][0].coverage").value(100))
.andExpect(jsonPath("$.data.data[1][0].count").value(5))
.andExpect(jsonPath("$.data.data[1][0].coverage").value(50))
.andExpect(jsonPath("$.data.overallStatisticsByQuery[0].count").value(10))
.andExpect(jsonPath("$.data.overallStatisticsByQuery[0].coverage").value(100))
.andExpect(jsonPath("$.data.overallStatisticsByQuery[0].proportion").value(0.1))
.andExpect(jsonPath("$.data.overallStatisticsByQuery[1].count").value(5))
.andExpect(jsonPath("$.data.overallStatisticsByQuery[1].coverage").value(50))
.andExpect(jsonPath("$.data.overallStatisticsByQuery[1].proportion").value(0.1))
.andExpect(jsonPath("$.info.dataVersion").value(1234))

verify(exactly = 1) { modelMock.evaluateQueriesOverTime(any(), any(), any(), any()) }
Expand All @@ -135,6 +146,7 @@ class QueriesOverTimeControllerTest(
dateRanges = listOf(DateRange(LocalDate.parse("2025-01-01"), LocalDate.parse("2025-01-31"))),
data = listOf(listOf(QueryOverTimeCell(count = 1, coverage = 2))),
totalCountsByDateRange = listOf(300),
overallStatisticsByQuery = listOf(OverallStatistics(count = 1, coverage = 2, proportion = 0.5)),
)

val mvcResult = mockMvc.perform(
Expand Down Expand Up @@ -246,6 +258,10 @@ class NucleotideMutationsOverTimeControllerTest(
listOf(MutationsOverTimeCell(count = 5, coverage = 50)),
),
totalCountsByDateRange = listOf(300),
overallStatisticsByMutation = listOf(
OverallStatistics(count = 10, coverage = 100, proportion = 0.1),
OverallStatistics(count = 5, coverage = 50, proportion = 0.1),
),
)

val mutationsSlot = slot<List<NucleotideMutation>>()
Expand Down Expand Up @@ -288,6 +304,12 @@ class NucleotideMutationsOverTimeControllerTest(
.andExpect(jsonPath("$.data.data[0][0].coverage").value(100))
.andExpect(jsonPath("$.data.data[1][0].count").value(5))
.andExpect(jsonPath("$.data.data[1][0].coverage").value(50))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[0].count").value(10))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[0].coverage").value(100))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[0].proportion").value(0.1))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[1].count").value(5))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[1].coverage").value(50))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[1].proportion").value(0.1))
.andExpect(jsonPath("$.info.dataVersion").value(1234))

verify(exactly = 1) { modelMock.evaluateNucleotideMutations(any(), any(), any(), any()) }
Expand All @@ -312,6 +334,7 @@ class NucleotideMutationsOverTimeControllerTest(
dateRanges = listOf(DateRange(LocalDate.parse("2025-01-01"), LocalDate.parse("2025-01-31"))),
data = listOf(listOf(MutationsOverTimeCell(count = 1, coverage = 2))),
totalCountsByDateRange = listOf(300),
overallStatisticsByMutation = listOf(OverallStatistics(count = 1, coverage = 2, proportion = 0.5)),
)

val mvcResult = mockMvc.perform(
Expand Down Expand Up @@ -421,6 +444,10 @@ class AminoAcidMutationsOverTimeControllerTest(
listOf(MutationsOverTimeCell(count = 5, coverage = 50)),
),
totalCountsByDateRange = listOf(300),
overallStatisticsByMutation = listOf(
OverallStatistics(count = 10, coverage = 100, proportion = 0.1),
OverallStatistics(count = 5, coverage = 50, proportion = 0.1),
),
)

val mutationsSlot = slot<List<AminoAcidMutation>>()
Expand Down Expand Up @@ -463,6 +490,12 @@ class AminoAcidMutationsOverTimeControllerTest(
.andExpect(jsonPath("$.data.data[0][0].coverage").value(100))
.andExpect(jsonPath("$.data.data[1][0].count").value(5))
.andExpect(jsonPath("$.data.data[1][0].coverage").value(50))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[0].count").value(10))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[0].coverage").value(100))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[0].proportion").value(0.1))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[1].count").value(5))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[1].coverage").value(50))
.andExpect(jsonPath("$.data.overallStatisticsByMutation[1].proportion").value(0.1))
.andExpect(jsonPath("$.info.dataVersion").value(1234))

verify(exactly = 1) { modelMock.evaluateAminoAcidMutations(any(), any(), any(), any()) }
Expand All @@ -487,6 +520,7 @@ class AminoAcidMutationsOverTimeControllerTest(
dateRanges = listOf(DateRange(LocalDate.parse("2025-01-01"), LocalDate.parse("2025-01-31"))),
data = listOf(listOf(MutationsOverTimeCell(count = 1, coverage = 2))),
totalCountsByDateRange = listOf(3),
overallStatisticsByMutation = listOf(OverallStatistics(count = 1, coverage = 2, proportion = 0.5)),
)

val mvcResult = mockMvc.perform(
Expand Down
19 changes: 19 additions & 0 deletions ...tlin/org/genspectrum/lapis/model/mutationsOverTime/AminoAcidMutationsOverTimeModelTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ AminoAcidMutationsOverTimeModelTest {
assertThat(result.data, equalTo(emptyList()))
assertThat(result.dateRanges, equalTo(dateRanges))
assertThat(result.totalCountsByDateRange, equalTo(emptyList()))
assertThat(result.overallStatisticsByMutation, equalTo(emptyList()))
assertThat(dataVersion.dataVersion, notNullValue())
}

Expand All @@ -103,6 +104,7 @@ AminoAcidMutationsOverTimeModelTest {
assertThat(result.data, equalTo(emptyList()))
assertThat(result.dateRanges, equalTo(emptyList()))
assertThat(result.totalCountsByDateRange, equalTo(emptyList()))
assertThat(result.overallStatisticsByMutation, equalTo(emptyList()))
assertThat(dataVersion.dataVersion, notNullValue())
}

Expand Down Expand Up @@ -188,6 +190,15 @@ AminoAcidMutationsOverTimeModelTest {
result.totalCountsByDateRange,
equalTo(listOf(10, 23)),
)
assertThat(
result.overallStatisticsByMutation,
equalTo(
listOf(
OverallStatistics(count = 3, coverage = 11, proportion = 3.0 / 11.0),
OverallStatistics(count = 7, coverage = 2, proportion = 7.0 / 2.0),
),
),
)
assertThat(dataVersion.dataVersion, notNullValue())
}

Expand Down Expand Up @@ -236,6 +247,14 @@ AminoAcidMutationsOverTimeModelTest {
),
)
assertThat(result.totalCountsByDateRange, equalTo(listOf(0, 0)))
assertThat(
result.overallStatisticsByMutation,
equalTo(
listOf(
OverallStatistics(count = 0, coverage = 0, proportion = null),
),
),
)
}

@Test
Expand Down
19 changes: 19 additions & 0 deletions ...lin/org/genspectrum/lapis/model/mutationsOverTime/NucleotideMutationsOverTimeModelTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ class NucleotideMutationsOverTimeModelTest {
assertThat(result.data, equalTo(emptyList()))
assertThat(result.dateRanges, equalTo(dateRanges))
assertThat(result.totalCountsByDateRange, equalTo(emptyList()))
assertThat(result.overallStatisticsByMutation, equalTo(emptyList()))
assertThat(dataVersion.dataVersion, notNullValue())
}

Expand All @@ -102,6 +103,7 @@ class NucleotideMutationsOverTimeModelTest {
assertThat(result.data, equalTo(emptyList()))
assertThat(result.dateRanges, equalTo(emptyList()))
assertThat(result.totalCountsByDateRange, equalTo(emptyList()))
assertThat(result.overallStatisticsByMutation, equalTo(emptyList()))
assertThat(dataVersion.dataVersion, notNullValue())
}

Expand Down Expand Up @@ -187,6 +189,15 @@ class NucleotideMutationsOverTimeModelTest {
result.totalCountsByDateRange,
equalTo(listOf(10, 23)),
)
assertThat(
result.overallStatisticsByMutation,
equalTo(
listOf(
OverallStatistics(count = 3, coverage = 11, proportion = 3.0 / 11.0),
OverallStatistics(count = 7, coverage = 2, proportion = 7.0 / 2.0),
),
),
)
assertThat(dataVersion.dataVersion, notNullValue())
}

Expand Down Expand Up @@ -238,6 +249,14 @@ class NucleotideMutationsOverTimeModelTest {
result.totalCountsByDateRange,
equalTo(listOf(0, 0)),
)
assertThat(
result.overallStatisticsByMutation,
equalTo(
listOf(
OverallStatistics(count = 0, coverage = 0, proportion = null),
),
),
)
}

@Test
Expand Down
19 changes: 19 additions & 0 deletions ...src/test/kotlin/org/genspectrum/lapis/model/mutationsOverTime/QueriesOverTimeModelTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ class QueriesOverTimeModelTest {
assertThat(result.data, equalTo(emptyList()))
assertThat(result.dateRanges, equalTo(dateRanges))
assertThat(result.totalCountsByDateRange, equalTo(emptyList()))
assertThat(result.overallStatisticsByQuery, equalTo(emptyList()))
assertThat(dataVersion.dataVersion, notNullValue())
}

Expand All @@ -115,6 +116,7 @@ class QueriesOverTimeModelTest {
assertThat(result.data, equalTo(emptyList()))
assertThat(result.dateRanges, equalTo(emptyList()))
assertThat(result.totalCountsByDateRange, equalTo(emptyList()))
assertThat(result.overallStatisticsByQuery, equalTo(emptyList()))
assertThat(dataVersion.dataVersion, notNullValue())
}

Expand Down Expand Up @@ -198,6 +200,15 @@ class QueriesOverTimeModelTest {
result.totalCountsByDateRange,
equalTo(listOf(10, 23)),
)
assertThat(
result.overallStatisticsByQuery,
equalTo(
listOf(
OverallStatistics(count = 3, coverage = 11, proportion = 3.0 / 11.0),
OverallStatistics(count = 7, coverage = 2, proportion = 7.0 / 2.0),
),
),
)
assertThat(dataVersion.dataVersion, notNullValue())
}

Expand Down Expand Up @@ -254,6 +265,14 @@ class QueriesOverTimeModelTest {
result.totalCountsByDateRange,
equalTo(listOf(0, 0)),
)
assertThat(
result.overallStatisticsByQuery,
equalTo(
listOf(
OverallStatistics(count = 0, coverage = 0, proportion = null),
),
),
)
}

@Test
Expand Down
Loading