Skip to content

Refactor index_data.painless script and params #1207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ellisandrews-toast wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Refactor index_data.painless script and params #1207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 59 additions & 46 deletions config/schema/artifacts/datastore_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1852,7 +1852,7 @@ indices:
index.number_of_shards: 1
index.max_result_window: 10000
scripts:
update_WidgetCurrency_from_Widget_a63538953d45f84cba67edd655d129a8:
update_WidgetCurrency_from_Widget_e72813bd1a8767343222b77bf53be31c:
context: update
script:
lang: painless
Expand Down Expand Up @@ -1996,7 +1996,7 @@ scripts:
return false;
}

Map data = params.data;
Map data = params.topLevelFields;
// A variable to accumulate script errors so that we can surface _all_ issues and not just the first.
List scriptErrors = new ArrayList();
if (ctx._source.details == null) {
Expand Down Expand Up @@ -2156,67 +2156,67 @@ scripts:

// No timestamp values matched the params, so return `false`.
return false;
update_index_data_1fdfaf1c9261c96019decc89b515bd9a:
update_index_data_db769e42fd258f3884d2c5b721bb3c03:
context: update
script:
lang: painless
source: |-
Map source = ctx._source;
String sourceId = params.sourceId;
String relationship = params.relationship;

// Numbers in JSON appear to be parsed as doubles, but we want the version stored as a long, so we need to cast it here.
long eventVersion = (long) params.version;
// --- Helper Functions --- //
void setup(Map source, String relationship, Map counts) {
if (source.__sources == null) {
source.__sources = [];
}

if (source.__sources == null) {
source.__sources = [];
}
if (source.__versions == null) {
source.__versions = [:];
}

if (source.__versions == null) {
source.__versions = [:];
}
if (source.__versions[relationship] == null) {
source.__versions[relationship] = [:];
}

if (source.__versions[relationship] == null) {
source.__versions[relationship] = [:];
if (counts != null && source.__counts == null) {
source.__counts = [:];
}
}

Map relationshipVersionsMap = source.__versions.get(relationship);
List previousSourceIdsForRelationship = relationshipVersionsMap.keySet().stream().filter(id -> id != sourceId).collect(Collectors.toList());
void validateSource(Map source, String id, String relationship, String sourceId, long eventVersion) {
Map relationshipVersionsMap = source.__versions.get(relationship);
List previousSourceIdsForRelationship = relationshipVersionsMap.keySet().stream().filter(key -> key != sourceId).collect(Collectors.toList());

if (previousSourceIdsForRelationship.size() > 0) {
String previousIdDescription = previousSourceIdsForRelationship.size() == 1 ? previousSourceIdsForRelationship.get(0) : previousSourceIdsForRelationship.toString();
throw new IllegalArgumentException(
"Cannot update document " + params.id + " " +
"with data from related " + relationship + " " + sourceId + " " +
"because the related " + relationship + " has apparently changed (was: " + previousSourceIdsForRelationship + "), " +
"but mutations of relationships used with `sourced_from` are not supported because " +
"allowing it could break ElasticGraph's out-of-order processing guarantees."
);
}
if (previousSourceIdsForRelationship.size() > 0) {
throw new IllegalArgumentException(
"Cannot update document " + id + " " +
"with data from related " + relationship + " " + sourceId + " " +
"because the related " + relationship + " has apparently changed (was: " + previousSourceIdsForRelationship + "), " +
"but mutations of relationships used with `sourced_from` are not supported because " +
"allowing it could break ElasticGraph's out-of-order processing guarantees."
);
}

Number maybeDocVersion = source.__versions.get(params.relationship)?.get(params.sourceId);
Number maybeDocVersion = relationshipVersionsMap.get(sourceId);

// Our JSON schema requires event versions to be non-negative, so we can safely use Long.MIN_VALUE as a stand-in when the value is null.
long docVersion = maybeDocVersion == null ? Long.MIN_VALUE : maybeDocVersion.longValue();
// Our JSON schema requires event versions to be non-negative, so we can safely use Long.MIN_VALUE as a stand-in when the value is null.
long docVersion = maybeDocVersion == null ? Long.MIN_VALUE : maybeDocVersion.longValue();

if (docVersion >= eventVersion) {
throw new IllegalArgumentException("ElasticGraph update was a no-op: [" +
params.id + "]: version conflict, current version [" +
docVersion + "] is higher or equal to the one provided [" +
eventVersion + "]");
} else {
source.putAll(params.data);
Map __counts = params.__counts;
if (docVersion >= eventVersion) {
throw new IllegalArgumentException("ElasticGraph update was a no-op: [" +
id + "]: version conflict, current version [" +
docVersion + "] is higher or equal to the one provided [" +
eventVersion + "]");
}
}

if (__counts != null) {
if (source.__counts == null) {
source.__counts = [:];
}
void applyTopLevelFields(Map source, Map topLevelFields, Map counts) {
source.putAll(topLevelFields);

source.__counts.putAll(__counts);
if (counts != null) {
source.__counts.putAll(counts);
}
}

source.id = params.id;
void recordSource(Map source, String id, String relationship, String sourceId, long eventVersion) {
source.id = id;
source.__versions[relationship][sourceId] = eventVersion;

// Record the relationship in `__sources` if it's not already there. We maintain it as an append-only set using a sorted list.
Expand All @@ -2233,3 +2233,16 @@ scripts:
source.__sources.add(-sourceBinarySearchResult - 1, relationship);
}
}

// --- Main script body --- //
Map source = ctx._source;
String id = params.id;
String relationship = params.relationship;
String sourceId = params.sourceId;
long eventVersion = (long) params.version; // Cast to long since JSON parses numbers as doubles
Map counts = params.__counts;

setup(source, relationship, counts);
validateSource(source, id, relationship, sourceId, eventVersion);
applyTopLevelFields(source, params.topLevelFields, counts);
recordSource(source, id, relationship, sourceId, eventVersion);
40 changes: 20 additions & 20 deletions config/schema/artifacts/runtime_metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3096,7 +3096,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Address
AddressAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -3280,7 +3280,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: BrokerWholesaler
ColorListFilterInput:
graphql_fields_by_name:
Expand Down Expand Up @@ -3321,7 +3321,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Company
Component:
graphql_fields_by_name:
Expand Down Expand Up @@ -3429,7 +3429,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Component
ComponentAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -3729,7 +3729,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: DirectWholesaler
DistributionChannel:
graphql_fields_by_name:
Expand Down Expand Up @@ -3906,7 +3906,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: ElectricalPart
ElectricalPartAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -4389,7 +4389,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Manufacturer
ManufacturerAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -4549,7 +4549,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: MechanicalPart
MechanicalPartAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -5510,7 +5510,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: OnlineStore
PageInfo:
graphql_fields_by_name:
Expand Down Expand Up @@ -5707,7 +5707,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Person
PersonAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -5772,7 +5772,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: PhysicalStore
PhysicalStoreAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -6307,7 +6307,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Sponsor
SponsorAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -6689,7 +6689,7 @@ object_types_by_name:
relationship: __self
rollover_timestamp_value_source: formed_on
routing_value_source: league
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Team
TeamAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -7737,7 +7737,7 @@ object_types_by_name:
id_source: cost.currency
rollover_timestamp_value_source: cost_currency_introduced_on
routing_value_source: cost_currency_primary_continent
script_id: update_WidgetCurrency_from_Widget_a63538953d45f84cba67edd655d129a8
script_id: update_WidgetCurrency_from_Widget_e72813bd1a8767343222b77bf53be31c
type: WidgetCurrency
- data_params:
amount_cents:
Expand Down Expand Up @@ -7817,7 +7817,7 @@ object_types_by_name:
relationship: __self
rollover_timestamp_value_source: created_at
routing_value_source: workspace_id2
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Widget
- data_params:
widget_cost:
Expand Down Expand Up @@ -7848,7 +7848,7 @@ object_types_by_name:
version:
cardinality: one
relationship: widget
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Component
WidgetAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -8085,7 +8085,7 @@ object_types_by_name:
relationship: __self
rollover_timestamp_value_source: introduced_on
routing_value_source: primary_continent
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: WidgetCurrency
WidgetCurrencyAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -9112,7 +9112,7 @@ object_types_by_name:
cardinality: one
relationship: __self
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: WidgetWorkspace
- data_params:
workspace_name:
Expand All @@ -9133,7 +9133,7 @@ object_types_by_name:
relationship: workspace
rollover_timestamp_value_source: widget.created_at
routing_value_source: id
script_id: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
script_id: update_index_data_db769e42fd258f3884d2c5b721bb3c03
type: Widget
WidgetWorkspaceAggregatedValues:
graphql_fields_by_name:
Expand Down Expand Up @@ -9364,4 +9364,4 @@ static_script_ids_by_scoped_name:
field/as_day_of_week: field_as_day_of_week_f2b5c7d9e8f75bf2457b52412bfb6537
field/as_time_of_day: field_as_time_of_day_ed82aba44fc66bff5635bec4305c1c66
filter/by_time_of_day: filter_by_time_of_day_ea12d0561b24961789ab68ed38435612
update/index_data: update_index_data_1fdfaf1c9261c96019decc89b515bd9a
update/index_data: update_index_data_db769e42fd258f3884d2c5b721bb3c03
Loading
Loading