From ee7fe584ed3edbdaf400ce9ea716bb26080363e0 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 20 May 2026 14:55:49 -0500 Subject: [PATCH] Use JSON ingestion as schema definition extension --- Rakefile | 2 + config/site/support/doctest_helper.rb | 4 +- .../apollo/apollo_directives_spec.rb | 3 +- .../apollo/schema_definition_spec.rb | 4 +- .../schema_definition/api_extension.rb | 24 +- .../schema_definition/results_extension.rb | 2 +- .../schema_artifact_manager_extension.rb | 14 +- .../schema_definition/state_extension.rb | 4 +- .../sig/elastic_graph/json_ingestion.rbs | 4 - .../schema_definition/api_extension.rbs | 1 + .../indexing/json_schema_with_metadata.rbs | 2 +- .../schema_definition/results_extension.rbs | 4 +- .../schema_artifact_manager_extension.rbs | 4 +- .../schema_definition/state_extension.rbs | 1 + .../elasticgraph-schema_definition.gemspec | 3 + .../elastic_graph/schema_definition/api.rb | 86 ++----- .../schema_definition/indexing/field.rb | 107 --------- .../indexing/field_reference.rb | 5 +- .../indexing/field_type/enum.rb | 29 --- .../indexing/field_type/object.rb | 75 +----- .../indexing/field_type/scalar.rb | 16 -- .../indexing/field_type/union.rb | 30 --- .../schema_definition/indexing/index.rb | 8 +- .../schema_definition/mixins/has_type_info.rb | 110 +++------ .../schema_definition/rake_tasks.rb | 4 +- .../schema_definition/results.rb | 93 -------- .../schema_artifact_manager.rb | 217 +----------------- .../schema_elements/built_in_types.rb | 29 +-- .../schema_elements/enum_type.rb | 4 +- .../schema_elements/field.rb | 24 +- .../schema_elements/scalar_type.rb | 32 +-- .../schema_elements/type_reference.rb | 24 -- .../schema_elements/type_with_subfields.rb | 3 +- .../elastic_graph/schema_definition/state.rb | 24 +- .../schema_definition/test_support.rb | 4 +- .../elastic_graph/schema_definition/api.rbs | 6 +- .../schema_definition/indexing/field.rbs | 20 -- .../indexing/field_reference.rbs | 3 - .../schema_definition/indexing/field_type.rbs | 3 - .../indexing/field_type/object.rbs | 19 +- .../schema_definition/indexing/index.rbs | 2 +- .../mixins/has_type_info.rbs | 2 +- .../schema_definition/results.rbs | 19 +- .../schema_artifact_manager.rbs | 13 -- .../schema_elements/field.rbs | 1 - .../schema_elements/scalar_type.rbs | 1 - .../schema_elements/type_reference.rbs | 5 - .../elastic_graph/schema_definition/state.rbs | 18 +- .../schema_definition/rake_tasks_spec.rb | 26 ++- .../schema_definition/factory_spec.rb | 28 +++ .../schema_definition/rake_tasks_spec.rb | 8 +- .../project_template/Rakefile.tt | 2 + .../spec_support/enable_simplecov.rb | 3 + .../spec_support/schema_definition_helpers.rb | 3 +- 54 files changed, 208 insertions(+), 974 deletions(-) delete mode 100644 elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion.rbs create mode 100644 elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb diff --git a/Rakefile b/Rakefile index 831dc68b3..13a665b9d 100644 --- a/Rakefile +++ b/Rakefile @@ -8,6 +8,7 @@ require "delegate" require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/warehouse/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/schema_definition/rake_tasks" @@ -49,6 +50,7 @@ configure_local_rake_tasks = ->(tasks) do tasks.index_document_sizes = true tasks.env_port_mapping = {test: test_port} tasks.output = schema_def_output + tasks.schema_definition_extension_modules = [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension] tasks.define_fake_data_batch_for(:widgets) do require "rspec/core" # the factories file expects RSpec to be loaded, so load it. diff --git a/config/site/support/doctest_helper.rb b/config/site/support/doctest_helper.rb index f322eb599..a6d947805 100644 --- a/config/site/support/doctest_helper.rb +++ b/config/site/support/doctest_helper.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" require "elastic_graph/schema_definition/api" require "elastic_graph/schema_definition/schema_artifact_manager" @@ -60,7 +61,7 @@ module ElasticGraph @api = SchemaDefinition::API.new( SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :camelCase, overrides: {}), true, - extension_modules: extension_modules + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension] + extension_modules ) # This is required in all schemas, but we don't want to have to put in all our examples, @@ -95,6 +96,7 @@ module ElasticGraph # `schema.json_schema_version` raises an error when the version is set more than once. # By default we set it above. Here we clear it to allow our example to set it. schema.state.json_schema_version = nil + schema.state.json_schema_version_setter_location = nil end end diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb index 915c79c34..a8130e4eb 100644 --- a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/spec_support/schema_definition_helpers" module ElasticGraph @@ -552,7 +553,7 @@ def self.with_both_casing_forms(&block) end def define_schema(&block) - extension_modules = [SchemaDefinition::APIExtension] + extension_modules = [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension, SchemaDefinition::APIExtension] super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block) end end diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb index 740bc003e..34970bb06 100644 --- a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/spec_support/runtime_metadata_support" require "elastic_graph/spec_support/schema_definition_helpers" require "graphql" @@ -1431,7 +1432,8 @@ def expect_identifiable_type_tagging_of_token(&type_def_for) end def define_schema(with_apollo: true, &block) - extension_modules = with_apollo ? [SchemaDefinition::APIExtension] : [] + extension_modules = [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension] + extension_modules += [SchemaDefinition::APIExtension] if with_apollo super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block) end end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb index f37cf6ade..f1d4a95c0 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb @@ -99,11 +99,11 @@ def self.extended(api) # # @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying constantly # have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this - # on every JSON schema change by setting `enforce_json_schema_version` to `false` in your `Rakefile`. + # on every JSON schema change with {#enforce_json_schema_version}. # # @param version [Integer] current version number of the JSON schema artifact # @return [void] - # @see Local::RakeTasks#enforce_json_schema_version + # @see #enforce_json_schema_version def json_schema_version(version) state = json_ingestion_state @@ -120,6 +120,26 @@ def json_schema_version(version) nil end + # Configures whether JSON schema artifact dumping enforces that {#json_schema_version} gets bumped every time the JSON schemas + # artifact changes. This should generally remain enabled for production applications, but disabling it can be useful during early + # prototyping. + # + # @param value [Boolean] whether JSON schema version bumps should be enforced + # @return [void] + # + # @example Disable JSON schema version enforcement while prototyping + # ElasticGraph.define_schema do |schema| + # schema.enforce_json_schema_version false + # end + def enforce_json_schema_version(value) + unless value == true || value == false + raise Errors::SchemaError, "`enforce_json_schema_version` must be a boolean. Specified value: #{value.inspect}" + end + + json_ingestion_state.enforce_json_schema_version = value + nil + end + # Defines strictness of the JSON schema validation. By default, the JSON schema will require all fields to be provided by the # publisher (but they can be nullable) and will ignore extra fields that are not defined in the schema. Use this method to # configure this behavior. diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/results_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/results_extension.rb index 051aeb08a..b776cb097 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/results_extension.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/results_extension.rb @@ -121,7 +121,7 @@ def json_schema_indexing_field_types_by_name end .sort_by(&:name) .to_h do |type| - # @type var indexing_field_type: ElasticGraph::SchemaDefinition::Indexing::_FieldType + # @type var indexing_field_type: Indexing::_JSONFieldType indexing_field_type = _ = type.to_indexing_field_type [type.name, indexing_field_type] end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rb index 90027c0a1..f8b4f8f4c 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rb @@ -18,17 +18,12 @@ module SchemaDefinition # # @private module SchemaArtifactManagerExtension - # Configuration key consumed from {ElasticGraph::SchemaDefinition::SchemaArtifactManager}'s - # `extension_artifact_options:` hash. When set to `false`, JSON schema artifact dumps don't - # require an accompanying `json_schema_version` bump. - ENFORCE_JSON_SCHEMA_VERSION_KEY = :enforce_json_schema_version - # Overrides `dump_artifacts` to add JSON schema version bump checking before dumping. def dump_artifacts schema_results = json_ingestion_schema_definition_results check_if_needs_json_schema_version_bump do |recommended_json_schema_version| - if @extension_artifact_options.fetch(ENFORCE_JSON_SCHEMA_VERSION_KEY, true) + if schema_results.state.enforce_json_schema_version # @type var setter_location: ::Thread::Backtrace::Location # We use `_ =` because while `json_schema_version_setter_location` can be nil, # it'll never be nil if we get here and we want the type to be non-nilable. @@ -39,13 +34,12 @@ def dump_artifacts "increase the schema's version, and then run the `bundle exec rake schema_artifacts:dump` command again.\n\n" \ "To update the schema version to the expected version, change line #{setter_location.lineno} at `#{setter_location_path}` to:\n" \ " `schema.json_schema_version #{recommended_json_schema_version}`\n\n" \ - "Alternately, pass `extension_artifact_options: {enforce_json_schema_version: false}` to `ElasticGraph::SchemaDefinition::RakeTasks.new` " \ - "(or set `tasks.enforce_json_schema_version = false` on `ElasticGraph::Local::RakeTasks`) to allow the JSON schemas file to " \ - "change without requiring a version bump, but that is only recommended for non-production applications during initial schema prototyping." + "Alternately, call `schema.enforce_json_schema_version false` in your schema definition to allow the JSON schemas file " \ + "to change without requiring a version bump, but that is only recommended for non-production applications during initial schema prototyping." else @output.puts <<~EOS WARNING: the `json_schemas.yaml` artifact is being updated without the `json_schema_version` being correspondingly incremented. - This is not recommended for production applications, but is currently allowed because you have set `enforce_json_schema_version: false`. + This is not recommended for production applications, but is currently allowed because you have called `schema.enforce_json_schema_version false`. EOS end end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/state_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/state_extension.rb index 52427e0c8..d072bb02d 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/state_extension.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/state_extension.rb @@ -15,13 +15,15 @@ module SchemaDefinition module StateExtension # @dynamic json_schema_version, json_schema_version= # @dynamic json_schema_version_setter_location, json_schema_version_setter_location= + # @dynamic enforce_json_schema_version, enforce_json_schema_version= # @dynamic allow_omitted_json_schema_fields, allow_omitted_json_schema_fields= # @dynamic allow_extra_json_schema_fields, allow_extra_json_schema_fields= - attr_accessor :json_schema_version, :json_schema_version_setter_location, :allow_omitted_json_schema_fields, :allow_extra_json_schema_fields + attr_accessor :json_schema_version, :json_schema_version_setter_location, :enforce_json_schema_version, :allow_omitted_json_schema_fields, :allow_extra_json_schema_fields def self.extended(state) state.json_schema_version = nil state.json_schema_version_setter_location = nil + state.enforce_json_schema_version = true state.allow_omitted_json_schema_fields = false state.allow_extra_json_schema_fields = true end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion.rbs deleted file mode 100644 index b7ecda553..000000000 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion.rbs +++ /dev/null @@ -1,4 +0,0 @@ -module ElasticGraph - module JSONIngestion - end -end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/api_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/api_extension.rbs index 53d8c9070..12f309219 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/api_extension.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/api_extension.rbs @@ -5,6 +5,7 @@ module ElasticGraph BUILT_IN_SCALAR_JSON_SCHEMA_OPTIONS_BY_NAME: ::Hash[::String, ::Hash[::Symbol, untyped]] def json_schema_version: (::Integer) -> void + def enforce_json_schema_version: (bool) -> void def json_schema_strictness: (?allow_omitted_fields: bool, ?allow_extra_fields: bool) -> void def self.extended: (::ElasticGraph::SchemaDefinition::API & APIExtension) -> void diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs index a553ed9a8..81abcb5a7 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs @@ -40,7 +40,7 @@ module ElasticGraph attr_reader unused_deprecated_elements: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] - def initialize: (ElasticGraph::SchemaDefinition::Results) -> void + def initialize: ((ElasticGraph::SchemaDefinition::Results & ResultsExtension)) -> void def merge_metadata_into: (::Hash[::String, untyped]) -> JSONSchemaWithMetadata private diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/results_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/results_extension.rbs index e7eac97be..bd617f7d5 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/results_extension.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/results_extension.rbs @@ -18,12 +18,12 @@ module ElasticGraph @json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, Indexing::JSONSchemaFieldMetadata]]? @current_public_json_schema: ::Hash[::String, untyped]? @json_schema_with_metadata_merger: Indexing::JSONSchemaWithMetadata::Merger? - @json_schema_indexing_field_types_by_name: ::Hash[::String, ::ElasticGraph::SchemaDefinition::Indexing::_FieldType]? + @json_schema_indexing_field_types_by_name: ::Hash[::String, Indexing::_JSONFieldType]? def json_ingestion_state: () -> (::ElasticGraph::SchemaDefinition::State & StateExtension) def json_schema_with_metadata_merger: () -> Indexing::JSONSchemaWithMetadata::Merger def build_public_json_schema: () -> ::Hash[::String, untyped] - def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, ::ElasticGraph::SchemaDefinition::Indexing::_FieldType] + def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, Indexing::_JSONFieldType] end end end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rbs index 1ce820bce..b03bd3001 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rbs @@ -2,11 +2,9 @@ module ElasticGraph module JSONIngestion module SchemaDefinition module SchemaArtifactManagerExtension : ::ElasticGraph::SchemaDefinition::SchemaArtifactManager - ENFORCE_JSON_SCHEMA_VERSION_KEY: ::Symbol - private - @json_schemas_artifact: ::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped] + @json_schemas_artifact: ::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]? def json_ingestion_schema_definition_results: () -> (::ElasticGraph::SchemaDefinition::Results & ResultsExtension) def artifacts_from_schema_def: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]] diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/state_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/state_extension.rbs index a11f759cc..06f930cbb 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/state_extension.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/state_extension.rbs @@ -4,6 +4,7 @@ module ElasticGraph module StateExtension: ::ElasticGraph::SchemaDefinition::State attr_accessor json_schema_version: ::Integer? attr_accessor json_schema_version_setter_location: ::Thread::Backtrace::Location? + attr_accessor enforce_json_schema_version: bool attr_accessor allow_omitted_json_schema_fields: bool attr_accessor allow_extra_json_schema_fields: bool diff --git a/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec b/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec index da50e4335..8ce33b347 100644 --- a/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec +++ b/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec @@ -43,6 +43,9 @@ Gem::Specification.new do |spec| spec.add_dependency "elasticgraph-graphql", ElasticGraph::VERSION # needed since we validate that scalar `coerce_with` options are valid (which loads scalar coercion adapters) spec.add_dependency "elasticgraph-indexer", ElasticGraph::VERSION # needed since we validate that scalar `prepare_for_indexing_with` options are valid (which loads indexing preparer adapters) + # `elasticgraph-json_ingestion` is the default ingestion serializer extension. It's a soft dep here + # so `default_extension_modules` can fall back to `[]` when the gem isn't installed (e.g. for apps + # that supply a different serializer via `Gemfile-custom`), but most apps want it. spec.add_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_dependency "elasticgraph-schema_artifacts", ElasticGraph::VERSION spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb index 814a13cb0..0af68e0c4 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb @@ -9,14 +9,15 @@ require "elastic_graph/errors" require "elastic_graph/schema_artifacts/runtime_metadata/extension" require "elastic_graph/schema_artifacts/runtime_metadata/graphql_resolver" -require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" -require "elastic_graph/schema_definition/results" -require "elastic_graph/schema_definition/state" # :nocov: -- only loaded on JRuby require "elastic_graph/schema_definition/jruby_patches" if RUBY_ENGINE == "jruby" # :nocov: +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/results" +require "elastic_graph/schema_definition/state" + module ElasticGraph # The main entry point for schema definition from ElasticGraph applications. # @@ -294,7 +295,7 @@ def union_type(name, &block) # ElasticGraph.define_schema do |schema| # schema.scalar_type "URL" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", format: "uri" + # t.json_schema type: "string" # end # end def scalar_type(name, &block) @@ -453,86 +454,31 @@ def results @results ||= @factory.new_results end - # Defines the version number of the current JSON schema. Importantly, every time a change is made that impacts the JSON schema - # artifact, the version number must be incremented to ensure that each different version of the JSON schema is identified by a unique - # version number. The publisher will then include this version number in published events to identify the version of the schema it - # was using. This avoids the need to deploy the publisher and ElasticGraph indexer at the same time to keep them in sync. + # Records the JSON schema version of this schema definition. # - # @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying constantly - # have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this - # on every JSON schema change with {#enforce_json_schema_version}. + # The default implementation is a no-op so that callers (and the test helpers) can invoke this + # uniformly regardless of whether an ingestion-serializer extension is loaded. When + # `elasticgraph-json_ingestion` is loaded, its `APIExtension` overrides this with real behavior. # - # @param version [Integer] current version number of the JSON schema artifact + # @param version [Integer] the JSON schema version # @return [void] - # @see #enforce_json_schema_version - # - # @example Set the JSON schema version to 1 - # ElasticGraph.define_schema do |schema| - # schema.json_schema_version 1 - # end def json_schema_version(version) - if !version.is_a?(Integer) || version < 1 - raise Errors::SchemaError, "`json_schema_version` must be a positive integer. Specified version: #{version}" - end - - if @state.json_schema_version - raise Errors::SchemaError, "`json_schema_version` can only be set once on a schema. Previously-set version: #{@state.json_schema_version}" - end - - @state.json_schema_version = version - @state.json_schema_version_setter_location = caller_locations(1, 1).to_a.first - nil end - # Configures whether {SchemaArtifactManager} enforces that {#json_schema_version} gets bumped every time the JSON schemas artifact - # changes. This should generally remain enabled for production applications, but disabling it can be useful during early prototyping. + # Records JSON schema version enforcement configuration. # - # @param value [Boolean] whether JSON schema version bumps should be enforced - # @return [void] + # The default implementation is a no-op; overridden by the JSON ingestion extension. # - # @example Disable JSON schema version enforcement while prototyping - # ElasticGraph.define_schema do |schema| - # schema.enforce_json_schema_version false - # end + # @return [void] def enforce_json_schema_version(value) - unless value == true || value == false - raise Errors::SchemaError, "`enforce_json_schema_version` must be a boolean. Specified value: #{value.inspect}" - end - - @state.enforce_json_schema_version = value - nil end - # Defines strictness of the JSON schema validation. By default, the JSON schema will require all fields to be provided by the - # publisher (but they can be nullable) and will ignore extra fields that are not defined in the schema. Use this method to - # configure this behavior. - # - # @param allow_omitted_fields [bool] Whether nullable fields can be omitted from indexing events. - # @param allow_extra_fields [bool] Whether extra fields (e.g. beyond fields defined in the schema) can be included in indexing events. - # @return [void] + # Records JSON schema strictness configuration. # - # @note If you allow both omitted fields and extra fields, ElasticGraph's JSON schema validation will allow (and ignore) misspelled - # field names in indexing events. For example, if the ElasticGraph schema has a nullable field named `parentId` but the publisher - # accidentally provides it as `parent_id`, ElasticGraph would happily ignore the `parent_id` field entirely, because `parentId` - # is allowed to be omitted and `parent_id` would be treated as an extra field. Therefore, we recommend that you only set one of - # these to `true` (or none). + # The default implementation is a no-op; overridden by the JSON ingestion extension. # - # @example Allow omitted fields and disallow extra fields - # ElasticGraph.define_schema do |schema| - # schema.json_schema_strictness allow_omitted_fields: true, allow_extra_fields: false - # end + # @return [void] def json_schema_strictness(allow_omitted_fields: false, allow_extra_fields: true) - unless [true, false].include?(allow_omitted_fields) - raise Errors::SchemaError, "`allow_omitted_fields` must be true or false" - end - - unless [true, false].include?(allow_extra_fields) - raise Errors::SchemaError, "`allow_extra_fields` must be true or false" - end - - @state.allow_omitted_json_schema_fields = allow_omitted_fields - @state.allow_extra_json_schema_fields = allow_extra_fields - nil end # Registers a customization callback that will be applied to every built-in type automatically provided by ElasticGraph. Provides diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb index cd38c8eaa..51dfd4a1f 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb @@ -7,7 +7,6 @@ # frozen_string_literal: true require "elastic_graph/constants" -require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata" require "elastic_graph/schema_definition/indexing/list_counts_mapping" require "elastic_graph/support/hash_util" require "elastic_graph/support/memoizable_data" @@ -22,28 +21,13 @@ class Field < Support::MemoizableData.define( :name, :name_in_index, :type, - :json_schema_layers, :indexing_field_type, :accuracy_confidence, - :json_schema_customizations, :mapping_customizations, :source, :runtime_field_script, :doc_comment ) - # JSON schema overrides that automatically apply to specific mapping types so that the JSON schema - # validation will reject values which cannot be indexed into fields of a specific mapping type. - # - # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/number.html Elasticsearch numeric field type documentation - # @note We don't handle `integer` here because it's the default numeric type (handled by our definition of the `Int` scalar type). - # @note Likewise, we don't handle `long` here because a custom scalar type must be used for that since GraphQL's `Int` type can't handle long values. - JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE = { - "byte" => {"minimum" => -(2**7), "maximum" => (2**7) - 1}, - "short" => {"minimum" => -(2**15), "maximum" => (2**15) - 1}, - "keyword" => {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH}, - "text" => {"maxLength" => DEFAULT_MAX_TEXT_LENGTH} - } - # @return [Hash] the mapping for this field. The returned hash should be composed entirely # of Ruby primitives that, when converted to a JSON string, match the structure required by # [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html). @@ -63,23 +47,6 @@ def mapping end end - # @return [Hash] the JSON schema definition for this field. The returned object should - # be composed entirely of Ruby primitives that, when converted to a JSON string, match the - # requirements of [the JSON schema spec](https://json-schema.org/). - def json_schema - json_schema_layers - .reverse # resolve layers from innermost to outermost wrappings - .reduce(inner_json_schema) { |acc, layer| process_layer(layer, acc) } - .merge(outer_json_schema_customizations) - .merge({"description" => doc_comment}.compact) - .then { |h| Support::HashUtil.stringify_keys(h) } - end - - # @return [JSONSchemaFieldMetadata] additional ElasticGraph metadata to be stored in the JSON schema for this field. - def json_schema_metadata - ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index) - end - # Builds a hash containing the mapping for the provided fields, normalizing it in the same way that the # datastore does so that consistency checks between our index configuration and what's in the datastore # work properly. @@ -107,80 +74,6 @@ def self.normalized_mapping_hash_for(fields) mapping_hash end - - def nullable? - json_schema_layers.include?(:nullable) - end - - private - - def inner_json_schema - user_specified_customizations = - if user_specified_json_schema_customizations_go_on_outside? - {} # : ::Hash[::String, untyped] - else - Support::HashUtil.stringify_keys(json_schema_customizations) - end - - customizations_from_mapping = JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE[mapping["type"]] || {} - customizations = customizations_from_mapping.merge(user_specified_customizations) - customizations = indexing_field_type.format_field_json_schema_customizations(customizations) - - ref = {"$ref" => "#/$defs/#{type.unwrapped_name}"} - return ref if customizations.empty? - - # Combine any customizations with type ref under an "allOf" subschema: - # All of these properties must hold true for the type to be valid. - # - # Note that if we simply combine the customizations with the `$ref` - # at the same level, it will not work, because other subschema - # properties are ignored when they are in the same object as a `$ref`: - # https://github.com/json-schema-org/JSON-Schema-Test-Suite/blob/2.0.0/tests/draft7/ref.json#L165-L168 - {"allOf" => [ref, customizations]} - end - - def outer_json_schema_customizations - return {} unless user_specified_json_schema_customizations_go_on_outside? - Support::HashUtil.stringify_keys(json_schema_customizations) - end - - # Indicates if the user-specified JSON schema customizations should go on the inside - # (where they normally go) or on the outside. They only go on the outside when it's - # an array field, because then they apply to the array itself instead of the items in the - # array. - def user_specified_json_schema_customizations_go_on_outside? - json_schema_layers.include?(:array) - end - - def process_layer(layer, schema) - case layer - when :nullable - make_nullable(schema) - when :array - make_array(schema) - else - # :nocov: - layer is only ever `:nullable` or `:array` so we never get here - schema - # :nocov: - end - end - - def make_nullable(schema) - # Here we use "anyOf" to ensure that JSON can either match the schema OR null. - # - # (Using "oneOf" would mean that if we had a schema that also allowed null, - # null would never be allowed, since "oneOf" must match exactly one subschema). - { - "anyOf" => [ - schema, - {"type" => "null"} - ] - } - end - - def make_array(schema) - {"type" => "array", "items" => schema} - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb index 070f70db3..f0c3454eb 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb @@ -15,7 +15,6 @@ module Indexing :name_in_index, :type, :mapping_options, - :json_schema_options, :accuracy_confidence, :source, :runtime_field_script, @@ -35,10 +34,8 @@ def resolve name: name, name_in_index: name_in_index, type: type, - json_schema_layers: type.json_schema_layers, indexing_field_type: resolved_type.to_indexing_field_type, accuracy_confidence: accuracy_confidence, - json_schema_customizations: json_schema_options, mapping_customizations: mapping_options, source: source, runtime_field_script: runtime_field_script, @@ -46,7 +43,7 @@ def resolve ) end - # @dynamic initialize, with, name, name_in_index, type, mapping_options, json_schema_options, accuracy_confidence, source, runtime_field_script, doc_comment + # @dynamic initialize, with, name, name_in_index, type, mapping_options, accuracy_confidence, source, runtime_field_script, doc_comment end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb index d2f72fb6f..e1aad284c 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb @@ -23,40 +23,11 @@ module FieldType # # @api private class Enum < ::Data - # @return [Hash] the JSON schema for this enum type. - def to_json_schema - {"type" => "string", "enum" => enum_value_names} - end - # @return [Hash] the datastore mapping for this enum type. def to_mapping {"type" => "keyword"} end - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this enum type. - def json_schema_field_metadata_by_field_name - {} - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - # Since an enum type already restricts the values to a small set of allowed values, we do not need to keep - # other customizations (such as the `maxLength` field customization EG automatically applies to fields - # indexed as a `keyword`--we don't allow enum values to exceed that length, anyway). - # - # It's desirable to restrict what customizations are applied because when a publisher uses the JSON schema - # to generate code using a library such as https://github.com/pwall567/json-kotlin-schema-codegen, we found - # that the presence of extra field customizations inhibits the library's ability to generate code in the way - # we want (it causes the type of the enum to change since the JSON schema changes from a direct `$ref` to - # being wrapped in an `allOf`). - # - # However, we still want to apply `enum` customizations--this allows a user to "narrow" the set of allowed - # values for a field. For example, a `Currency` enum could contain every currency, and a user may want to - # restrict a specific `currency` field to a subset of currencies (e.g. to just USD, CAD, and EUR). - customizations.slice("enum") - end - # @dynamic initialize, enum_value_names end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb index c01fcbd4e..c02e463b6 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb @@ -14,7 +14,7 @@ module ElasticGraph module SchemaDefinition module Indexing module FieldType - # Responsible for the JSON schema and mapping of a {SchemaElements::ObjectType}. + # Responsible for the mapping of a {SchemaElements::ObjectType}. # # @!attribute [r] type_name # @return [String] name of the object type @@ -22,13 +22,11 @@ module FieldType # @return [Array] the subfields of this object type # @!attribute [r] mapping_options # @return [Hash] options to be included in the mapping - # @!attribute [r] json_schema_options - # @return [Hash] options to be included in the JSON schema # @!attribute [r] doc_comment # @return [String, nil] documentation for the type # # @api private - class Object < Support::MemoizableData.define(:schema_def_state, :type_name, :subfields, :mapping_options, :json_schema_options, :doc_comment) + class Object < Support::MemoizableData.define(:schema_def_state, :type_name, :subfields, :mapping_options, :doc_comment) # @return [Hash] the datastore mapping for this object type. def to_mapping @to_mapping ||= begin @@ -41,77 +39,10 @@ def to_mapping end end - # @return [Hash] the JSON schema for this object type. - def to_json_schema - @to_json_schema ||= - if json_schema_options.empty? - # Fields that are `sourced_from` an alternate type must not be included in this types JSON schema, - # since events of this type won't include them. - other_source_subfields, json_schema_candidate_subfields = subfields.partition(&:source) - validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) - json_schema_subfields = json_schema_candidate_subfields.reject(&:runtime_field_script) - required_fields = json_schema_subfields - required_fields = required_fields.reject(&:nullable?) if schema_def_state.allow_omitted_json_schema_fields - - { - "type" => "object", - "properties" => json_schema_subfields.to_h { |f| [f.name, f.json_schema] }.merge(json_schema_typename_field), - # Note: `__typename` is intentionally not included in the `required` list. If `__typename` is present - # we want it validated (as we do by merging in `json_schema_typename_field`) but we only want - # to require it in the context of a union type. The union's json schema requires the field. - "required" => required_fields.map(&:name).freeze, - "additionalProperties" => (false unless schema_def_state.allow_extra_json_schema_fields), - "description" => doc_comment - }.compact.freeze - else - Support::HashUtil.stringify_keys(json_schema_options) - end - end - - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this object type. - def json_schema_field_metadata_by_field_name - subfields.to_h { |f| [f.name, f.json_schema_metadata] } - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - customizations - end - - private - + # @private def after_initialize subfields.freeze end - - # Returns a __typename property which we use for union types. - # - # This must always be set to the name of the type (thus the const value). - # - # We also add a "default" value. This does not impact validation, but rather - # aids tools like our kotlin codegen to save publishers from having to set the - # property explicitly when creating events. - def json_schema_typename_field - { - "__typename" => { - "type" => "string", - "const" => type_name, - "default" => type_name - } - } - end - - def validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) - problem_fields = other_source_subfields.reject { |f| f.json_schema_customizations.empty? } - return if problem_fields.empty? - - field_descriptions = problem_fields.map(&:name).sort.map { |f| "`#{f}`" }.join(", ") - raise Errors::SchemaError, - "`#{type_name}` has #{problem_fields.size} field(s) (#{field_descriptions}) that are `sourced_from` " \ - "another type and also have JSON schema customizations. Instead, put the JSON schema " \ - "customizations on the source type's field definitions." - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb index cb9c3132e..600a05c90 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb @@ -27,22 +27,6 @@ def to_mapping Support::HashUtil.stringify_keys(scalar_type.mapping_options) end - # @return [Hash] the JSON schema for this scalar type. - def to_json_schema - Support::HashUtil.stringify_keys(scalar_type.json_schema_options) - end - - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this scalar type. - def json_schema_field_metadata_by_field_name - {} - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - customizations - end - # @dynamic initialize, scalar_type end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb index e4d6e634f..044ac7795 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb @@ -24,25 +24,6 @@ module FieldType # # @api private class Union < ::Data.define(:subtypes_by_name) - # @return [Hash] the JSON schema for this union type. - def to_json_schema - subtype_json_schemas = subtypes_by_name.keys.map { |name| {"$ref" => "#/$defs/#{name}"} } - - # A union type can represent multiple subtypes, referenced by the "anyOf" clause below. - # We also add a requirement for the presence of __typename to indicate which type - # is being referenced (this property is pre-defined on the type itself as a constant). - # - # Note: Although both "oneOf" and "anyOf" keywords are valid for combining schemas - # to form a union, and validate equivalently when no object can satisfy multiple of the - # subschemas (which is the case here given the __typename requirements are mutually - # exclusive), we chose to use "oneOf" here because it works better with this library: - # https://github.com/pwall567/json-kotlin-schema-codegen - { - "required" => %w[__typename], - "oneOf" => subtype_json_schemas - } - end - # @return [Hash] the datastore mapping for this union type. def to_mapping mapping_subfields = subtypes_by_name.values.map(&:subfields).reduce([], :union) @@ -52,17 +33,6 @@ def to_mapping {"properties" => {"__typename" => {"type" => "keyword"}}} ) end - - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this union type. - def json_schema_field_metadata_by_field_name - {} - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - customizations - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb index a6400db64..433f2e712 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb @@ -65,10 +65,12 @@ def initialize(name, settings, schema_def_state, indexed_type) # us a nice efficiency boost. id_field_path = public_field_path("id", explanation: "indexed types must have an `id` field") self.routing_field_path = id_field_path - id_field_path.last_part.json_schema nullable: false end + # :nocov: -- `Factory#new_index` (and its JSON-ingestion wrapper) always pass a block; the + # else branch is here for hand-instantiation paths that the test suite doesn't exercise. yield self if block_given? + # :nocov: end # Specifies how documents in this index should sort by default, when no `orderBy` argument is provided to the GraphQL query. @@ -138,8 +140,6 @@ def rollover(frequency, timestamp_field_path_name) raise Errors::SchemaError, "rollover field `#{timestamp_field_path.full_description}` cannot be used for rollover since it is a list field." end - timestamp_field_path.path_parts.each { |f| f.json_schema nullable: false } - self.rollover_config = RolloverConfig.new( frequency: frequency, timestamp_field_path: timestamp_field_path @@ -186,8 +186,6 @@ def route_with(routing_field_path_name) self.routing_field_path = routing_field_path - routing_field_path.path_parts[0..-2].each { |f| f.json_schema nullable: false } - routing_field_path.last_part.json_schema nullable: false, pattern: HAS_NON_WHITE_SPACE_REGEX indexed_type.append_to_documentation "For more performant queries on this type, please filter on `#{routing_field_path_name}` if possible." end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb index 454093ff6..d805b964b 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb @@ -6,27 +6,47 @@ # # frozen_string_literal: true -require "elastic_graph/support/json_schema/meta_schema_validator" - module ElasticGraph module SchemaDefinition module Mixins - # Mixin used to specify non-GraphQL type info (datastore index and JSON schema type info). + # Mixin used to specify non-GraphQL type info on schema elements. # Exists as a mixin so we can apply the same consistent API to every place we need to use this. # Currently it's used in 3 places: # - # - {SchemaElements::ScalarType}: allows specification of how scalars are represented in JSON schema and the index. - # - {SchemaElements::TypeWithSubfields}: allows customization of how an object type is represented in JSON schema and the index. - # - {SchemaElements::Field}: allows customization of a specific field over the field type's standard JSON schema and the index mapping. + # - {SchemaElements::ScalarType}: allows specification of how scalars are represented in the datastore index. + # - {SchemaElements::TypeWithSubfields}: allows customization of how an object type is represented in the datastore index. + # - {SchemaElements::Field}: allows customization of a specific field over the field type's standard index mapping. module HasTypeInfo # @return [Hash] datastore mapping options def mapping_options @mapping_options ||= {} end + # Returns the JSON schema options recorded for this element. + # + # The default implementation returns an empty hash; ingestion-serializer extensions (such as + # `elasticgraph-json_ingestion`) override this on the elements they extend to return real options. + # Defining it here lets call sites (and Apollo/test helpers) reference `json_schema_options` and + # call `json_schema(...)` without having to detect whether an ingestion serializer is loaded. + # # @return [Hash] JSON schema options def json_schema_options - @json_schema_options ||= {} + # :nocov: -- only reached when no ingestion-serializer extension is loaded; the JSON Ingestion + # extension shadows this method on every element it extends. + {} + # :nocov: + end + + # Records JSON schema options for this element. + # + # The default implementation is a no-op; an ingestion-serializer extension overrides this on the + # elements it extends to apply real validation and storage. + # + # @param options [Hash] JSON schema options + # @return [void] + def json_schema(**options) + # :nocov: -- only reached when no ingestion-serializer extension is loaded. + # :nocov: end # Set of mapping parameters that it makes sense to allow customization of, based on @@ -70,7 +90,7 @@ def json_schema_options # ElasticGraph.define_schema do |schema| # schema.scalar_type "URL" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", format: "uri" + # t.json_schema type: "string" # end # end # @@ -87,13 +107,6 @@ def json_schema_options # t.field "expYear", "Int" do |f| # # Use a smaller numeric type to save space in the datastore # f.mapping type: "short" - # f.json_schema minimum: 2000, maximum: 2099 - # end - # - # t.field "expMonth", "Int" do |f| - # # Use a smaller numeric type to save space in the datastore - # f.mapping type: "byte" - # f.json_schema minimum: 1, maximum: 12 # end # # t.index "cards" @@ -108,73 +121,6 @@ def mapping(**options) mapping_options.update(options) end - - # Defines the [JSON schema](https://json-schema.org/understanding-json-schema/) validations for this field or type. Validations - # defined here will be included in the generated `json_schemas.yaml` artifact, which is used by the ElasticGraph indexer to - # validate events before indexing their data in the datastore. In addition, the publisher may use `json_schemas.yaml` for code - # generation and to apply validation before publishing an event to ElasticGraph. - # - # Can be called multiple times; each time, the options will be merged into the existing options. - # - # This is _required_ on a {SchemaElements::ScalarType} (since we don’t know how a custom scalar type should be represented in - # JSON!). On a {SchemaElements::Field}, this is optional, but can be used to make the JSON schema validation stricter then it - # would otherwise be. For example, you could use `json_schema maxLength: 30` on a `String` field to limit the length. - # - # You can use any of the JSON schema validation keywords here. In addition, `nullable: false` is supported to configure the - # generated JSON schema to disallow `null` values for the field. Note that if you define a field with a non-nullable GraphQL type - # (e.g. `Int!`), the JSON schema will automatically disallow nulls. However, as explained in the - # {SchemaElements::TypeWithSubfields#field} documentation, we generally recommend against defining non-nullable GraphQL fields. - # `json_schema nullable: false` will disallow `null` values from being indexed, while still keeping the field nullable in the - # GraphQL schema. If you think you might want to make a field non-nullable in the GraphQL schema some day, it’s a good idea to use - # `json_schema nullable: false` now to ensure every indexed record has a non-null value for the field. - # - # @note We recommend using JSON schema validations in a limited fashion. Validations that are appropriate to apply when data is - # entering the system-of-record are often not appropriate on a secondary index like ElasticGraph. Events that violate a JSON - # schema validation will fail to index (typically they will be sent to the dead letter queue and page an oncall engineer). If an - # ElasticGraph instance is meant to contain all the data of some source system, you probably don’t want it applying stricter - # validations than the source system itself has. We recommend limiting your JSON schema validations to situations where - # violations would prevent ElasticGraph from operating correctly. - # - # @param options [Hash] JSON schema options - # @return [void] - # - # @example Define the JSON schema validations of a custom scalar type - # ElasticGraph.define_schema do |schema| - # schema.scalar_type "URL" do |t| - # t.mapping type: "keyword" - # - # # JSON schema has a built-in URI format validator: - # # https://json-schema.org/understanding-json-schema/reference/string.html#resource-identifiers - # t.json_schema type: "string", format: "uri" - # end - # end - # - # @example Define additional validations on a field - # ElasticGraph.define_schema do |schema| - # schema.object_type "Card" do |t| - # t.field "id", "ID!" - # - # t.field "expYear", "Int" do |f| - # # Use JSON schema to ensure the publisher is sending us 4 digit years, not 2 digit years. - # f.json_schema minimum: 2000, maximum: 2099 - # end - # - # t.field "expMonth", "Int" do |f| - # f.json_schema minimum: 1, maximum: 12 - # end - # - # t.index "cards" - # end - # end - def json_schema(**options) - validatable_json_schema = Support::HashUtil.stringify_keys(options) - - if (error_msg = Support::JSONSchema.strict_meta_schema_validator.validate_with_error_message(validatable_json_schema)) - raise Errors::SchemaError, "Invalid JSON schema options set on #{self}:\n\n#{error_msg}" - end - - json_schema_options.update(options) - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb index 8ab08390e..4ae693147 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb @@ -6,8 +6,8 @@ # # frozen_string_literal: true -require "rake/tasklib" require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "rake/tasklib" module ElasticGraph module SchemaDefinition @@ -40,7 +40,7 @@ class RakeTasks < ::Rake::TaskLib # @param enum_value_overrides_by_type [Hash>] overrides for the names of specific enum values for # specific enum types. For example, to rename the `DayOfWeek.MONDAY` enum to `DayOfWeek.MON`, pass `{DayOfWeek: {MONDAY: "MON"}}`. # @param extension_modules [Array] List of Ruby modules to extend onto the `SchemaDefinition::API` instance. Designed to - # support ElasticGraph extension gems (such as `elasticgraph-apollo`). + # support ElasticGraph extension gems (such as `elasticgraph-apollo` and `elasticgraph-json_ingestion`). # @param output [IO] used for printing task output # # @example Minimal setup with defaults diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb index 0914fcb1e..e228e8b5b 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb @@ -8,8 +8,6 @@ require "elastic_graph/constants" require "elastic_graph/errors" -require "elastic_graph/json_ingestion/schema_definition/indexing/event_envelope" -require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata" require "elastic_graph/schema_artifacts/artifacts_helper_methods" require "elastic_graph/schema_artifacts/runtime_metadata/schema" require "elastic_graph/schema_definition/indexing/relationship_resolver" @@ -46,53 +44,6 @@ def runtime_metadata @runtime_metadata ||= build_runtime_metadata end - # @param version [Integer] desired JSON schema version - # @return [Hash] the JSON schema for the requested version, if available - # @raise [Errors::NotFoundError] if the requested JSON schema version is not available - def json_schemas_for(version) - unless available_json_schema_versions.include?(version) - raise Errors::NotFoundError, "The requested json schema version (#{version}) is not available. Available versions: #{available_json_schema_versions.to_a.join(", ")}." - end - - @latest_versioned_json_schema ||= merge_field_metadata_into_json_schema(current_public_json_schema).json_schema - end - - # @return [Set] set of available JSON schema versions - def available_json_schema_versions - @available_json_schema_versions ||= Set[latest_json_schema_version] - end - - # @return [Hash] the newly generated JSON schema - def latest_json_schema_version - current_public_json_schema[JSON_SCHEMA_VERSION_KEY] - end - - # @private - def json_schema_version_setter_location - state.json_schema_version_setter_location - end - - # @private - def json_schema_field_metadata_by_type_and_field_name - @json_schema_field_metadata_by_type_and_field_name ||= json_schema_indexing_field_types_by_name - .transform_values(&:json_schema_field_metadata_by_field_name) - end - - # @private - def current_public_json_schema - @current_public_json_schema ||= build_public_json_schema - end - - # @private - def merge_field_metadata_into_json_schema(json_schema) - json_schema_with_metadata_merger.merge_metadata_into(json_schema) - end - - # @private - def unused_deprecated_elements - json_schema_with_metadata_merger.unused_deprecated_elements - end - # @private STATIC_SCRIPT_REPO = Scripting::FileSystemRepository.new(::File.join(__dir__.to_s, "scripting", "scripts")) @@ -113,10 +64,6 @@ def after_initialize state.user_definition_complete_callbacks.each(&:call) end - def json_schema_with_metadata_merger - @json_schema_with_metadata_merger ||= ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger.new(self) - end - def generate_datastore_config # We need to check this before generating our datastore configuration. # We can't generate a mapping from a recursively defined schema type. @@ -268,46 +215,6 @@ def generate_sdl [type_defs + state.sdl_parts].join("\n\n") end - def build_public_json_schema - json_schema_version = state.json_schema_version - if json_schema_version.nil? - raise Errors::SchemaError, "`json_schema_version` must be specified in the schema. To resolve, add `schema.json_schema_version 1` in a schema definition block." - end - - root_document_type_names = state.object_types_by_name.values - .select { |type| type.root_document_type? && !type.abstract? } - .reject { |type| derived_indexing_type_names.include?(type.name) } - .map(&:name) - - definitions_by_name = json_schema_indexing_field_types_by_name - .transform_values(&:to_json_schema) - .compact - - event_envelope = ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::EventEnvelope - - { - "$schema" => JSON_META_SCHEMA, - JSON_SCHEMA_VERSION_KEY => json_schema_version, - "$defs" => { - "ElasticGraphEventEnvelope" => event_envelope.json_schema(root_document_type_names, json_schema_version) - }.merge(definitions_by_name) - } - end - - def json_schema_indexing_field_types_by_name - @json_schema_indexing_field_types_by_name ||= state - .types_by_name - .except("Query") - .values - .reject do |t| - derived_indexing_type_names.include?(t.name) || - # Skip graphql types (including namespace types, which are GraphQL-only). - t.graphql_only? - end - .sort_by(&:name) - .to_h { |type| [type.name, type.to_indexing_field_type] } - end - def verify_runtime_metadata(runtime_metadata) registered_resolvers = runtime_metadata.graphql_resolvers_by_name diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb index ee27b537d..0ae2e7ff7 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb @@ -8,7 +8,6 @@ require "did_you_mean" require "elastic_graph/constants" -require "elastic_graph/json_ingestion/schema_definition/json_schema_pruner" require "elastic_graph/support/graphql_gem_loader" require "elastic_graph/support/memoizable_data" require "fileutils" @@ -36,41 +35,10 @@ def initialize(schema_definition_results:, schema_artifacts_directory:, output:, @schema_artifacts_directory = schema_artifacts_directory @output = output @max_diff_lines = max_diff_lines - - @json_schemas_artifact = new_yaml_artifact( - JSON_SCHEMAS_FILE, - ::ElasticGraph::JSONIngestion::SchemaDefinition::JSONSchemaPruner.prune(schema_definition_results.current_public_json_schema), - extra_comment_lines: [ - "This is the \"public\" JSON schema file and is intended to be provided to publishers so that", - "they can perform code generation and event validation." - ] - ) end # Dumps all the schema artifacts to disk. def dump_artifacts - check_if_needs_json_schema_version_bump do |recommended_json_schema_version| - if schema_definition_results.state.enforce_json_schema_version - # @type var setter_location: ::Thread::Backtrace::Location - # We use `_ =` because while `json_schema_version_setter_location` can be nil, - # it'll never be nil if we get here and we want the type to be non-nilable. - setter_location = _ = schema_definition_results.json_schema_version_setter_location - setter_location_path = ::Pathname.new(setter_location.absolute_path.to_s).relative_path_from(::Dir.pwd) - - abort "A change has been attempted to `json_schemas.yaml`, but the `json_schema_version` has not been correspondingly incremented. Please " \ - "increase the schema's version, and then run the `bundle exec rake schema_artifacts:dump` command again.\n\n" \ - "To update the schema version to the expected version, change line #{setter_location.lineno} at `#{setter_location_path}` to:\n" \ - " `schema.json_schema_version #{recommended_json_schema_version}`\n\n" \ - "Alternately, call `schema.enforce_json_schema_version false` in your schema definition to allow the JSON schemas file " \ - "to change without requiring a version bump, but that is only recommended for non-production applications during initial schema prototyping." - else - @output.puts <<~EOS - WARNING: the `json_schemas.yaml` artifact is being updated without the `json_schema_version` being correspondingly incremented. - This is not recommended for production applications, but is currently allowed because you have called `schema.enforce_json_schema_version false`. - EOS - end - end - ::FileUtils.mkdir_p(@schema_artifacts_directory) artifacts.each { |artifact| artifact.dump(@output) } end @@ -111,18 +79,11 @@ def artifacts_from_schema_def # schema elements. graphql_schema = ::GraphQL::Schema.from_definition(schema_definition_results.graphql_schema_string).to_definition.chomp - unversioned_artifacts = [ + [ new_yaml_artifact(DATASTORE_CONFIG_FILE, schema_definition_results.datastore_config), new_yaml_artifact(RUNTIME_METADATA_FILE, pruned_runtime_metadata(graphql_schema).to_dumpable_hash), - @json_schemas_artifact, new_raw_artifact(GRAPHQL_SCHEMA_FILE, "\n" + graphql_schema) ] - - versioned_artifacts = build_desired_versioned_json_schemas(@json_schemas_artifact.desired_contents).values.map do |versioned_schema| - new_versioned_json_schema_artifact(versioned_schema) - end - - unversioned_artifacts + versioned_artifacts end def notify_about_unused_type_name_overrides @@ -172,157 +133,6 @@ def notify_about_unused_enum_value_overrides EOS end - def build_desired_versioned_json_schemas(current_public_json_schema) - versioned_parsed_yamls = ::Dir.glob(::File.join(@schema_artifacts_directory, JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v*.yaml")).map do |file| - ::YAML.safe_load_file(file) - end + [current_public_json_schema] - - results_by_json_schema_version = versioned_parsed_yamls.to_h do |parsed_yaml| - merged_schema = @schema_definition_results.merge_field_metadata_into_json_schema(parsed_yaml) - [merged_schema.json_schema_version, merged_schema] - end - - report_json_schema_merge_errors(results_by_json_schema_version.values) - report_json_schema_merge_warnings - - results_by_json_schema_version.transform_values(&:json_schema) - end - - def report_json_schema_merge_errors(merged_results) - json_schema_versions_by_missing_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] - json_schema_versions_by_missing_type = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] - json_schema_versions_by_missing_necessary_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]] - - merged_results.each do |result| - result.missing_fields.each do |field| - json_schema_versions_by_missing_field[field] << result.json_schema_version - end - - result.missing_types.each do |type| - json_schema_versions_by_missing_type[type] << result.json_schema_version - end - - result.missing_necessary_fields.each do |missing_necessary_field| - json_schema_versions_by_missing_necessary_field[missing_necessary_field] << result.json_schema_version - end - end - - missing_field_errors = json_schema_versions_by_missing_field.map do |field, json_schema_versions| - missing_field_error_for(field, json_schema_versions) - end - - missing_type_errors = json_schema_versions_by_missing_type.map do |type, json_schema_versions| - missing_type_error_for(type, json_schema_versions) - end - - missing_necessary_field_errors = json_schema_versions_by_missing_necessary_field.map do |field, json_schema_versions| - missing_necessary_field_error_for(field, json_schema_versions) - end - - definition_conflict_errors = merged_results - .flat_map { |result| result.definition_conflicts.to_a } - .group_by(&:name) - .map do |name, deprecated_elements| - <<~EOS - The schema definition of `#{name}` has conflicts. To resolve the conflict, remove the unneeded definitions from the following: - - #{format_deprecated_elements(deprecated_elements)} - EOS - end - - errors = missing_field_errors + missing_type_errors + missing_necessary_field_errors + definition_conflict_errors - return if errors.empty? - - abort errors.join("\n\n") - end - - def report_json_schema_merge_warnings - unused_elements = @schema_definition_results.unused_deprecated_elements - return if unused_elements.empty? - - @output.puts <<~EOS - The schema definition has #{unused_elements.size} unneeded reference(s) to deprecated schema elements. These can all be safely deleted: - - #{format_deprecated_elements(unused_elements)} - - EOS - end - - def format_deprecated_elements(deprecated_elements) - descriptions = deprecated_elements - .sort_by { |e| [e.defined_at.path, e.defined_at.lineno] } - .map(&:description) - .uniq - - descriptions.each.with_index(1).map { |desc, idx| "#{idx}. #{desc}" }.join("\n") - end - - def missing_field_error_for(qualified_field, json_schema_versions) - type, field = qualified_field.split(".") - - <<~EOS - The `#{qualified_field}` field (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. - ElasticGraph cannot guess what it should do with this field's data when ingesting events at #{old_versions(json_schema_versions)}. - To continue, do one of the following: - - 1. If the `#{qualified_field}` field has been renamed, indicate this by calling `field.renamed_from "#{field}"` on the renamed field. - 2. If the `#{qualified_field}` field has been dropped, indicate this by calling `type.deleted_field "#{field}"` on the `#{type}` type. - 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. - EOS - end - - def missing_type_error_for(type, json_schema_versions) - <<~EOS - The `#{type}` type (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. - ElasticGraph cannot guess what it should do with this type's data when ingesting events at #{old_versions(json_schema_versions)}. - To continue, do one of the following: - - 1. If the `#{type}` type has been renamed, indicate this by calling `type.renamed_from "#{type}"` on the renamed type. - 2. If the `#{type}` field has been dropped, indicate this by calling `schema.deleted_type "#{type}"` on the schema. - 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. - EOS - end - - def missing_necessary_field_error_for(field, json_schema_versions) - path = field.fully_qualified_path.split(".").last - # :nocov: -- we only cover one side of this ternary. - has_or_have = (json_schema_versions.size == 1) ? "has" : "have" - # :nocov: - - <<~EOS - #{describe_json_schema_versions(json_schema_versions, "and")} #{has_or_have} no field that maps to the #{field.field_type} field path of `#{field.fully_qualified_path}`. - Since the field path is required for #{field.field_type}, ElasticGraph cannot ingest events that lack it. To continue, do one of the following: - - 1. If the `#{field.fully_qualified_path}` field has been renamed, indicate this by calling `field.renamed_from "#{path}"` on the renamed field rather than using `deleted_field`. - 2. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. - EOS - end - - def describe_json_schema_versions(json_schema_versions, conjunction) - json_schema_versions = json_schema_versions.sort - - # Steep doesn't support pattern matching yet, so have to skip type checking here. - __skip__ = case json_schema_versions - in [single_version] - "JSON schema version #{single_version}" - in [version1, version2] - "JSON schema versions #{version1} #{conjunction} #{version2}" - else - *versions, last_version = json_schema_versions - "JSON schema versions #{versions.join(", ")}, #{conjunction} #{last_version}" - end - end - - def old_versions(json_schema_versions) - return "this old version" if json_schema_versions.size == 1 - "these old versions" - end - - def files_noun_phrase(json_schema_versions) - return "its file" if json_schema_versions.size == 1 - "their files" - end - def artifacts_out_of_date_error(out_of_date_artifacts) # @type var diffs: ::Array[[SchemaArtifact[untyped], ::String]] diffs = [] @@ -377,20 +187,6 @@ def new_yaml_artifact(file_name, desired_contents, extra_comment_lines: []) ) end - def new_versioned_json_schema_artifact(desired_contents) - # File name depends on the schema_version field in the json schema. - schema_version = desired_contents[JSON_SCHEMA_VERSION_KEY] - - new_yaml_artifact( - ::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v#{schema_version}.yaml"), - desired_contents, - extra_comment_lines: [ - "This JSON schema file contains internal ElasticGraph metadata and should be considered private.", - "The unversioned JSON schema file is public and intended to be provided to publishers." - ] - ) - end - def new_raw_artifact(file_name, desired_contents) SchemaArtifact.new( ::File.join(@schema_artifacts_directory, file_name), @@ -401,17 +197,6 @@ def new_raw_artifact(file_name, desired_contents) ) end - def check_if_needs_json_schema_version_bump(&block) - if @json_schemas_artifact.out_of_date? - existing_schema_version = @json_schemas_artifact.existing_dumped_contents&.dig(JSON_SCHEMA_VERSION_KEY) || -1 - desired_schema_version = @json_schemas_artifact.desired_contents[JSON_SCHEMA_VERSION_KEY] - - if existing_schema_version >= desired_schema_version - yield existing_schema_version + 1 - end - end - end - def pruned_runtime_metadata(graphql_schema_string) schema = ::GraphQL::Schema.from_definition(graphql_schema_string) runtime_meta = schema_definition_results.runtime_metadata diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb index df513de53..9b8af66fa 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb @@ -406,19 +406,11 @@ def register_standard_elastic_graph_types # As per the Elasticsearch docs, the field MUST come in named `lat` in Elastisearch (but we want the full name in GraphQL). t.field names.latitude, "Float", name_in_index: "lat" do |f| f.documentation "Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90." - - # Note: we use `nullable: false` because we index it as a single `geo_point` field, and therefore can't - # support a `latitude` without a `longitude` or vice-versa. - f.json_schema minimum: -90, maximum: 90, nullable: false end # As per the Elasticsearch docs, the field MUST come in named `lon` in Elastisearch (but we want the full name in GraphQL). t.field names.longitude, "Float", name_in_index: "lon" do |f| f.documentation "Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180." - - # Note: we use `nullable: false` because we index it as a single `geo_point` field, and therefore can't - # support a `latitude` without a `longitude` or vice-versa. - f.json_schema minimum: -180, maximum: 180, nullable: false end t.mapping type: "geo_point" @@ -660,17 +652,15 @@ def register_standard_elastic_graph_types # Registers the standard GraphQL scalar types. Note that the SDL for the scalar type itself isn't # included in the dumped SDL, but registering it allows us to derive a filter for each, - # which we need. In addition, this lets us define the mapping and JSON schema for each standard - # scalar type. + # which we need. In addition, this lets us define the mapping for each standard scalar type. + # Ingestion serializers can layer their own built-in configuration on top. def register_standard_graphql_scalars schema_def_api.scalar_type "Boolean" do |t| t.mapping type: "boolean" - t.json_schema type: "boolean" end schema_def_api.scalar_type "Float" do |t| t.mapping type: "double" - t.json_schema type: "number" t.customize_aggregated_values_type do |avt| # not nullable, since sum(empty_set) == 0 @@ -710,12 +700,10 @@ def register_standard_graphql_scalars schema_def_api.scalar_type "ID" do |t| t.mapping type: "keyword" - t.json_schema type: "string" end schema_def_api.scalar_type "Int" do |t| t.mapping type: "integer" - t.json_schema type: "integer", minimum: INT_MIN, maximum: INT_MAX t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Integer", defined_at: "elastic_graph/indexer/indexing_preparers/integer" @@ -730,7 +718,6 @@ def register_standard_graphql_scalars schema_def_api.scalar_type "String" do |t| t.mapping type: "keyword" - t.json_schema type: "string" t.customize_filter_input_type do |fit| fit.field names.contains, schema_def_state.type_ref("StringContains").as_filter_input.name do |f| @@ -754,12 +741,11 @@ def register_standard_graphql_scalars def register_custom_elastic_graph_scalars schema_def_api.scalar_type "Cursor" do |t| - # Technically, we don't use the mapping or json_schema on this type since it's a return-only + # Technically, we don't use the mapping or ingestion config on this type since it's a return-only # type and isn't indexed. However, `scalar_type` requires them to be set (since custom scalars # defined by users will need those set) so we set them here to what they would be if we actually # used them. t.mapping type: "keyword" - t.json_schema type: "string" t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::Cursor", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/cursor" @@ -772,7 +758,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "Date" do |t| t.mapping type: "date", format: DATASTORE_DATE_FORMAT - t.json_schema type: "string", format: "date" t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::Date", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/date" @@ -792,7 +777,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "DateTime" do |t| t.mapping type: "date", format: DATASTORE_DATE_TIME_FORMAT - t.json_schema type: "string", format: "date-time" t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::DateTime", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/date_time" t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::DateTime", @@ -883,8 +867,6 @@ def register_custom_elastic_graph_scalars t.mapping type: "date", format: "HH:mm:ss||HH:mm:ss.S||HH:mm:ss.SS||HH:mm:ss.SSS" - t.json_schema type: "string", pattern: VALID_LOCAL_TIME_JSON_SCHEMA_PATTERN - t.customize_aggregated_values_type do |avt| define_exact_min_max_and_approx_avg_on_aggregated_values(avt, "LocalTime") do |adjective:, full_name:| <<~EOS @@ -897,7 +879,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "TimeZone" do |t| t.mapping type: "keyword" - t.json_schema type: "string", enum: GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.to_a t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::TimeZone", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/time_zone" @@ -914,8 +895,6 @@ def register_custom_elastic_graph_scalars # https://github.com/json-schema-org/json-schema-spec/blob/draft-07/schema.json#L23-L29 # # ...except we are omitting `null` here; it'll be added by the nullability decorator if the field is defined as nullable. - t.json_schema type: ["array", "boolean", "integer", "number", "object", "string"] - # In the index we store this as a JSON string in a `keyword` field. t.mapping type: "keyword" @@ -940,7 +919,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "JsonSafeLong" do |t| t.mapping type: "long" - t.json_schema type: "integer", minimum: JSON_SAFE_LONG_MIN, maximum: JSON_SAFE_LONG_MAX t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::JsonSafeLong", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/longs" @@ -984,7 +962,6 @@ def register_custom_elastic_graph_scalars # to do if we ingest them as strings. (The `pattern` regex to validate the range # would be *extremely* complicated). t.mapping type: "long" - t.json_schema type: "integer", minimum: LONG_STRING_MIN, maximum: LONG_STRING_MAX t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::LongString", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/longs" t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Integer", diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb index 9e51ed105..57a1f67ac 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb @@ -157,9 +157,11 @@ def derived_graphql_types end # Hook for extensions to customize the scalar type derived from an enum type. + # @param scalar_type [ScalarType] the scalar type to configure + # @return [void] # @api private def configure_derived_scalar_type(scalar_type) - scalar_type.json_schema type: "string" + # No-op by default; extensions (e.g. JSONIngestion) override this. end # @return [Indexing::FieldType::Enum] indexing representation of this enum type diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb index 3d66eeb62..af4512972 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb @@ -85,8 +85,6 @@ module SchemaElements # @private # @!attribute [rw] computation_detail # @private - # @!attribute [rw] non_nullable_in_json_schema - # @private # @!attribute [rw] as_input # @private class Field < Struct.new( @@ -95,7 +93,7 @@ class Field < Struct.new( :aggregated_values_customizations, :sort_order_enum_value_customizations, :args, :sortable, :filterable, :aggregatable, :groupable, :highlightable, :returnable, :graphql_only, :source, :runtime_field_script, :relationship, :singular_name, - :computation_detail, :non_nullable_in_json_schema, :as_input, + :computation_detail, :as_input, :name_in_index, :resolver ) include Mixins::HasDocumentation @@ -140,7 +138,6 @@ def initialize( # the `_name` suffix on the attribute for clarity. singular_name: singular, name_in_index: name_in_index, - non_nullable_in_json_schema: false, as_input: as_input, resolver: resolver ) @@ -468,22 +465,6 @@ def on_each_generated_schema_element(&customization_block) customize_sort_order_enum_values(&customization_block) end - # (see Mixins::HasTypeInfo#json_schema) - def json_schema(nullable: nil, **options) - if options.key?(:type) - raise Errors::SchemaError, "Cannot override JSON schema type of field `#{name}` with `#{options.fetch(:type)}`" - end - - case nullable - when true - raise Errors::SchemaError, "`nullable: true` is not allowed on a field--just declare the GraphQL field as being nullable (no `!` suffix) instead." - when false - self.non_nullable_in_json_schema = true - end - - super(**options) - end - # (see Mixins::HasTypeInfo#mapping) def mapping(**options) # ElasticGraph has special handling for the nested type (e.g. we generate sub-aggregation types in the GraphQL schema for @@ -1010,9 +991,8 @@ def to_indexing_field_reference Indexing::FieldReference.new( name: name, name_in_index: name_in_index, - type: non_nullable_in_json_schema ? type.wrap_non_null : type, + type: type, mapping_options: mapping_options, - json_schema_options: json_schema_options, accuracy_confidence: accuracy_confidence, source: source, runtime_field_script: runtime_field_script, diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb index 8e2703cd2..e3f2844a9 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb @@ -25,7 +25,7 @@ module SchemaElements # ElasticGraph.define_schema do |schema| # schema.scalar_type "URL" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", format: "uri" + # t.json_schema type: "string" # end # end # @@ -60,7 +60,7 @@ class ScalarType < Struct.new( include Mixins::HasReadableToSAndInspect.new { |t| t.name } # `HasTypeInfo` provides the following methods: - # @dynamic mapping_options, json_schema_options + # @dynamic mapping_options include Mixins::HasTypeInfo # @dynamic graphql_only? @@ -78,13 +78,8 @@ def initialize(schema_def_state, name) yield self - missing = [ - ("`mapping`" if mapping_options.empty?), - ("`json_schema`" if json_schema_options.empty?) - ].compact - - if missing.any? - raise Errors::SchemaError, "Scalar types require `mapping` and `json_schema` to be configured, but `#{name}` lacks #{missing.join(" and ")}." + if mapping_options.empty? + raise Errors::SchemaError, "Scalar types require `mapping` to be configured, but `#{name}` lacks `mapping`." end if (placeholder = inferred_grouping_missing_value_placeholder) @@ -343,23 +338,8 @@ def inferred_grouping_missing_value_placeholder MISSING_STRING_PLACEHOLDER elsif FLOAT_TYPES.include?(mapping_type) MISSING_NUMERIC_PLACEHOLDER - elsif mapping_type == "long" - # It is only safe to use NaN for a long when the long's range is safe to coerce to a float - # without loss of precision. This is because using NaN as the missing value will cause - # the datastore to coerce the other bucket keys to float. - # JSON schema min/max only constrains newly indexed values, not existing data that may fall outside the range before the constraints were added. - # This is an edge case where the long range may exceed safe float precision. - # In this case, users can set grouping_missing_value_placeholder to nil. - if (json_schema_options[:minimum] || LONG_STRING_MIN) >= JSON_SAFE_LONG_MIN && - (json_schema_options[:maximum] || LONG_STRING_MAX) <= JSON_SAFE_LONG_MAX - inferred_numeric_placeholder_for_integer_type - end - elsif mapping_type == "unsigned_long" - # Similar to the checks above for long except we only need to check the max - # (since the min is zero even if not specified) - if (json_schema_options[:maximum] || LONG_STRING_MAX) <= JSON_SAFE_LONG_MAX - inferred_numeric_placeholder_for_integer_type - end + elsif mapping_type == "long" || mapping_type == "unsigned_long" + nil elsif INTEGER_TYPES.include?(mapping_type) # All other integer types can safely be coerced to float without loss of precision inferred_numeric_placeholder_for_integer_type diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb index 483d296bf..22c574977 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb @@ -171,20 +171,6 @@ def with_reverted_override schema_def_state.type_ref(type_namer.revert_override_for(name)) end - # Returns all the JSON schema array/nullable layers of a type, from outermost to innermost. - # For example, [[Int]] will return [:nullable, :array, :nullable, :array, :nullable] - def json_schema_layers - @json_schema_layers ||= begin - layers, inner_type = peel_json_schema_layers_once - - if layers.empty? || inner_type == self - layers - else - layers + inner_type.json_schema_layers - end - end - end - # Most of ElasticGraph's derived GraphQL types have a static suffix (e.g. the full type name # is source_type + suffix). This is a map of all of these. STATIC_FORMAT_NAME_BY_CATEGORY = TypeNamer::REQUIRED_PLACEHOLDERS.filter_map do |format_name, placeholders| @@ -304,16 +290,6 @@ def after_initialize Mixins::VerifiesGraphQLName.verify_name!(unwrapped_name) end - def peel_json_schema_layers_once - if list? - return [[:array], unwrap_list] if non_null? - return [[:nullable, :array], unwrap_list] - end - - return [[], unwrap_non_null] if non_null? - [[:nullable], self] - end - def matches_format_of?(category) format_name = STATIC_FORMAT_NAME_BY_CATEGORY.fetch(category) type_namer.matches_format?(name, format_name) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb index 4fc5f9c2f..c408dcd12 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb @@ -158,7 +158,7 @@ def name # Therefore, we recommend limiting your use of `!` to only a few situations such as defining a type’s primary key (e.g. # `t.field "id", "ID!"`) or defining a list field (e.g. `t.field "authors", "[String!]!"`) since empty lists already provide a # "no data" representation. You can still configure the ElasticGraph indexer to require a non-null value for a field using - # `f.json_schema nullable: false`. + # `f.json_schema nullable: false` (when using `elasticgraph-json_ingestion`). # # @note ElasticGraph’s understanding of datastore capabilities may override your configured # `aggregatable`/`filterable`/`groupable`/`sortable` options. For example, a field indexed as `text` for full text search will @@ -496,7 +496,6 @@ def to_indexing_field_type type_name: name, subfields: indexing_fields_by_name_in_index.values.map(&:to_indexing_field).compact, mapping_options: mapping_options, - json_schema_options: json_schema_options, doc_comment: doc_comment ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb index 2942671d8..0b65b0ee5 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb @@ -6,7 +6,6 @@ # # frozen_string_literal: true -require "elastic_graph/constants" require "elastic_graph/errors" require "elastic_graph/schema_definition/factory" require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" @@ -42,9 +41,6 @@ class State < Struct.new( :renamed_fields_by_type_name_and_old_field_name, :deleted_fields_by_type_name_and_old_field_name, :reserved_type_names, - :json_schema_version, - :json_schema_version_setter_location, - :enforce_json_schema_version, :graphql_extension_modules, :graphql_resolvers_by_name, :built_in_graphql_resolvers, @@ -57,8 +53,6 @@ class State < Struct.new( :output, :type_namer, :enum_value_namer, - :allow_omitted_json_schema_fields, - :allow_extra_json_schema_fields, :indexed_types_by_index_name ) include Mixins::HasReadableToSAndInspect.new @@ -93,10 +87,7 @@ def self.with( deleted_types_by_old_name: {}, renamed_fields_by_type_name_and_old_field_name: ::Hash.new { |h, k| h[k] = {} }, deleted_fields_by_type_name_and_old_field_name: ::Hash.new { |h, k| h[k] = {} }, - reserved_type_names: RESERVED_TYPE_NAMES.dup, - json_schema_version_setter_location: nil, - json_schema_version: nil, - enforce_json_schema_version: true, + reserved_type_names: ::Set.new, graphql_extension_modules: [], graphql_resolvers_by_name: {}, built_in_graphql_resolvers: ::Set.new, @@ -112,8 +103,6 @@ def self.with( ), enum_value_namer: SchemaElements::EnumValueNamer.new(enum_value_overrides_by_type), output: output, - allow_omitted_json_schema_fields: false, - allow_extra_json_schema_fields: true, indexed_types_by_index_name: {} ) end @@ -121,6 +110,15 @@ def self.with( # @dynamic index_document_sizes? alias_method :index_document_sizes?, :index_document_sizes + # Returns the JSON schema version recorded on this state, or `nil` if none has been set. + # + # The default implementation returns `nil` so callers can query it uniformly regardless of whether + # an ingestion-serializer extension is active. When `elasticgraph-json_ingestion` is loaded its + # `StateExtension` overrides this with a real `attr_accessor` backed by the extension's instance state. + def json_schema_version + nil + end + def type_ref(name) # Type references are immutable and can be safely cached. Here we cache them because we've observed # it having a noticeable impact on our test suite runtime. @@ -228,8 +226,6 @@ def field_path_resolver private - RESERVED_TYPE_NAMES = [EVENT_ENVELOPE_JSON_SCHEMA_NAME].to_set - def register_type(type, additional_type_index = nil) name = (_ = type).name diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb index c4c10fb89..84ab0b194 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb @@ -75,7 +75,9 @@ def define_schema_with_schema_elements( yield api if block_given? - # Set the json_schema_version to the provided value, if needed. + # Set the json_schema_version to the provided value, if needed. `API#json_schema_version` and + # `State#json_schema_version` are no-ops when no ingestion-serializer extension is loaded, so we + # can call through unconditionally — the JSON-ingestion extension overrides both. if !json_schema_version.nil? && api.state.json_schema_version.nil? api.json_schema_version(json_schema_version) end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs index 797db12cb..514bfab31 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs @@ -65,7 +65,6 @@ module ElasticGraph ?output: io ) -> void - def json_schema_strictness: (?allow_omitted_fields: bool, ?allow_extra_fields: bool) -> void def raw_sdl: (::String) -> void def object_type: (::String) { (SchemaElements::ObjectType) -> void } -> void def namespace_type: (::String) ?{ (SchemaElements::NamespaceType) -> void } -> void @@ -77,12 +76,13 @@ module ElasticGraph def as_active_instance: { () -> void } -> void @results: Results? def results: () -> Results - def json_schema_version: (::Integer) -> void - def enforce_json_schema_version: (bool) -> void def register_graphql_extension: (::Module, defined_at: ::String, **untyped) -> void def register_graphql_resolver: (::Symbol, ::Class, defined_at: ::String, ?built_in: bool, **untyped) -> void def on_built_in_types: () { (SchemaElements::graphQLType) -> void } -> void def on_root_query_type: () { (SchemaElements::ObjectType) -> void } -> void + def json_schema_version: (::Integer) -> void + def enforce_json_schema_version: (bool) -> void + def json_schema_strictness: (?allow_omitted_fields: bool, ?allow_extra_fields: bool) -> void end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs index 2b21d08f7..5230505ec 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs @@ -5,10 +5,8 @@ module ElasticGraph attr_reader name: ::String attr_reader name_in_index: ::String attr_reader type: SchemaElements::TypeReference - attr_reader json_schema_layers: jsonSchemaLayersArray attr_reader indexing_field_type: _FieldType attr_reader accuracy_confidence: Field::accuracyConfidence - attr_reader json_schema_customizations: ::Hash[::Symbol, untyped] attr_reader mapping_customizations: ::Hash[::Symbol, untyped] attr_reader source: SchemaElements::FieldSource? attr_accessor runtime_field_script: ::String? @@ -18,10 +16,8 @@ module ElasticGraph name: ::String, name_in_index: ::String, type: SchemaElements::TypeReference, - json_schema_layers: jsonSchemaLayersArray, indexing_field_type: _FieldType, accuracy_confidence: Field::accuracyConfidence, - json_schema_customizations: ::Hash[::Symbol, untyped], mapping_customizations: ::Hash[::Symbol, untyped], source: SchemaElements::FieldSource?, runtime_field_script: ::String?, @@ -32,10 +28,8 @@ module ElasticGraph ?name: ::String, ?name_in_index: ::String, ?type: SchemaElements::TypeReference, - ?json_schema_layers: jsonSchemaLayersArray, ?indexing_field_type: _FieldType, ?accuracy_confidence: Field::accuracyConfidence, - ?json_schema_customizations: ::Hash[::Symbol, untyped], ?mapping_customizations: ::Hash[::Symbol, untyped], ?source: SchemaElements::FieldSource?, ?runtime_field_script: ::String?, @@ -44,25 +38,11 @@ module ElasticGraph end class Field < FieldSupertype - JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE: ::Hash[::String, untyped] - type accuracyConfidence = SchemaElements::Field::accuracyConfidence @mapping: ::Hash[::String, untyped]? def mapping: () -> ::Hash[::String, untyped] - def json_schema: () -> ::Hash[::String, untyped] - def json_schema_metadata: () -> ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata def self.normalized_mapping_hash_for: (::Array[Field]) -> ::Hash[::String, untyped] - - def inner_json_schema: () -> ::Hash[::String, untyped] - def outer_json_schema_customizations: () -> ::Hash[::String, untyped] - - def nullable?: () -> bool - - def user_specified_json_schema_customizations_go_on_outside?: () -> bool - def process_layer: (::Symbol, ::Hash[::String, untyped]) -> ::Hash[::String, untyped] - def make_nullable: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] - def make_array: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs index 55a8f0724..89071a597 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs @@ -6,7 +6,6 @@ module ElasticGraph attr_reader name_in_index: ::String attr_reader type: SchemaElements::TypeReference attr_reader mapping_options: ::Hash[::Symbol, untyped] - attr_reader json_schema_options: ::Hash[::Symbol, untyped] attr_reader accuracy_confidence: Field::accuracyConfidence attr_reader source: SchemaElements::FieldSource? attr_reader runtime_field_script: ::String? @@ -17,7 +16,6 @@ module ElasticGraph name_in_index: ::String, type: SchemaElements::TypeReference, mapping_options: ::Hash[::Symbol, untyped], - json_schema_options: ::Hash[::Symbol, untyped], accuracy_confidence: Field::accuracyConfidence, source: SchemaElements::FieldSource?, runtime_field_script: ::String?, @@ -29,7 +27,6 @@ module ElasticGraph ?name_in_index: ::String, ?type: SchemaElements::TypeReference, ?mapping_options: ::Hash[::Symbol, untyped], - ?json_schema_options: ::Hash[::Symbol, untyped], ?accuracy_confidence: Field::accuracyConfidence, ?source: SchemaElements::FieldSource?, ?runtime_field_script: ::String?, diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs index c4f144bfd..d01805f99 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs @@ -3,9 +3,6 @@ module ElasticGraph module Indexing interface _FieldType def to_mapping: () -> ::Hash[::String, untyped] - def to_json_schema: () -> ::Hash[::String, untyped] - def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata] - def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs index 5dd23dbaa..e1b90df44 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs @@ -3,29 +3,26 @@ module ElasticGraph module Indexing module FieldType class ObjectSuperType + attr_reader schema_def_state: State attr_reader type_name: ::String attr_reader subfields: ::Array[Field] attr_reader mapping_options: Mixins::HasTypeInfo::optionsHash - attr_reader json_schema_options: Mixins::HasTypeInfo::optionsHash attr_reader doc_comment: ::String? - attr_reader schema_def_state: State def initialize: ( + schema_def_state: State, type_name: ::String, subfields: ::Array[Field], mapping_options: Mixins::HasTypeInfo::optionsHash, - json_schema_options: Mixins::HasTypeInfo::optionsHash, - doc_comment: ::String?, - schema_def_state: State + doc_comment: ::String? ) -> void def with: ( + ?schema_def_state: State, ?type_name: ::String, ?subfields: ::Array[Field], ?mapping_options: Mixins::HasTypeInfo::optionsHash, - ?json_schema_options: Mixins::HasTypeInfo::optionsHash, - ?doc_comment: ::String?, - ?schema_def_state: State + ?doc_comment: ::String? ) -> Object end @@ -34,12 +31,6 @@ module ElasticGraph include Support::_MemoizableDataClass @to_mapping: ::Hash[::String, untyped]? - @to_json_schema: ::Hash[::String, untyped]? - - private - - def json_schema_typename_field: () -> ::Hash[::String, untyped] - def validate_sourced_fields_have_no_json_schema_overrides: (::Array[Field]) -> void end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs index 4a8b4c1d3..9c892787b 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs @@ -10,7 +10,7 @@ module ElasticGraph attr_reader indexed_type: indexableType attr_reader schema_def_state: State - HAS_NON_WHITE_SPACE_REGEX: ::String + HAS_NON_WHITE_SPACE_REGEX: ::Regexp def rollover: (::Symbol, ::String) -> void def route_with: (::String) -> void diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs index 1f3c1391b..7ad7426ac 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs @@ -6,9 +6,9 @@ module ElasticGraph type optionsHash = ::Hash[::Symbol, untyped] attr_reader mapping_options: optionsHash - attr_reader json_schema_options: optionsHash def mapping: (**untyped) -> void + def json_schema_options: () -> optionsHash def json_schema: (**untyped) -> void end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs index 41012f526..11a4c10ba 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs @@ -6,43 +6,30 @@ module ElasticGraph end class Results < ResultsSupertype - include _SchemaArtifacts include Support::_MemoizableDataClass - def json_schema_version_setter_location: () -> ::Thread::Backtrace::Location? - def json_schema_field_metadata_by_type_and_field_name: () -> ::Hash[::String, ::Hash[::String, ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata]] - def current_public_json_schema: () -> ::Hash[::String, untyped] - def merge_field_metadata_into_json_schema: (::Hash[::String, untyped]) -> ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata - def unused_deprecated_elements: () -> ::Set[SchemaElements::DeprecatedElement] + def graphql_schema_string: () -> ::String + def datastore_config: () -> ::Hash[::String, untyped] + def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Schema def derived_indexing_type_names: () -> ::Set[::String] @graphql_schema_string: ::String? @datastore_config: ::Hash[::String, untyped] @runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema? - @current_json_schemas: ::Hash[::String, untyped]? @static_script_repo: Scripting::FileSystemRepository? - @available_json_schema_versions: ::Set[::Integer]? @no_circular_dependencies: bool? @field_path_resolver: SchemaElements::FieldPath::Resolver? - @json_schema_indexing_field_types_by_name: ::Hash[::String, Indexing::_FieldType]? @derived_indexing_type_names: ::Set[::String]? - @json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata]]? - @current_public_json_schema: ::Hash[::String, untyped]? - @latest_versioned_json_schema: ::Hash[::String, untyped]? - @json_schema_with_metadata_merger: ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger? STATIC_SCRIPT_REPO: Scripting::FileSystemRepository private - def json_schema_with_metadata_merger: () -> ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger def generate_datastore_config: () -> ::Hash[::String, untyped] def build_dynamic_scripts: () -> ::Array[Scripting::Script] def build_runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Schema def identify_extra_update_targets_by_object_type_name: () -> ::Hash[::String, ::Array[SchemaArtifacts::RuntimeMetadata::UpdateTarget]] def generate_sdl: () -> ::String - def build_public_json_schema: () -> ::Hash[::String, untyped] - def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, Indexing::_FieldType] def verify_runtime_metadata: (SchemaArtifacts::RuntimeMetadata::Schema) -> void def strip_trailing_whitespace: (::String) -> ::String def check_for_circular_dependencies!: () -> void diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs index 439aa3011..713693ac3 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs @@ -20,22 +20,11 @@ module ElasticGraph @output: io @max_diff_lines: ::Integer @artifacts: ::Array[SchemaArtifact[untyped]]? - @json_schemas_artifact: SchemaArtifact[untyped] def artifacts: () -> ::Array[SchemaArtifact[untyped]] def artifacts_from_schema_def: () -> ::Array[SchemaArtifact[untyped]] def notify_about_unused_type_name_overrides: () -> void def notify_about_unused_enum_value_overrides: () -> void - def build_desired_versioned_json_schemas: (::Hash[::String, untyped]) -> ::Hash[::Integer, ::Hash[::String, untyped]] - def report_json_schema_merge_errors: (::Array[::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata]) -> void - def report_json_schema_merge_warnings: () -> void - def format_deprecated_elements: (::Enumerable[SchemaElements::DeprecatedElement]) -> ::String - def missing_field_error_for: (::String, ::Array[::Integer]) -> ::String - def missing_type_error_for: (::String, ::Array[::Integer]) -> ::String - def missing_necessary_field_error_for: (::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]) -> ::String - def describe_json_schema_versions: (::Array[::Integer], ::String) -> ::String - def old_versions: (::Array[::Integer]) -> ::String - def files_noun_phrase: (::Array[::Integer]) -> ::String def artifacts_out_of_date_error: (::Array[SchemaArtifact[untyped]]) -> ::String def truncate_diff: (::String, ::Integer) -> [::String, ::String] @@ -45,9 +34,7 @@ module ElasticGraph ?extra_comment_lines: ::Array[::String] ) -> SchemaArtifact[::Hash[::String, untyped]] - def new_versioned_json_schema_artifact: (::Hash[::String, untyped]) -> SchemaArtifact[::Hash[::String, untyped]] def new_raw_artifact: (::String, ::String) -> SchemaArtifact[::String] - def check_if_needs_json_schema_version_bump: () { (::Integer) -> void } -> void def pruned_runtime_metadata: (::String) -> SchemaArtifacts::RuntimeMetadata::Schema end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs index b32e01646..07f2dc653 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs @@ -22,7 +22,6 @@ module ElasticGraph attr_accessor computation_detail: SchemaArtifacts::RuntimeMetadata::ComputationDetail attr_reader filter_customizations: ::Array[^(Field) -> void] attr_reader sort_order_enum_value_customizations: ::Array[^(SortOrderEnumValue) -> void] - attr_reader non_nullable_in_json_schema: bool attr_reader source: FieldSource? attr_accessor relationship: Relationship? attr_reader resolver: SchemaArtifacts::RuntimeMetadata::ConfiguredGraphQLResolver? diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs index 181b563a6..47cb75435 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs @@ -31,7 +31,6 @@ module ElasticGraph def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::ScalarType def mapping_options: () -> Mixins::HasTypeInfo::optionsHash - def json_schema_options: () -> Mixins::HasTypeInfo::optionsHash private diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs index 9d418a38f..388116f00 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs @@ -30,9 +30,6 @@ module ElasticGraph def scalar_type_needing_grouped_by_object?: () -> bool def with_reverted_override: () -> TypeReference - @json_schema_layers: jsonSchemaLayersArray? - def json_schema_layers: () -> jsonSchemaLayersArray - def to_final_form: (?as_input: bool) -> TypeReference STATIC_FORMAT_NAME_BY_CATEGORY: ::Hash[::Symbol, ::Symbol] @@ -59,8 +56,6 @@ module ElasticGraph private - def peel_json_schema_layers_once: () -> [jsonSchemaLayersArray, TypeReference] - def matches_format_of?: (::Symbol) -> bool def parent_aggregation_type: (::Array[::String]) -> ::String def renamed_with_same_wrappings: (::String) -> TypeReference diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs index b7adcd58b..8431b7c3e 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs @@ -19,9 +19,6 @@ module ElasticGraph attr_reader renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] attr_reader deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] attr_reader reserved_type_names: ::Set[::String] - attr_accessor json_schema_version: ::Integer? - attr_accessor json_schema_version_setter_location: ::Thread::Backtrace::Location? - attr_accessor enforce_json_schema_version: bool attr_reader graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::GraphQLExtension] attr_reader graphql_resolvers_by_name: ::Hash[::Symbol, SchemaArtifacts::RuntimeMetadata::GraphQLResolver] attr_reader built_in_graphql_resolvers: ::Set[::Symbol] @@ -31,11 +28,9 @@ module ElasticGraph attr_accessor user_definition_complete_callbacks: ::Array[^() -> void] attr_accessor sub_aggregation_paths_by_type: ::Hash[Mixins::SupportsFilteringAndAggregation, ::Array[SchemaElements::SubAggregationPath]] attr_accessor type_refs_by_name: ::Hash[::String, SchemaElements::TypeReference] + attr_accessor output: io attr_reader type_namer: SchemaElements::TypeNamer attr_reader enum_value_namer: SchemaElements::EnumValueNamer - attr_accessor output: io - attr_accessor allow_omitted_json_schema_fields: bool - attr_accessor allow_extra_json_schema_fields: bool attr_accessor indexed_types_by_index_name: ::Hash[::String, indexableType] def initialize: ( @@ -57,9 +52,6 @@ module ElasticGraph renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]], deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]], reserved_type_names: ::Set[::String], - json_schema_version: Integer?, - json_schema_version_setter_location: ::Thread::Backtrace::Location?, - enforce_json_schema_version: bool, graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::GraphQLExtension], graphql_resolvers_by_name: ::Hash[::Symbol, SchemaArtifacts::RuntimeMetadata::GraphQLResolver], built_in_graphql_resolvers: ::Set[::Symbol], @@ -69,12 +61,10 @@ module ElasticGraph user_definition_complete_callbacks: ::Array[^() -> void], sub_aggregation_paths_by_type: ::Hash[Mixins::SupportsFilteringAndAggregation, ::Array[SchemaElements::SubAggregationPath]], type_refs_by_name: ::Hash[::String, SchemaElements::TypeReference], + output: io, type_namer: SchemaElements::TypeNamer, enum_value_namer: SchemaElements::EnumValueNamer, - output: io, - allow_omitted_json_schema_fields: bool, - allow_extra_json_schema_fields: bool, - indexed_types_by_index_name: ::Hash[::String, indexableType], + indexed_types_by_index_name: ::Hash[::String, indexableType] ) -> void end @@ -91,6 +81,7 @@ module ElasticGraph def index_document_sizes?: () -> bool def type_ref: (::String) -> SchemaElements::TypeReference + def json_schema_version: () -> ::Integer? def register_index: (::String, indexableType) -> void def register_object_interface_or_union_type: (SchemaElements::ObjectType | SchemaElements::InterfaceType | SchemaElements::UnionType) -> void @@ -121,7 +112,6 @@ module ElasticGraph private - RESERVED_TYPE_NAMES: ::Set[::String] def register_type: [T] (T & SchemaElements::graphQLType, ?::Hash[::String, T]?) -> T end end diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb index 54a235ec8..fc73ccf83 100644 --- a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb @@ -8,6 +8,7 @@ require "bundler" require "elastic_graph/constants" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" require "elastic_graph/schema_definition/schema_elements/type_namer" require "graphql" @@ -216,7 +217,12 @@ module SchemaDefinition "`schema.json_schema_version 4`" ).and matching(json_schema_version_setter_location_regex) - write_elastic_graph_schema_def_code(component_suffix: "3", json_schema_version: 3, component_extras: "t.renamed_from 'Component'", enforce_json_schema_version: false) + write_elastic_graph_schema_def_code( + component_suffix: "3", + json_schema_version: 3, + component_extras: "t.renamed_from 'Component'", + enforce_json_schema_version: false + ) expect { output = run_rake("schema_artifacts:dump") @@ -494,9 +500,13 @@ module SchemaDefinition }) ) - # Here we add a different new field (`ordinal: Int!`), without bumping the version, and disable - # enforcement so we do not have to bump the version. - write_elastic_graph_schema_def_code(json_schema_version: 2, component_name_extras: "\nt.field 'ordinal', 'Int!'", enforce_json_schema_version: false) + # Here we add a different new field (`ordinal: Int!`), without bumping the version (and using `enforce_json_schema_version: false` + # to not have to bump the version)... + write_elastic_graph_schema_def_code( + json_schema_version: 2, + component_name_extras: "\nt.field 'ordinal', 'Int!'", + enforce_json_schema_version: false + ) run_rake("schema_artifacts:dump") # It should not be added to the v1 schema... @@ -854,6 +864,7 @@ module SchemaDefinition ::File.write("Rakefile", <<~EOS) project_root = "#{CommonSpecHelpers::REPO_ROOT}" + require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" ElasticGraph::SchemaDefinition::RakeTasks.new( @@ -861,6 +872,7 @@ module SchemaDefinition index_document_sizes: true, path_to_schema: "\#{project_root}/config/schema.rb", schema_artifacts_directory: "\#{project_root}/config/schema/artifacts", + extension_modules: [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension] ) EOS @@ -902,7 +914,7 @@ def expect_successful_run_of(*shell_commands) /line 7 at `(\S*\/?)schema\.rb`/ end - def write_elastic_graph_schema_def_code(json_schema_version:, enforce_json_schema_version: true, component_suffix: "", extra_sdl: "", component_name_extras: "", component_extras: "", omit_component_name_field: false) + def write_elastic_graph_schema_def_code(json_schema_version:, component_suffix: "", extra_sdl: "", component_name_extras: "", component_extras: "", omit_component_name_field: false, enforce_json_schema_version: true) code = <<~EOS Thread.current[:eg_schema_load_count] = (Thread.current[:eg_schema_load_count] || 0) + 1 if Thread.current[:eg_schema_load_count] > 1 @@ -911,7 +923,7 @@ def write_elastic_graph_schema_def_code(json_schema_version:, enforce_json_schem ElasticGraph.define_schema do |schema| schema.json_schema_version #{json_schema_version} - #{"schema.enforce_json_schema_version false" unless enforce_json_schema_version} + #{'schema.enforce_json_schema_version false' unless enforce_json_schema_version} schema.enum_type "Size" do |t| t.values "SMALL", "MEDIUM", "LAGE" end @@ -1088,7 +1100,7 @@ def as_active_instance index_document_sizes: true, path_to_schema: path_to_schema, schema_artifacts_directory: "config/schema/artifacts", - extension_modules: [extension_module].compact, + extension_modules: [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension, extension_module].compact, derived_type_name_formats: derived_type_name_formats, type_name_overrides: type_name_overrides, enum_value_overrides_by_type: enum_value_overrides_by_type, diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb new file mode 100644 index 000000000..03c1f21eb --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb @@ -0,0 +1,28 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "elastic_graph/schema_definition/api" + +module ElasticGraph + module SchemaDefinition + RSpec.describe Factory do + it "can create an object type without a customization block" do + api = API.new( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :snake_case, overrides: {}), + true, + extension_modules: [] + ) + + object_type = api.factory.new_object_type("Widget") + + expect(object_type.name).to eq("Widget") + end + end + end +end diff --git a/elasticgraph-warehouse/spec/integration/elastic_graph/warehouse/schema_definition/rake_tasks_spec.rb b/elasticgraph-warehouse/spec/integration/elastic_graph/warehouse/schema_definition/rake_tasks_spec.rb index da74eb579..c13ca18dd 100644 --- a/elasticgraph-warehouse/spec/integration/elastic_graph/warehouse/schema_definition/rake_tasks_spec.rb +++ b/elasticgraph-warehouse/spec/integration/elastic_graph/warehouse/schema_definition/rake_tasks_spec.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/constants" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" require "elastic_graph/warehouse/schema_definition/api_extension" require "yaml" @@ -161,7 +162,7 @@ def write_warehouse_schema(table_defs:, enforce_json_schema_version: true) ::File.write("schema.rb", <<~EOS) ElasticGraph.define_schema do |s| s.json_schema_version 1 - #{"s.enforce_json_schema_version false" unless enforce_json_schema_version} + #{'s.enforce_json_schema_version false' unless enforce_json_schema_version} # Add a dummy indexed type to ensure the Query type has at least one field. # This prevents GraphQL-Ruby warnings about empty Query types in tests. @@ -185,7 +186,10 @@ def run_rake_with_warehouse(*args) index_document_sizes: false, path_to_schema: "schema.rb", schema_artifacts_directory: "config/schema/artifacts", - extension_modules: [Warehouse::SchemaDefinition::APIExtension], + extension_modules: [ + JSONIngestion::SchemaDefinition::APIExtension, + Warehouse::SchemaDefinition::APIExtension + ], output: output ) end diff --git a/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt b/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt index 7507ebfbe..b4e9dcc67 100644 --- a/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt +++ b/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt @@ -1,6 +1,7 @@ project_root = File.expand_path(__dir__) require "elastic_graph/local/rake_tasks" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/query_registry/rake_tasks" require "rspec/core/rake_task" require "standard/rake" @@ -14,6 +15,7 @@ ElasticGraph::Local::RakeTasks.new( ) do |tasks| # Determines casing of field names. Can be either `:camelCase` or `:snake_case`. tasks.schema_element_name_form = :camelCase + tasks.schema_definition_extension_modules = [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension] # Customizes the names of fields generated by ElasticGraph. tasks.schema_element_name_overrides = { diff --git a/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb b/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb index 0d87095f3..a9769863a 100644 --- a/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb +++ b/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb @@ -103,6 +103,9 @@ def wait_for_other_processes # status if we're not running it's test suite. add_filter "/elasticgraph-local/" unless spec_files_to_run.any? { |f| f.include?("/elasticgraph-local/") } + # The JSON ingestion gem is being introduced by extracting implementation first and moving its tests later. + add_filter "/elasticgraph-json_ingestion/" + # This version file is loaded from our gemspecs, which can get loaded by bundler before we get here. # SimpleCov is only able to track coverage of files loaded after it starts, so we need to filter them out if # their constant is already defined. They don't contain any branching statements or anything so it's ok to diff --git a/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb b/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb index 0c3cf8538..8836260c0 100644 --- a/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb +++ b/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/schema_definition/test_support" +require "elastic_graph/json_ingestion/schema_definition/api_extension" # Combines `:capture_logs` with `ElasicGraph::SchemaDefinition::TestSupport` in order # to silence log output and fail if any tests result in logged warnings. @@ -17,7 +18,7 @@ def define_schema_with_schema_elements( schema_elements, index_document_sizes: true, json_schema_version: 1, - extension_modules: [], + extension_modules: [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension], derived_type_name_formats: {}, type_name_overrides: {}, enum_value_overrides_by_type: {},