[API] Skip all tests in a file if there's a skip setup in top level

Author: picandocodigo

api-spec-testing/test_file.rb

@@ -21,16 +21,17 @@
 require 'logger'
 
 module Elasticsearch
-
   module RestAPIYAMLTests
+    # Custom exception to raise when a test file needs to be skipped. This is
+    # captured as soon as possible so the test runners can move on to the next test.
+    class SkipTestsException < StandardError
+    end
 
     # Class representing a single test file, containing a setup, teardown, and multiple tests.
     #
     # @since 6.2.0
     class TestFile
-
-      attr_reader :features_to_skip
-      attr_reader :name
+      attr_reader :features_to_skip, :name, :client
 
       # Initialize a single test file.
       #
@@ -41,8 +42,9 @@ class TestFile
       # @param [ Array<Symbol> ] skip_features The names of features to skip.
       #
       # @since 6.1.0
-      def initialize(file_name, features_to_skip = [])
+      def initialize(file_name, client, features_to_skip = [])
         @name = file_name
+        @client = client
         begin
           documents = YAML.load_stream(File.new(file_name))
         rescue StandardError => e
@@ -52,10 +54,40 @@ def initialize(file_name, features_to_skip = [])
         end
         @test_definitions = documents.reject { |doc| doc['setup'] || doc['teardown'] }
         @setup = documents.find { |doc| doc['setup'] }
+        skip_entire_test_file? if @setup
         @teardown = documents.find { |doc| doc['teardown'] }
         @features_to_skip = REST_API_YAML_SKIP_FEATURES + features_to_skip
       end
 
+      def skip_entire_test_file?
+        @skip = @setup['setup']&.select { |a| a['skip'] }
+        return false if @skip.empty?
+
+        raise SkipTestsException if skip_version?(@client, @skip.first['skip'])
+      end
+
+      def skip_version?(client, skip_definition)
+        return true if skip_definition['version'] == 'all'
+
+        range_partition = /\s*-\s*/
+        return unless (versions = skip_definition['version']) && skip_definition['version'].partition(range_partition)
+
+        low, high = __parse_versions(versions)
+        range = low..high
+        begin
+          server_version = client.info['version']['number']
+        rescue
+          warn('Could not determine Elasticsearch version when checking if test should be skipped.')
+        end
+        range.cover?(server_version)
+      end
+
+      def __parse_versions(versions)
+        low = (['', nil].include? versions[0]) ? '0' : versions[0]
+        high = (['', nil].include? versions[2]) ? '9999' : versions[2]
+        [low, high]
+      end
+
       # Get a list of tests in the test file.
       #
       # @example Get the list of tests

api-spec-testing/test_file/action.rb

@@ -56,6 +56,9 @@ def execute(client, test = nil)
         @definition.each.inject(client) do |client, (method_chain, args)|
           chain = method_chain.split('.')
 
+          # If we have a method nested in a namespace, client becomes the
+          # client/namespace. Eg for `indices.resolve_index`, `client =
+          # client.indices` and then we call `resolve_index` on `client`.
           if chain.size > 1
             client = chain[0...-1].inject(client) do |_client, _method|
               _client.send(_method)

api-spec-testing/test_file/test.rb

@@ -64,10 +64,7 @@ def split_and_parse_key(key)
           end
         end
 
-        attr_reader :description
-        attr_reader :test_file
-        attr_reader :cached_values
-        attr_reader :file_basename
+        attr_reader :description, :test_file, :cached_values, :file_basename
 
         # Actions that if followed by a 'do' action, indicate that they complete their task group.
         # For example, consider this sequence of actions:
@@ -117,7 +114,6 @@ def initialize(test_file, test_definition)
           @definition = test_definition[description].select { |doc| !doc.key?('skip') }
           @definition.delete_if { |doc| doc['skip'] }
           @cached_values = {}
-
           skip_definitions = test_definition[description].select { |doc| doc['skip'] }.compact
           @skip = skip_definitions unless skip_definitions.empty?
         end
@@ -133,7 +129,6 @@ def initialize(test_file, test_definition)
         def task_groups
           @task_groups ||= begin
             @definition.each_with_index.inject([]) do |task_groups, (action, i)|
-
               # the action has a catch, it's a singular task group
               if action['do'] && action['do']['catch']
                 task_groups << TaskGroup.new(self)
@@ -160,7 +155,7 @@ def task_groups
         #
         # @since 6.2.0
         def cache_value(cache_key, value)
-          @cached_values["#{cache_key}"] = value
+          @cached_values[cache_key] = value
           @cached_values
         end
 
@@ -225,7 +220,7 @@ def skip_test?(client, features_to_skip = test_file.features_to_skip)
 
           if @skip
             @skip.collect { |s| s['skip'] }.any? do |skip|
-              contains_features_to_skip?(features_to_skip, skip) || skip_version?(client, skip)
+              contains_features_to_skip?(features_to_skip, skip) || test_file.skip_version?(client, skip)
             end
           end
         end
@@ -259,34 +254,13 @@ def pre_defined_skip?
           end
         end
 
-        def skip_version?(client, skip_definition)
-          return true if skip_definition['version'] == 'all'
-          range_partition = /\s*-\s*/
-          if versions = skip_definition['version'] && skip_definition['version'].partition(range_partition)
-            low, high = __parse_versions(versions)
-            range = low..high
-            begin
-              server_version = client.info['version']['number']
-            rescue
-              warn('Could not determine Elasticsearch version when checking if test should be skipped.')
-            end
-            range.cover?(server_version)
-          end
-        end
-
         def is_a_validation?(action)
           GROUP_TERMINATORS.any? { |validation| action[validation] } || expects_exception?(action)
         end
 
         def expects_exception?(action)
           action['do'] && action['do']['catch']
         end
-
-        def __parse_versions(versions)
-          low = (['', nil].include? versions[0]) ? '0' : versions[0]
-          high = (['', nil].include? versions[2]) ? '9999' : versions[2]
-          [low, high]
-        end
       end
     end
   end

elasticsearch-api/spec/elasticsearch/api/rest_api_yaml_spec.rb

@@ -8,7 +8,13 @@
 describe 'Rest API YAML tests' do
   # Traverse YAML files and create TestFile object:
   REST_API_YAML_FILES.each do |file|
-    test_file = Elasticsearch::RestAPIYAMLTests::TestFile.new(file, REST_API_YAML_SKIP_FEATURES)
+    begin
+      test_file = Elasticsearch::RestAPIYAMLTests::TestFile.new(file, REST_API_YAML_SKIP_FEATURES)
+    rescue SkipTestsException => _e
+      # If the test file has a `skip` at the top level that applies to this
+      # version of Elasticsearch, continue with the next text.
+      next
+    end
 
     context "#{file.gsub("#{YAML_FILES_DIRECTORY}/", '')}" do
       test_file.tests.each do |test|

elasticsearch-xpack/spec/xpack/rest_api_yaml_spec.rb

@@ -20,7 +20,13 @@
 
 describe 'XPack Rest API YAML tests' do
   REST_API_YAML_FILES.each do |file|
-    test_file = Elasticsearch::RestAPIYAMLTests::TestFile.new(file, REST_API_YAML_SKIP_FEATURES)
+    begin
+      test_file = Elasticsearch::RestAPIYAMLTests::TestFile.new(file, DEFAULT_CLIENT, REST_API_YAML_SKIP_FEATURES)
+    rescue SkipTestsException => _e
+      # If the test file has a `skip` at the top level that applies to this
+      # version of Elasticsearch, continue with the next text.
+      next
+    end
 
     context "#{file.gsub("#{YAML_FILES_DIRECTORY}/", '')}" do
       before(:all) do