(GH-24) Allow parsing of manifests in tasks mode

The Puppet language introduced Puppet Tasks and Plans as part of
Puppet 5.4.0 [1].  A special parsing switch `--tasks` was
introduced which changes the behaviour of the Puppet parser, that
is switching it into "tasks" mode.  Unfortunately this switch is
global to all Puppet parsing therefore to dynamically switch it
like we do in Editor Services, this required a global mutex to be
used.

Note that this commit does not add Puppet Tasks/Plans intellisense
but merely allows the standard manifest providers (completion,
validation etc.) to actually function instead of silently failing

This commit;
* Modifies the Puppet parser, via monkey patching, to implement a
  "singleton" parsing method that dynamically, and safely, switch
  the parsing mode on a per request basis
* Modifies the Document Store to detect whether a File URI is a
  Puppet Task/Plan based on the relative path of the file versus
  the module root.
* Modifies the various providers to call the parsing helper in the
  correct mode
* Modifies the Message Router to detect whether a file is a Puppet
  Task/Plan and then sets the appropriate mode when calling the
  manifest providers (completion, validation etc.)

[1] https://puppet.com/docs/bolt/1.x/writing_plans.html#concept-4485

Author: glennsarti

lib/puppet-languageserver/document_store.rb

@@ -52,6 +52,15 @@ def self.document_type(uri)
       end
     end
 
+    # Plan files https://puppet.com/docs/bolt/1.x/writing_plans.html#concept-4485
+    # exist in modules (requires metadata.json) and are in the `/plans` directory
+    def self.module_plan_file?(uri)
+      return false unless store_has_module_metadata?
+      relative_path = PuppetLanguageServer::UriHelper.relative_uri_path(PuppetLanguageServer::UriHelper.build_file_uri(store_root_path), uri, !windows?)
+      return false if relative_path.nil?
+      relative_path.start_with?('/plans/')
+    end
+
     # Workspace management
     WORKSPACE_CACHE_TTL_SECONDS = 60
     def self.initialize_store(options = {})
@@ -150,5 +159,12 @@ def self.dir_exist?(path)
       Dir.exist?(path)
     end
     private_class_method :dir_exist?
+
+    def self.windows?
+      # Ruby only sets File::ALT_SEPARATOR on Windows and the Ruby standard
+      # library uses that to test what platform it's on.
+      !!File::ALT_SEPARATOR # rubocop:disable Style/DoubleNegation
+    end
+    private_class_method :windows?
   end
 end

lib/puppet-languageserver/manifest/completion_provider.rb

@@ -1,17 +1,23 @@
 module PuppetLanguageServer
   module Manifest
     module CompletionProvider
-      def self.complete(content, line_num, char_num)
+      def self.complete(content, line_num, char_num, options = {})
+        options = {
+          :tasks_mode => false
+        }.merge(options)
         items = []
         incomplete = false
 
-        result = PuppetLanguageServer::PuppetParserHelper.object_under_cursor(content, line_num, char_num, true, [Puppet::Pops::Model::QualifiedName, Puppet::Pops::Model::BlockExpression])
-
+        result = PuppetLanguageServer::PuppetParserHelper.object_under_cursor(content, line_num, char_num,
+                                                                              :multiple_attempts  => true,
+                                                                              :disallowed_classes => [Puppet::Pops::Model::QualifiedName, Puppet::Pops::Model::BlockExpression],
+                                                                              :tasks_mode         => options[:tasks_mode])
         if result.nil?
           # We are in the root of the document.
 
           # Add keywords
           keywords(%w[class define node application site]) { |x| items << x }
+          keywords(%w[plan]) { |x| items << x } if options[:tasks_mode]
 
           # Add resources
           all_resources { |x| items << x }

lib/puppet-languageserver/manifest/definition_provider.rb

@@ -1,9 +1,13 @@
 module PuppetLanguageServer
   module Manifest
     module DefinitionProvider
-      def self.find_definition(content, line_num, char_num)
-        result = PuppetLanguageServer::PuppetParserHelper.object_under_cursor(content, line_num, char_num, false, [Puppet::Pops::Model::BlockExpression])
-
+      def self.find_definition(content, line_num, char_num, options = {})
+        options = {
+          :tasks_mode => false
+        }.merge(options)
+        result = PuppetLanguageServer::PuppetParserHelper.object_under_cursor(content, line_num, char_num,
+                                                                              :disallowed_classes => [Puppet::Pops::Model::BlockExpression],
+                                                                              :tasks_mode         => options[:tasks_mode])
         return nil if result.nil?
 
         path = result[:path]

lib/puppet-languageserver/manifest/document_symbol_provider.rb

@@ -17,7 +17,7 @@ def self.workspace_symbols(query)
                 'fromline' => item.line,
                 'fromchar' => 0, # Don't have char pos for types
                 'toline'   => item.line,
-                'tochar'   => 1024, # Don't have char pos for types
+                'tochar'   => 1024 # Don't have char pos for types
               )
             )
 
@@ -30,7 +30,7 @@ def self.workspace_symbols(query)
                 'fromline' => item.line,
                 'fromchar' => 0, # Don't have char pos for functions
                 'toline'   => item.line,
-                'tochar'   => 1024, # Don't have char pos for functions
+                'tochar'   => 1024 # Don't have char pos for functions
               )
             )
 
@@ -43,17 +43,20 @@ def self.workspace_symbols(query)
                 'fromline' => item.line,
                 'fromchar' => 0, # Don't have char pos for classes
                 'toline'   => item.line,
-                'tochar'   => 1024, # Don't have char pos for classes
+                'tochar'   => 1024 # Don't have char pos for classes
               )
             )
           end
         end
         result
       end
 
-      def self.extract_document_symbols(content)
+      def self.extract_document_symbols(content, options = {})
+        options = {
+          :tasks_mode => false
+        }.merge(options)
         parser = Puppet::Pops::Parser::Parser.new
-        result = parser.parse_string(content, '')
+        result = parser.singleton_parse_string(content, options[:tasks_mode], '')
 
         if result.model.respond_to? :eAllContents
           # We are unable to build a document symbol tree for Puppet 4 AST
@@ -187,6 +190,28 @@ def self.recurse_document_symbols(object, path, parentsymbol, symbollist)
             'children'       => []
           )
 
+        # Puppet Plan
+        when 'Puppet::Pops::Model::PlanDefinition'
+          this_symbol = LanguageServer::DocumentSymbol.create(
+            'name'           => object.name,
+            'kind'           => LanguageServer::SYMBOLKIND_CLASS,
+            'detail'         => object.name,
+            'range'          => create_range_array(object.offset, object.length, object.locator),
+            'selectionRange' => create_range_array(object.offset, object.length, object.locator),
+            'children'       => []
+          )
+          # Load in the class parameters
+          object.parameters.each do |param|
+            param_symbol = LanguageServer::DocumentSymbol.create(
+              'name'           => '$' + param.name,
+              'kind'           => LanguageServer::SYMBOLKIND_PROPERTY,
+              'detail'         => '$' + param.name,
+              'range'          => create_range_array(param.offset, param.length, param.locator),
+              'selectionRange' => create_range_array(param.offset, param.length, param.locator),
+              'children'       => []
+            )
+            this_symbol['children'].push(param_symbol)
+          end
         end
 
         object._pcore_contents do |item|

lib/puppet-languageserver/manifest/hover_provider.rb

@@ -1,8 +1,13 @@
 module PuppetLanguageServer
   module Manifest
     module HoverProvider
-      def self.resolve(content, line_num, char_num)
-        result = PuppetLanguageServer::PuppetParserHelper.object_under_cursor(content, line_num, char_num, false, [Puppet::Pops::Model::QualifiedName, Puppet::Pops::Model::BlockExpression])
+      def self.resolve(content, line_num, char_num, options = {})
+        options = {
+          :tasks_mode => false
+        }.merge(options)
+        result = PuppetLanguageServer::PuppetParserHelper.object_under_cursor(content, line_num, char_num,
+                                                                              :disallowed_classes => [Puppet::Pops::Model::QualifiedName, Puppet::Pops::Model::BlockExpression],
+                                                                              :tasks_mode         => options[:tasks_mode])
         return LanguageServer::Hover.create_nil_response if result.nil?
 
         path = result[:path]

lib/puppet-languageserver/manifest/validation_provider.rb

@@ -28,7 +28,12 @@ def self.fix_validate_errors(content)
         [problems_fixed, linter.manifest]
       end
 
-      def self.validate(content, _max_problems = 100)
+      def self.validate(content, options = {})
+        options = {
+          :max_problems => 100,
+          :tasks_mode   => false
+        }.merge(options)
+
         result = []
         # TODO: Need to implement max_problems
         problems = 0
@@ -89,8 +94,16 @@ def self.validate(content, _max_problems = 100)
         Puppet.override({ loaders: loaders }, 'For puppet parser validate') do
           begin
             validation_environment = env
-            validation_environment.check_for_reparse
-            validation_environment.known_resource_types.clear
+            $PuppetParserMutex.synchronize do # rubocop:disable Style/GlobalVars
+              begin
+                original_taskmode = Puppet[:tasks] if Puppet.tasks_supported?
+                Puppet[:tasks] = options[:tasks_mode] if Puppet.tasks_supported?
+                validation_environment.check_for_reparse
+                validation_environment.known_resource_types.clear
+              ensure
+                Puppet[:tasks] = original_taskmode if Puppet.tasks_supported?
+              end
+            end
           rescue StandardError => detail
             # Sometimes the error is in the cause not the root object itself
             detail = detail.cause if !detail.respond_to?(:line) && detail.respond_to?(:cause)

lib/puppet-languageserver/message_router.rb

@@ -97,7 +97,7 @@ def receive_request(request)
         begin
           case documents.document_type(file_uri)
           when :manifest
-            request.reply_result(PuppetLanguageServer::Manifest::CompletionProvider.complete(content, line_num, char_num))
+            request.reply_result(PuppetLanguageServer::Manifest::CompletionProvider.complete(content, line_num, char_num, :tasks_mode => PuppetLanguageServer::DocumentStore.module_plan_file?(file_uri)))
           else
             raise "Unable to provide completion on #{file_uri}"
           end
@@ -123,7 +123,7 @@ def receive_request(request)
         begin
           case documents.document_type(file_uri)
           when :manifest
-            request.reply_result(PuppetLanguageServer::Manifest::HoverProvider.resolve(content, line_num, char_num))
+            request.reply_result(PuppetLanguageServer::Manifest::HoverProvider.resolve(content, line_num, char_num, :tasks_mode => PuppetLanguageServer::DocumentStore.module_plan_file?(file_uri)))
           else
             raise "Unable to provide hover on #{file_uri}"
           end
@@ -140,7 +140,7 @@ def receive_request(request)
         begin
           case documents.document_type(file_uri)
           when :manifest
-            request.reply_result(PuppetLanguageServer::Manifest::DefinitionProvider.find_definition(content, line_num, char_num))
+            request.reply_result(PuppetLanguageServer::Manifest::DefinitionProvider.find_definition(content, line_num, char_num, :tasks_mode => PuppetLanguageServer::DocumentStore.module_plan_file?(file_uri)))
           else
             raise "Unable to provide definition on #{file_uri}"
           end
@@ -155,8 +155,7 @@ def receive_request(request)
         begin
           case documents.document_type(file_uri)
           when :manifest
-            result = PuppetLanguageServer::Manifest::DocumentSymbolProvider.extract_document_symbols(content)
-            request.reply_result(result)
+            request.reply_result(PuppetLanguageServer::Manifest::DocumentSymbolProvider.extract_document_symbols(content, :tasks_mode => PuppetLanguageServer::DocumentStore.module_plan_file?(file_uri)))
           else
             raise "Unable to provide definition on #{file_uri}"
           end

lib/puppet-languageserver/puppet_monkey_patches.rb

@@ -1,3 +1,35 @@
+# Monkey Patch the Puppet language parser so we can globally lock any changes to the
+# global setting Puppet[:tasks].  We need to manage this so we can switch between
+# parsing modes.  Unfortunately we can't do this as method parameter, only via the
+# global Puppet settings which is not thread safe
+$PuppetParserMutex = Mutex.new # rubocop:disable Style/GlobalVars
+module Puppet
+  module Pops
+    module Parser
+      class Parser
+        def singleton_parse_string(code, task_mode = false, path = nil)
+          $PuppetParserMutex.synchronize do # rubocop:disable Style/GlobalVars
+            begin
+              original_taskmode = Puppet[:tasks] if Puppet.tasks_supported?
+              Puppet[:tasks] = task_mode if Puppet.tasks_supported?
+              return parse_string(code, path)
+            ensure
+              Puppet[:tasks] = original_taskmode if Puppet.tasks_supported?
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+module Puppet
+  # Tasks first appeared in Puppet 5.4.0
+  def self.tasks_supported?
+    Gem::Version.new(Puppet.version) >= Gem::Version.new('5.4.0')
+  end
+end
+
 # MUST BE LAST!!!!!!
 # Suppress any warning messages to STDOUT.  It can pollute stdout when running in STDIO mode
 Puppet::Util::Log.newdesttype :null_logger do

lib/puppet-languageserver/puppet_parser_helper.rb

@@ -56,13 +56,19 @@ def self.get_line_at(content, line_offsets, line_num)
       end
     end
 
-    def self.object_under_cursor(content, line_num, char_num, multiple_attempts = false, disallowed_classes = [])
+    def self.object_under_cursor(content, line_num, char_num, options)
+      options = {
+        :multiple_attempts  => false,
+        :disallowed_classes => [],
+        :tasks_mode         => false
+      }.merge(options)
+
       # Use Puppet to generate the AST
       parser = Puppet::Pops::Parser::Parser.new
 
       # Calculating the line offsets can be expensive and is only required
       # if we're doing mulitple passes of parsing
-      line_offsets = line_offsets(content) if multiple_attempts
+      line_offsets = line_offsets(content) if options[:multiple_attempts]
 
       result = nil
       move_offset = 0
@@ -108,10 +114,10 @@ def self.object_under_cursor(content, line_num, char_num, multiple_attempts = fa
         next if new_content.nil?
 
         begin
-          result = parser.parse_string(new_content, '')
+          result = parser.singleton_parse_string(new_content, options[:tasks_mode], '')
           break
         rescue Puppet::ParseErrorWithIssue => _exception
-          next if multiple_attempts
+          next if options[:multiple_attempts]
           raise
         end
       end
@@ -140,14 +146,14 @@ def self.object_under_cursor(content, line_num, char_num, multiple_attempts = fa
       valid_models = []
       if result.model.respond_to? :eAllContents
         valid_models = result.model.eAllContents.select do |item|
-          check_for_valid_item(item, abs_offset, disallowed_classes)
+          check_for_valid_item(item, abs_offset, options[:disallowed_classes])
         end
 
         valid_models.sort! { |a, b| a.length - b.length }
       else
         path = []
         result.model._pcore_all_contents(path) do |item|
-          if check_for_valid_item(item, abs_offset, disallowed_classes) # rubocop:disable Style/IfUnlessModifier  Nicer to read like this
+          if check_for_valid_item(item, abs_offset, options[:disallowed_classes]) # rubocop:disable Style/IfUnlessModifier  Nicer to read like this
             valid_models.push(model_path_struct.new(item, path.dup))
           end
         end

lib/puppet-languageserver/uri_helper.rb

@@ -1,7 +1,35 @@
+require 'uri'
+require 'puppet'
+
 module PuppetLanguageServer
   module UriHelper
     def self.build_file_uri(path)
-      path.start_with?('/') ? 'file://' + path : 'file:///' + path
+      'file://' + Puppet::Util.uri_encode(path.start_with?('/') ? path : '/' + path)
+    end
+
+    # Compares two URIs and returns the relative path
+    #
+    # @param root_uri [String] The root URI to compare to
+    # @param uri [String] The URI to compare to the root
+    # @param case_sensitive [Boolean] Whether the path comparison is case senstive or not.  Default is true
+    # @return [String] Returns the relative path string if the URI is indeed a child of the root, otherwise returns nil
+    def self.relative_uri_path(root_uri, uri, case_sensitive = true)
+      actual_root = URI(root_uri)
+      actual_uri = URI(uri)
+      return nil unless actual_root.scheme == actual_uri.scheme
+
+      # CGI.unescape doesn't handle space rules properly in uri paths
+      # URI.unescape does, but returns strings in their original encoding
+      # Mostly safe here as we're only worried about file based URIs
+      root_path = URI.unescape(actual_root.path) # rubocop:disable Lint/UriEscapeUnescape
+      uri_path = URI.unescape(actual_uri.path) # rubocop:disable Lint/UriEscapeUnescape
+      if case_sensitive
+        return nil unless uri_path.slice(0, root_path.length) == root_path
+      else
+        return nil unless uri_path.slice(0, root_path.length).casecmp(root_path).zero?
+      end
+
+      uri_path.slice(root_path.length..-1)
     end
   end
 end