From b3fbbe493153152c3806d162060141e370985568 Mon Sep 17 00:00:00 2001 From: Alexandre Terrasa Date: Wed, 20 Jul 2022 15:24:51 -0400 Subject: [PATCH 1/3] Extract DSL loader Co-authored-by: Ufuk Kayserilioglu Signed-off-by: Alexandre Terrasa --- lib/tapioca/commands/dsl.rb | 59 +++----------------------- lib/tapioca/internal.rb | 3 ++ lib/tapioca/loaders/dsl.rb | 78 +++++++++++++++++++++++++++++++++++ lib/tapioca/loaders/loader.rb | 77 ++++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 54 deletions(-) create mode 100644 lib/tapioca/loaders/dsl.rb create mode 100644 lib/tapioca/loaders/loader.rb diff --git a/lib/tapioca/commands/dsl.rb b/lib/tapioca/commands/dsl.rb index 0f9a71612..f4c1109ac 100644 --- a/lib/tapioca/commands/dsl.rb +++ b/lib/tapioca/commands/dsl.rb @@ -57,16 +57,15 @@ def initialize( @rbi_formatter = rbi_formatter super() - - @loader = T.let(nil, T.nilable(Runtime::Loader)) end sig { override.void } def execute - load_dsl_extensions - load_application(eager_load: @requested_constants.empty?) - abort_if_pending_migrations! - load_dsl_compilers + Loaders::Dsl.load_application( + tapioca_path: @tapioca_path, + compilers_path: @compiler_path, + eager_load: @requested_constants.empty? + ) if @should_verify say("Checking for out-of-date RBIs...") @@ -131,44 +130,6 @@ def execute private - sig { params(eager_load: T::Boolean).void } - def load_application(eager_load:) - say("Loading Rails application... ") - - loader.load_rails_application( - environment_load: true, - eager_load: eager_load - ) - - say("Done", :green) - end - - sig { void } - def abort_if_pending_migrations! - return unless File.exist?("config/application.rb") - return unless defined?(::Rake) - - Rails.application.load_tasks - if Rake::Task.task_defined?("db:abort_if_pending_migrations") - Rake::Task["db:abort_if_pending_migrations"].invoke - end - end - - sig { void } - def load_dsl_compilers - say("Loading DSL compiler classes... ") - - Dir.glob([ - "#{@compiler_path}/*.rb", - "#{@tapioca_path}/generators/**/*.rb", # TODO: Here for backcompat, remove later - "#{@tapioca_path}/compilers/**/*.rb", - ]).each do |compiler| - require File.expand_path(compiler) - end - - say("Done", :green) - end - sig { params(requested_constants: T::Array[String], path: Pathname).returns(T::Set[Pathname]) } def existing_rbi_filenames(requested_constants, path: @outpath) filenames = if requested_constants.empty? @@ -354,11 +315,6 @@ def rbi_files_in(path) end.sort end - sig { returns(Runtime::Loader) } - def loader - @loader ||= Runtime::Loader.new - end - sig { params(class_name: String).returns(String) } def underscore(class_name) return class_name unless /[A-Z-]|::/.match?(class_name) @@ -380,11 +336,6 @@ def rbi_filename_for(constant) def generate_command_for(constant) default_command(:dsl, constant) end - - sig { void } - def load_dsl_extensions - Dir["#{__dir__}/../dsl/extensions/*.rb"].sort.each { |f| require(f) } - end end end end diff --git a/lib/tapioca/internal.rb b/lib/tapioca/internal.rb index c9c772c19..cd626c8ef 100644 --- a/lib/tapioca/internal.rb +++ b/lib/tapioca/internal.rb @@ -50,6 +50,9 @@ require "tapioca/static/symbol_loader" require "tapioca/static/requires_compiler" +require "tapioca/loaders/loader" +require "tapioca/loaders/dsl" + require "tapioca/gem" require "tapioca/dsl" require "tapioca/commands" diff --git a/lib/tapioca/loaders/dsl.rb b/lib/tapioca/loaders/dsl.rb new file mode 100644 index 000000000..a3faee863 --- /dev/null +++ b/lib/tapioca/loaders/dsl.rb @@ -0,0 +1,78 @@ +# typed: strict +# frozen_string_literal: true + +module Tapioca + module Loaders + class Dsl < Loader + extend T::Sig + + sig { params(tapioca_path: String, compilers_path: String, eager_load: T::Boolean).void } + def self.load_application(tapioca_path:, compilers_path:, eager_load: true) + loader = new(tapioca_path: tapioca_path, compilers_path: compilers_path) + loader.load + end + + sig { override.void } + def load + load_dsl_extensions + load_application + abort_if_pending_migrations! + load_dsl_compilers + end + + protected + + sig { params(tapioca_path: String, compilers_path: String, eager_load: T::Boolean).void } + def initialize(tapioca_path:, compilers_path:, eager_load: true) + super() + + @tapioca_path = tapioca_path + @compilers_path = compilers_path + @eager_load = eager_load + end + + sig { void } + def load_dsl_extensions + Dir["#{__dir__}/../dsl/extensions/*.rb"].sort.each { |f| require(f) } + end + + sig { void } + def load_dsl_compilers + say("Loading DSL compiler classes... ") + + Dir.glob([ + "#{@compilers_path}/*.rb", + "#{@tapioca_path}/generators/**/*.rb", # TODO: Here for backcompat, remove later + "#{@tapioca_path}/compilers/**/*.rb", + ]).each do |compiler| + require File.expand_path(compiler) + end + + say("Done", :green) + end + + sig { void } + def load_application + say("Loading Rails application... ") + + load_rails_application( + environment_load: true, + eager_load: @eager_load + ) + + say("Done", :green) + end + + sig { void } + def abort_if_pending_migrations! + return unless File.exist?("config/application.rb") + return unless defined?(::Rake) + + Rails.application.load_tasks + if Rake::Task.task_defined?("db:abort_if_pending_migrations") + Rake::Task["db:abort_if_pending_migrations"].invoke + end + end + end + end +end diff --git a/lib/tapioca/loaders/loader.rb b/lib/tapioca/loaders/loader.rb new file mode 100644 index 000000000..6ead7ac1f --- /dev/null +++ b/lib/tapioca/loaders/loader.rb @@ -0,0 +1,77 @@ +# typed: true +# frozen_string_literal: true + +module Tapioca + module Loaders + class Loader + extend T::Sig + extend T::Helpers + + include Thor::Base + include CliHelper + + abstract! + + sig { abstract.void } + def load; end + + private + + sig { params(environment_load: T::Boolean, eager_load: T::Boolean).void } + def load_rails_application(environment_load: false, eager_load: false) + return unless File.exist?("config/application.rb") + + silence_deprecations + + if environment_load + safe_require("./config/environment") + else + safe_require("./config/application") + end + + eager_load_rails_app if eager_load + end + + sig { params(path: String).void } + def safe_require(path) + require path + rescue LoadError + nil + end + + sig { void } + def silence_deprecations + # Stop any ActiveSupport Deprecations from being reported + Object.const_get("ActiveSupport::Deprecation").silenced = true + rescue NameError + nil + end + + sig { void } + def eager_load_rails_app + rails = Object.const_get("Rails") + application = rails.application + + if Object.const_defined?("ActiveSupport") + Object.const_get("ActiveSupport").run_load_hooks( + :before_eager_load, + application + ) + end + + if Object.const_defined?("Zeitwerk::Loader") + zeitwerk_loader = Object.const_get("Zeitwerk::Loader") + zeitwerk_loader.eager_load_all + end + + if rails.respond_to?(:autoloaders) && rails.autoloaders.zeitwerk_enabled? + rails.autoloaders.each(&:eager_load) + end + + if application.config.respond_to?(:eager_load_namespaces) + application.config.eager_load_namespaces.each(&:eager_load!) + end + end + end + end +end From bce2cfb15f9f6b9c4d11d0fd105c0bd7e25f4384 Mon Sep 17 00:00:00 2001 From: Alexandre Terrasa Date: Wed, 20 Jul 2022 15:26:40 -0400 Subject: [PATCH 2/3] Rename `compiler_path` -> `compilers_path` Co-authored-by: Ufuk Kayserilioglu Signed-off-by: Alexandre Terrasa --- lib/tapioca/cli.rb | 2 +- lib/tapioca/commands/dsl.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/tapioca/cli.rb b/lib/tapioca/cli.rb index 07980c9b5..22fdc6794 100644 --- a/lib/tapioca/cli.rb +++ b/lib/tapioca/cli.rb @@ -124,7 +124,7 @@ def dsl(*constants) only: options[:only], exclude: options[:exclude], file_header: options[:file_header], - compiler_path: Tapioca::Dsl::Compilers::DIRECTORY, + compilers_path: Tapioca::Dsl::Compilers::DIRECTORY, tapioca_path: TAPIOCA_DIR, should_verify: options[:verify], quiet: options[:quiet], diff --git a/lib/tapioca/commands/dsl.rb b/lib/tapioca/commands/dsl.rb index f4c1109ac..6bb797d7d 100644 --- a/lib/tapioca/commands/dsl.rb +++ b/lib/tapioca/commands/dsl.rb @@ -14,7 +14,7 @@ class Dsl < Command only: T::Array[String], exclude: T::Array[String], file_header: T::Boolean, - compiler_path: String, + compilers_path: String, tapioca_path: String, should_verify: T::Boolean, quiet: T::Boolean, @@ -31,7 +31,7 @@ def initialize( only:, exclude:, file_header:, - compiler_path:, + compilers_path:, tapioca_path:, should_verify: false, quiet: false, @@ -46,7 +46,7 @@ def initialize( @only = only @exclude = exclude @file_header = file_header - @compiler_path = compiler_path + @compilers_path = compilers_path @tapioca_path = tapioca_path @should_verify = should_verify @quiet = quiet @@ -63,7 +63,7 @@ def initialize( def execute Loaders::Dsl.load_application( tapioca_path: @tapioca_path, - compilers_path: @compiler_path, + compilers_path: @compilers_path, eager_load: @requested_constants.empty? ) From 22a4324a65dc29c4c329eea77688b7e82d451de2 Mon Sep 17 00:00:00 2001 From: Alexandre Terrasa Date: Wed, 20 Jul 2022 15:52:26 -0400 Subject: [PATCH 3/3] Extract gem loader Signed-off-by: Alexandre Terrasa Co-authored-by: Ufuk Kayserilioglu --- lib/tapioca/commands/gem.rb | 70 +++++------------- lib/tapioca/internal.rb | 2 +- lib/tapioca/loaders/gem.rb | 78 ++++++++++++++++++++ lib/tapioca/loaders/loader.rb | 63 ++++++++++++++++ lib/tapioca/runtime/loader.rb | 131 ---------------------------------- 5 files changed, 161 insertions(+), 183 deletions(-) create mode 100644 lib/tapioca/loaders/gem.rb delete mode 100644 lib/tapioca/runtime/loader.rb diff --git a/lib/tapioca/commands/gem.rb b/lib/tapioca/commands/gem.rb index 7f1ec25a3..e94798170 100644 --- a/lib/tapioca/commands/gem.rb +++ b/lib/tapioca/commands/gem.rb @@ -55,8 +55,7 @@ def initialize( super() - @loader = T.let(nil, T.nilable(Runtime::Loader)) - @bundle = T.let(nil, T.nilable(Gemfile)) + @bundle = T.let(Gemfile.new(exclude), Gemfile) @existing_rbis = T.let(nil, T.nilable(T::Hash[String, String])) @expected_rbis = T.let(nil, T.nilable(T::Hash[String, String])) @include_doc = T.let(include_doc, T::Boolean) @@ -66,7 +65,12 @@ def initialize( sig { override.void } def execute - require_gem_file + Loaders::Gem.load_application( + bundle: @bundle, + prerequire: @prerequire, + postrequire: @postrequire, + default_command: default_command(:require), + ) gem_queue = gems_to_generate(@gem_names).reject { |gem| @exclude.include?(gem.name) } anything_done = [ @@ -87,7 +91,7 @@ def execute gem_dir: @outpath.to_s, dsl_dir: @dsl_dir, auto_strictness: @auto_strictness, - gems: bundle.dependencies + gems: @bundle.dependencies ) say("All operations performed in working directory.", [:green, :bold]) @@ -117,7 +121,7 @@ def sync(should_verify: false, exclude: []) gem_dir: @outpath.to_s, dsl_dir: @dsl_dir, auto_strictness: @auto_strictness, - gems: bundle.dependencies + gems: @bundle.dependencies ) say("All operations performed in working directory.", [:green, :bold]) @@ -131,42 +135,12 @@ def sync(should_verify: false, exclude: []) private - sig { returns(Runtime::Loader) } - def loader - @loader ||= Runtime::Loader.new - end - - sig { returns(Gemfile) } - def bundle - @bundle ||= Gemfile.new(@exclude) - end - - sig { void } - def require_gem_file - say("Requiring all gems to prepare for compiling... ") - begin - loader.load_bundle(bundle, @prerequire, @postrequire) - rescue LoadError => e - explain_failed_require(@postrequire, e) - exit(1) - end - - Runtime::Trackers::Autoload.eager_load_all! - - say(" Done", :green) - unless bundle.missing_specs.empty? - say(" completed with missing specs: ") - say(bundle.missing_specs.join(", "), :yellow) - end - puts - end - sig { params(gem_names: T::Array[String]).returns(T::Array[Gemfile::GemSpec]) } def gems_to_generate(gem_names) - return bundle.dependencies if gem_names.empty? + return @bundle.dependencies if gem_names.empty? gem_names.map do |gem_name| - gem = bundle.gem(gem_name) + gem = @bundle.gem(gem_name) if gem.nil? say("Error: Cannot find gem '#{gem_name}'", :red) exit(1) @@ -263,7 +237,12 @@ def perform_additions if gems.empty? say("Nothing to do.") else - require_gem_file + Loaders::Gem.load_application( + bundle: @bundle, + prerequire: @prerequire, + postrequire: @postrequire, + default_command: default_command(:require), + ) Executor.new(gems, number_of_workers: @number_of_workers).run_in_parallel do |gem_name| filename = expected_rbi(gem_name) @@ -273,7 +252,7 @@ def perform_additions move(old_filename, filename) unless old_filename == filename end - gem = T.must(bundle.gem(gem_name)) + gem = T.must(@bundle.gem(gem_name)) compile_gem_rbi(gem) puts end @@ -287,17 +266,6 @@ def perform_additions anything_done end - sig { params(file: String, error: LoadError).void } - def explain_failed_require(file, error) - say_error("\n\nLoadError: #{error}", :bold, :red) - say_error("\nTapioca could not load all the gems required by your application.", :yellow) - say_error("If you populated ", :yellow) - say_error("#{file} ", :bold, :blue) - say_error("with ", :yellow) - say_error("`#{default_command(:require)}`", :bold, :blue) - say_error("you should probably review it and remove the faulty line.", :yellow) - end - sig { returns(T::Array[String]) } def removed_rbis (existing_rbis.keys - expected_rbis.keys).sort @@ -359,7 +327,7 @@ def existing_rbis sig { returns(T::Hash[String, String]) } def expected_rbis - @expected_rbis ||= bundle.dependencies + @expected_rbis ||= @bundle.dependencies .reject { |gem| @exclude.include?(gem.name) } .to_h { |gem| [gem.name, gem.version.to_s] } end diff --git a/lib/tapioca/internal.rb b/lib/tapioca/internal.rb index cd626c8ef..ae92d09c6 100644 --- a/lib/tapioca/internal.rb +++ b/lib/tapioca/internal.rb @@ -27,7 +27,6 @@ require "tapioca/runtime/dynamic_mixin_compiler" require "tapioca/helpers/gem_helper" -require "tapioca/runtime/loader" require "tapioca/helpers/sorbet_helper" require "tapioca/helpers/rbi_helper" @@ -51,6 +50,7 @@ require "tapioca/static/requires_compiler" require "tapioca/loaders/loader" +require "tapioca/loaders/gem" require "tapioca/loaders/dsl" require "tapioca/gem" diff --git a/lib/tapioca/loaders/gem.rb b/lib/tapioca/loaders/gem.rb new file mode 100644 index 000000000..28da36c7e --- /dev/null +++ b/lib/tapioca/loaders/gem.rb @@ -0,0 +1,78 @@ +# typed: strict +# frozen_string_literal: true + +module Tapioca + module Loaders + class Gem < Loader + extend T::Sig + + sig do + params( + bundle: Gemfile, + prerequire: T.nilable(String), + postrequire: String, + default_command: String + ).void + end + def self.load_application(bundle:, prerequire:, postrequire:, default_command:) + loader = new(bundle: bundle, prerequire: prerequire, postrequire: postrequire, default_command: default_command) + loader.load + end + + sig { override.void } + def load + require_gem_file + end + + protected + + sig do + params( + bundle: Gemfile, + prerequire: T.nilable(String), + postrequire: String, + default_command: String + ).void + end + def initialize(bundle:, prerequire:, postrequire:, default_command:) + super() + + @bundle = bundle + @prerequire = prerequire + @postrequire = postrequire + @default_command = default_command + end + + sig { void } + def require_gem_file + say("Requiring all gems to prepare for compiling... ") + begin + load_bundle(@bundle, @prerequire, @postrequire) + rescue LoadError => e + explain_failed_require(@postrequire, e) + exit(1) + end + + Runtime::Trackers::Autoload.eager_load_all! + + say(" Done", :green) + unless @bundle.missing_specs.empty? + say(" completed with missing specs: ") + say(@bundle.missing_specs.join(", "), :yellow) + end + puts + end + + sig { params(file: String, error: LoadError).void } + def explain_failed_require(file, error) + say_error("\n\nLoadError: #{error}", :bold, :red) + say_error("\nTapioca could not load all the gems required by your application.", :yellow) + say_error("If you populated ", :yellow) + say_error("#{file} ", :bold, :blue) + say_error("with ", :yellow) + say_error("`#{@default_command}`", :bold, :blue) + say_error("you should probably review it and remove the faulty line.", :yellow) + end + end + end +end diff --git a/lib/tapioca/loaders/loader.rb b/lib/tapioca/loaders/loader.rb index 6ead7ac1f..746a9d8f4 100644 --- a/lib/tapioca/loaders/loader.rb +++ b/lib/tapioca/loaders/loader.rb @@ -9,6 +9,7 @@ class Loader include Thor::Base include CliHelper + include Tapioca::GemHelper abstract! @@ -17,6 +18,21 @@ def load; end private + sig do + params(gemfile: Tapioca::Gemfile, initialize_file: T.nilable(String), require_file: T.nilable(String)).void + end + def load_bundle(gemfile, initialize_file, require_file) + require_helper(initialize_file) + + load_rails_application + + gemfile.require_bundle + + require_helper(require_file) + + load_rails_engines + end + sig { params(environment_load: T::Boolean, eager_load: T::Boolean).void } def load_rails_application(environment_load: false, eager_load: false) return unless File.exist?("config/application.rb") @@ -32,6 +48,43 @@ def load_rails_application(environment_load: false, eager_load: false) eager_load_rails_app if eager_load end + sig { void } + def load_rails_engines + rails_engines.each do |engine| + errored_files = [] + + engine.config.eager_load_paths.each do |load_path| + Dir.glob("#{load_path}/**/*.rb").sort.each do |file| + require(file) + rescue LoadError, StandardError + errored_files << file + end + end + + # Try files that have errored one more time + # It might have been a load order problem + errored_files.each do |file| + require(file) + rescue LoadError, StandardError + nil + end + end + end + + sig { returns(T::Array[T.untyped]) } + def rails_engines + return [] unless Object.const_defined?("Rails::Engine") + + safe_require("active_support/core_ext/class/subclasses") + + project_path = Bundler.default_gemfile.parent.expand_path + # We can use `Class#descendants` here, since we know Rails is loaded + Object.const_get("Rails::Engine") + .descendants + .reject(&:abstract_railtie?) + .reject { |engine| gem_in_app_dir?(project_path, engine.config.root.to_path) } + end + sig { params(path: String).void } def safe_require(path) require path @@ -72,6 +125,16 @@ def eager_load_rails_app application.config.eager_load_namespaces.each(&:eager_load!) end end + + sig { params(file: T.nilable(String)).void } + def require_helper(file) + return unless file + + file = File.absolute_path(file) + return unless File.exist?(file) + + require(file) + end end end end diff --git a/lib/tapioca/runtime/loader.rb b/lib/tapioca/runtime/loader.rb deleted file mode 100644 index 62f7efe8e..000000000 --- a/lib/tapioca/runtime/loader.rb +++ /dev/null @@ -1,131 +0,0 @@ -# typed: strict -# frozen_string_literal: true - -module Tapioca - module Runtime - class Loader - extend(T::Sig) - include Tapioca::GemHelper - - sig do - params(gemfile: Tapioca::Gemfile, initialize_file: T.nilable(String), require_file: T.nilable(String)).void - end - def load_bundle(gemfile, initialize_file, require_file) - require_helper(initialize_file) - - load_rails_application - - gemfile.require_bundle - - require_helper(require_file) - - load_rails_engines - end - - sig { params(environment_load: T::Boolean, eager_load: T::Boolean).void } - def load_rails_application(environment_load: false, eager_load: false) - return unless File.exist?("config/application.rb") - - silence_deprecations - - if environment_load - safe_require("./config/environment") - else - safe_require("./config/application") - end - - eager_load_rails_app if eager_load - end - - private - - sig { params(file: T.nilable(String)).void } - def require_helper(file) - return unless file - - file = File.absolute_path(file) - return unless File.exist?(file) - - require(file) - end - - sig { returns(T::Array[T.untyped]) } - def rails_engines - return [] unless Object.const_defined?("Rails::Engine") - - safe_require("active_support/core_ext/class/subclasses") - - project_path = Bundler.default_gemfile.parent.expand_path - # We can use `Class#descendants` here, since we know Rails is loaded - Object.const_get("Rails::Engine") - .descendants - .reject(&:abstract_railtie?) - .reject { |engine| gem_in_app_dir?(project_path, engine.config.root.to_path) } - end - - sig { params(path: String).void } - def safe_require(path) - require path - rescue LoadError - nil - end - - sig { void } - def silence_deprecations - # Stop any ActiveSupport Deprecations from being reported - Object.const_get("ActiveSupport::Deprecation").silenced = true - rescue NameError - nil - end - - sig { void } - def eager_load_rails_app - rails = Object.const_get("Rails") - application = rails.application - - if Object.const_defined?("ActiveSupport") - Object.const_get("ActiveSupport").run_load_hooks( - :before_eager_load, - application - ) - end - - if Object.const_defined?("Zeitwerk::Loader") - zeitwerk_loader = Object.const_get("Zeitwerk::Loader") - zeitwerk_loader.eager_load_all - end - - if rails.respond_to?(:autoloaders) && rails.autoloaders.zeitwerk_enabled? - rails.autoloaders.each(&:eager_load) - end - - if application.config.respond_to?(:eager_load_namespaces) - application.config.eager_load_namespaces.each(&:eager_load!) - end - end - - sig { void } - def load_rails_engines - rails_engines.each do |engine| - errored_files = [] - - engine.config.eager_load_paths.each do |load_path| - Dir.glob("#{load_path}/**/*.rb").sort.each do |file| - require(file) - rescue LoadError, StandardError - errored_files << file - end - end - - # Try files that have errored one more time - # It might have been a load order problem - errored_files.each do |file| - require(file) - rescue LoadError, StandardError - nil - end - end - end - end - end -end