Merge pull request #7 from lingua-pupuli/add-magic-comments

(GH-6) Add magic comments for Puppetfile parser

Author: glennsarti

README.md

@@ -54,6 +54,42 @@ The parser converts the content of a Puppetfile into a document model (`Puppetfi
 
 Currently only one Parser is available, `R10KEval`, which uses the same parsing logic as the [R10K Parser](https://github.com/puppetlabs/r10k/blob/master/lib/r10k/puppetfile.rb). In the future other parsers may be added, such as a [Ruby AST based parser](https://github.com/puppetlabs/r10k/pull/885).
 
+#### R10KEval Parser
+
+Sometimes it necessary to instruct the Resolver to ignore certain rules, for example ignoring the Puppet restrictions on a module because it has old metadata. The R10KEval Parser can read ruby comments which have special meanings, similar to how [rubocop](https://docs.rubocop.org/en/stable/) uses [magic comments](https://docs.rubocop.org/en/stable/configuration/#disabling-cops-within-source-code).
+
+The R10KEval Parser uses the magic comment of `# resolver:disable` to disable tasks.  For example;
+
+You can set a comment for one module like this:
+
+``` ruby
+mod 'puppetlabs-stdlib', :latest # resolver:disable Dependency/All
+```
+
+or for many modules by wrapping the modules like this:
+
+``` ruby
+# resolver:disable Dependency/All
+mod 'puppetlabs-stdlib', :latest
+mod 'puppetlabs-apache', '1.0.0'
+# resolver:ensable Dependency/All
+```
+
+You can also set multiple items by using a comma like this:
+
+``` ruby
+mod 'puppetlabs-stdlib', :latest # resolver:disable Dependency/All,Dependency/Puppet
+```
+
+Note there is no space between items.
+
+The available settings are:
+
+| Setting Name        | Description |
+| --------------------| ----------- |
+| `Dependency/Puppet` | Instructs the resolver to ignore any Puppet version in its dependency traversal for the specified modules. Useful for modules with outdated metadata.json information |
+| `Dependency/All`    | Instructs the resolver to ignore any, and all, dependencies in its dependency traversal of the specified module. Useful for modules with outdated metadata.json information. |
+
 ### Puppetfile Document Validation
 
 Even though a Puppetfile can be parsed, doesn't mean it's valid. For example, defining a module twice.

lib/puppetfile-resolver/models/module_specification.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'puppetfile-resolver/models/module_dependency'
+require 'puppetfile-resolver/models/puppet_dependency'
+require 'puppetfile-resolver/puppetfile'
 
 module PuppetfileResolver
   module Models
@@ -9,6 +11,7 @@ class ModuleSpecification
       attr_accessor :owner
       attr_accessor :version
       attr_accessor :origin # Same as R10K module :type
+      attr_accessor :resolver_flags
 
       def initialize(options = {})
         require 'semantic_puppet'
@@ -36,6 +39,7 @@ def initialize(options = {})
         @origin = options[:origin]
         @dependencies = nil
         @metadata = options[:metadata]
+        @resolver_flags = options[:resolver_flags].nil? ? [] : options[:resolver_flags]
         @version = ::SemanticPuppet::Version.parse(options[:version]) unless options[:version].nil?
       end
 
@@ -78,8 +82,9 @@ def metadata(_cache, _resolver_ui)
       def dependencies(cache, resolver_ui)
         return @dependencies unless @dependencies.nil?
 
-        meta = metadata(cache, resolver_ui)
+        return (@dependencies = []) if resolver_flags.include?(PuppetfileResolver::Puppetfile::DISABLE_ALL_DEPENDENCIES_FLAG)
 
+        meta = metadata(cache, resolver_ui)
         @dependencies = []
         unless meta[:dependencies].nil? || meta[:dependencies].empty?
           @dependencies = meta[:dependencies].map do |dep|
@@ -90,14 +95,16 @@ def dependencies(cache, resolver_ui)
           end
         end
 
-        puppet_requirement = nil
-        unless meta[:requirements].nil? || meta[:requirements].empty? # rubocop:disable Style/IfUnlessModifier
-          puppet_requirement = meta[:requirements].find { |req| req[:name] == 'puppet' && !req[:version_requirement].nil? }
-        end
-        if puppet_requirement.nil?
-          @dependencies << PuppetDependency.new('>= 0')
-        else
-          @dependencies << PuppetDependency.new(puppet_requirement[:version_requirement])
+        unless resolver_flags.include?(PuppetfileResolver::Puppetfile::DISABLE_PUPPET_DEPENDENCY_FLAG)
+          puppet_requirement = nil
+          unless meta[:requirements].nil? || meta[:requirements].empty? # rubocop:disable Style/IfUnlessModifier
+            puppet_requirement = meta[:requirements].find { |req| req[:name] == 'puppet' && !req[:version_requirement].nil? }
+          end
+          if puppet_requirement.nil?
+            @dependencies << PuppetDependency.new('>= 0')
+          else
+            @dependencies << PuppetDependency.new(puppet_requirement[:version_requirement])
+          end
         end
 
         @dependencies

lib/puppetfile-resolver/puppetfile.rb

@@ -1,5 +1,17 @@
 # frozen_string_literal: true
 
+module PuppetfileResolver
+  module Puppetfile
+    # Resolver Flags
+    #
+    # DISABLE_PUPPET_DEPENDENCY_FLAG - Instructs the resolver to not consider Puppet version in its dependency traversal. Useful for modules with outdated metadata.json information.
+    # DISABLE_ALL_DEPENDENCIES_FLAG - Instructs the resolver to ignore any dependencies in its dependency traversal. Useful for modules with outdated metadata.json information.
+    #
+    DISABLE_PUPPET_DEPENDENCY_FLAG = :disable_puppet_dependency
+    DISABLE_ALL_DEPENDENCIES_FLAG = :disable_all_dependencies
+  end
+end
+
 require 'puppetfile-resolver/puppetfile/document'
 require 'puppetfile-resolver/puppetfile/validation_errors'
 require 'puppetfile-resolver/puppetfile/base_module'

lib/puppetfile-resolver/puppetfile/base_module.rb

@@ -27,12 +27,19 @@ class BaseModule
 
       attr_reader :module_type
 
+      # Array of flags that will instruct the resolver to change its default behaviour. Current flags are
+      # set out in the PuppetfileResolver::Puppetfile::..._FLAG constants
+      # @api private
+      # @return [Array[Symbol]] Array of flags that will instruct the resolver to change its default behaviour
+      attr_accessor :resolver_flags
+
       def initialize(title)
         @title = title
         unless title.nil? # rubocop:disable Style/IfUnlessModifier
           @owner, @name = parse_title(@title)
         end
         @location = DocumentLocation.new
+        @resolver_flags = []
       end
 
       def to_s

lib/puppetfile-resolver/puppetfile/parser/r10k_eval.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'puppetfile-resolver/puppetfile'
 require 'puppetfile-resolver/puppetfile/parser/errors'
 require 'puppetfile-resolver/puppetfile/document'
 require 'puppetfile-resolver/puppetfile/parser/r10k_eval/dsl'
@@ -51,8 +52,81 @@ def self.parse(puppetfile_contents)
             raise err, e.backtrace
           end
 
+          # Post process magic comments
+          post_process_flags!(document)
+
+          # Freeze the flags so they can't be modified
+          document.modules.each { |mod| mod.resolver_flags.freeze }
+
           document
         end
+
+        # Parses a Puppetfile and applies the "magic comments"
+        def self.post_process_flags!(document)
+          flag_ranges = {}
+          document.content.lines.each_with_index do |line, index|
+            if (matches = line.match(%r{^\s*# resolver:disable ([A-Za-z\/,]+)(?:\s|$)}))
+              flags_from_line(matches[1]).each do |flag|
+                # Start a flag range if there isn't already one going
+                next unless flag_ranges[flag].nil?
+                flag_ranges[flag] = index
+              end
+            elsif (matches = line.match(%r{# resolver:disable ([A-Za-z\/,]+)(?:\s|$)}))
+              flags_from_line(matches[1]).each do |flag|
+                # Assert the flag if we're not already within a range
+                next unless flag_ranges[flag].nil?
+                assert_resolver_flag(document, flag, index, index)
+              end
+            elsif (matches = line.match(%r{^\s*# resolver:enable ([A-Za-z\/,]+)(?:\s|$)}))
+              flags_from_line(matches[1]).each do |flag|
+                # End a flag range if there isn't already one going
+                next if flag_ranges[flag].nil?
+                assert_resolver_flag(document, flag, flag_ranges[flag], index)
+                flag_ranges.delete(flag)
+              end
+            end
+          end
+
+          return if flag_ranges.empty?
+          # Any remaining flag ranges will be at the document end
+          end_line = document.content.lines.count
+          flag_ranges.each do |flag, start_line|
+            assert_resolver_flag(document, flag, start_line, end_line)
+          end
+        end
+        private_class_method :post_process_flags!
+
+        # Extracts the flags from the text based definitions
+        # @return [Array[Symbol]]
+        def self.flags_from_line(line)
+          line.split(',').map do |flag_name|
+            case flag_name.downcase
+            when 'dependency/puppet'
+              PuppetfileResolver::Puppetfile::DISABLE_PUPPET_DEPENDENCY_FLAG
+            when 'dependency/all'
+              PuppetfileResolver::Puppetfile::DISABLE_ALL_DEPENDENCIES_FLAG
+            else # rubocop:disable Style/EmptyElse We will be adding something here later
+              # TODO: Should we log a warning/info here?
+              nil
+            end
+          end.compact
+        end
+        private_class_method :flags_from_line
+
+        # Sets the specified flag on modules which are between from_line to to_line
+        def self.assert_resolver_flag(document, flag, from_line, to_line)
+          document.modules.each do |mod|
+            # If we don't know where the module is (?) then ignore it
+            next if mod.location.start_line.nil? || mod.location.end_line.nil?
+
+            # If the module doesn't span the range we're looking for (from_line --> to_line) ignore it
+            next unless mod.location.start_line >= from_line && mod.location.start_line <= to_line ||
+                        mod.location.end_line >= from_line && mod.location.end_line <= to_line
+            mod.resolver_flags << flag unless mod.resolver_flags.include?(flag)
+          end
+          nil
+        end
+        private_class_method :assert_resolver_flag
       end
     end
   end

lib/puppetfile-resolver/resolution_provider.rb

@@ -137,23 +137,23 @@ def find_all_module_specifications(dependency)
       unless mod.nil?
         case mod.module_type
         when Puppetfile::FORGE_MODULE
-          @module_info[dependency.name] = safe_spec_search { SpecSearchers::Forge.find_all(dependency, @cache, @resolver_ui) }
+          @module_info[dependency.name] = safe_spec_search(dependency) { SpecSearchers::Forge.find_all(dependency, @cache, @resolver_ui) }
         when Puppetfile::GIT_MODULE
-          @module_info[dependency.name] = safe_spec_search { SpecSearchers::Git.find_all(mod, dependency, @cache, @resolver_ui) }
+          @module_info[dependency.name] = safe_spec_search(dependency) { SpecSearchers::Git.find_all(mod, dependency, @cache, @resolver_ui) }
         else # rubocop:disable Style/EmptyElse
           # Errr.... Nothing
         end
       end
       return @module_info[dependency.name] unless @module_info[dependency.name].empty?
 
       # It's not in the Puppetfile, so perhaps it's in our modulepath?
-      @module_info[dependency.name] = safe_spec_search { SpecSearchers::Local.find_all(mod, @puppet_module_paths, dependency, @cache, @resolver_ui) }
+      @module_info[dependency.name] = safe_spec_search(dependency) { SpecSearchers::Local.find_all(mod, @puppet_module_paths, dependency, @cache, @resolver_ui) }
       return @module_info[dependency.name] unless @module_info[dependency.name].empty?
 
       # It's not in the Puppetfile and not on disk, so perhaps it's on the Forge?
       # The forge needs an owner and name to be able to resolve
       if dependency.name && dependency.owner # rubocop:disable Style/IfUnlessModifier
-        @module_info[dependency.name] = safe_spec_search { SpecSearchers::Forge.find_all(dependency, @cache, @resolver_ui) }
+        @module_info[dependency.name] = safe_spec_search(dependency) { SpecSearchers::Forge.find_all(dependency, @cache, @resolver_ui) }
       end
 
       # If we can't find any specifications for the module and we're allowing missing modules
@@ -164,8 +164,13 @@ def find_all_module_specifications(dependency)
       @module_info[dependency.name]
     end
 
-    def safe_spec_search
-      yield
+    def safe_spec_search(dependency)
+      results = yield
+      # The PuppetfileDependency has the resolver flags, so we need to inject them into the specifications
+      return results unless dependency.is_a?(PuppetfileResolver::Models::PuppetfileDependency) || results.empty?
+      results.each { |spec| spec.resolver_flags = dependency.puppetfile_module.resolver_flags }
+
+      results
     rescue StandardError => e
       if @allow_missing_modules
         @resolver_ui.debug { "Error while querying a specification searcher #{e.inspect}" }