Add support for HTTP proxies

This adds a few new configuration options to the searchers

- `git.proxy`: URL for the proxy to Git based requests
- `forge.proxy`: URL for the proxy to Forge requests

This allows users to use HTTP proxies when making requests to GitHub and the
Forge. Note that the HTTPS_PROXY and HTTP_PROXY environment variables are still
viable options to set a HTTTP/S proxy.

Author: beechtom

lib/puppetfile-resolver/spec_searchers/forge.rb

@@ -44,19 +44,17 @@ def self.fetch_all_module_releases(owner, name, resolver_ui, config, &block)
         loops = 0
         loop do
           resolver_ui.debug { "Querying the forge for a module with #{uri}" }
-
-          http_options = { :use_ssl => uri.class == URI::HTTPS }
-          # Because on Windows Ruby doesn't use the Windows certificate store which has up-to date
-          # CA certs, we can't depend on someone setting the environment variable correctly. So use our
-          # static CA PEM file if SSL_CERT_FILE is not set.
-          http_options[:ca_file] = PuppetfileResolver::Util.static_ca_cert_file if ENV['SSL_CERT_FILE'].nil?
-
+          err_msg = "Unable to find module #{owner}-#{name} on #{config.forge_api}"
+          err_msg += config.proxy ? " with proxy #{config.proxy}: " : ': '
           response = nil
-          Net::HTTP.start(uri.host, uri.port, http_options) do |http|
-            request = Net::HTTP::Get.new uri
-            response = http.request request
+
+          begin
+            response = ::PuppetfileResolver::Util.net_http_get(uri, config.proxy)
+          rescue ::StandardError => e
+            raise err_msg + e.message
           end
-          raise "Expected HTTP Code 200, but received #{response.code} for URI #{uri}: #{response.inspect}" unless response.code == '200'
+
+          raise err_msg + "Expected HTTP Code 200, but received #{response.code}" unless response.code == '200'
 
           reply = ::JSON.parse(response.body)
           yield reply['results']
@@ -66,7 +64,7 @@ def self.fetch_all_module_releases(owner, name, resolver_ui, config, &block)
 
           # Circuit breaker in case the worst happens (max 1000 module releases)
           loops += 1
-          raise "Too many Forge API requests #{loops * 50}" if loops > 20
+          raise err_msg + "Too many Forge API requests #{loops}" if loops > 20
         end
       end
       private_class_method :fetch_all_module_releases

lib/puppetfile-resolver/spec_searchers/forge_configuration.rb

@@ -10,6 +10,8 @@ def forge_api
       end
 
       attr_writer :forge_api
+
+      attr_accessor :proxy
     end
   end
 end

lib/puppetfile-resolver/spec_searchers/git.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
+require 'puppetfile-resolver/util'
 require 'puppetfile-resolver/spec_searchers/common'
 require 'puppetfile-resolver/spec_searchers/git_configuration'
 
 module PuppetfileResolver
   module SpecSearchers
     module Git
-      def self.find_all(puppetfile_module, dependency, cache, resolver_ui, _config)
+      def self.find_all(puppetfile_module, dependency, cache, resolver_ui, config)
         dep_id = ::PuppetfileResolver::SpecSearchers::Common.dependency_cache_id(self, dependency)
         # Has the information been cached?
         return cache.load(dep_id) if cache.exist?(dep_id)
@@ -40,9 +41,19 @@ def self.find_all(puppetfile_module, dependency, cache, resolver_ui, _config)
         require 'net/http'
         require 'uri'
 
-        resolver_ui.debug { "Querying the Github with #{metadata_url}" }
-        response = Net::HTTP.get_response(URI.parse(metadata_url))
+        resolver_ui.debug { "Querying GitHub with #{metadata_url}" }
+        err_msg = "Unable to find module at #{puppetfile_module.remote}"
+        err_msg += config.proxy ? " with proxy #{config.proxy}: " : ': '
+        response = nil
+
+        begin
+          response = ::PuppetfileResolver::Util.net_http_get(metadata_url, config.proxy)
+        rescue ::StandardError => e
+          raise err_msg + e.message
+        end
+
         if response.code != '200'
+          resolver_ui.debug(err_msg + "Expected HTTP Code 200, but received #{response.code}")
           cache.save(dep_id, [])
           return []
         end

lib/puppetfile-resolver/spec_searchers/git_configuration.rb

@@ -3,6 +3,7 @@
 module PuppetfileResolver
   module SpecSearchers
     class GitConfiguration
+      attr_accessor :proxy
     end
   end
 end

lib/puppetfile-resolver/util.rb

@@ -19,5 +19,29 @@ def self.symbolise_object(object)
     def self.static_ca_cert_file
       @static_ca_cert_file ||= File.expand_path(File.join(__dir__, 'data', 'ruby_ca_certs.pem'))
     end
+
+    # Execute a HTTP/S GET query and return the response
+    # @param [String, URI] uri The URI to request
+    # @param [nil, String, URI] proxy The URI of the proxy server to use. Defaults to nil (No proxy server)
+    # @return [Net::HTTPResponse] the response of the request
+    def self.net_http_get(uri, proxy = nil)
+      uri = URI.parse(uri) unless uri.is_a?(URI)
+
+      http_options = { :use_ssl => uri.class == URI::HTTPS }
+      # Because on Windows Ruby doesn't use the Windows certificate store which has up-to date
+      # CA certs, we can't depend on someone setting the environment variable correctly. So use our
+      # static CA PEM file if SSL_CERT_FILE is not set.
+      http_options[:ca_file] = PuppetfileResolver::Util.static_ca_cert_file if ENV['SSL_CERT_FILE'].nil?
+
+      start_args = [uri.host, uri.port]
+
+      unless proxy.nil?
+        proxy = URI.parse(proxy) unless proxy.is_a?(URI)
+        start_args.concat([proxy.host, proxy.port, proxy.user, proxy.password])
+      end
+
+      Net::HTTP.start(*start_args, http_options) { |http| return http.request(Net::HTTP::Get.new(uri)) }
+      nil
+    end
   end
 end

puppetfile-cli.rb

@@ -18,6 +18,7 @@ def self.parse(options)
       cache_dir: nil,
       module_paths: [],
       path: nil,
+      proxy: nil,
       puppet_version: nil,
       strict: false
     }
@@ -41,6 +42,10 @@ def self.parse(options)
         args[:debug] = true
       end
 
+      opts.on('--proxy=PROXY_URL', 'HTTP/S Proxy server to use. For example http://localhost:8888') do |proxy|
+        args[:proxy] = proxy
+      end
+
       opts.on('-s', '--strict', 'Do not allow missing dependencies. Default false which marks dependencies as missing and does not raise an error.') do
         args[:strict] = true
       end
@@ -91,6 +96,10 @@ def self.parse(options)
 
 config = PuppetfileResolver::SpecSearchers::Configuration.new
 config.local.puppet_module_paths = options[:module_paths]
+unless options[:proxy].nil?
+  config.git.proxy = options[:proxy]
+  config.forge.proxy = options[:proxy]
+end
 
 opts = { cache: cache, ui: ui, spec_searcher_configuration: config, allow_missing_modules: !options[:strict] }
 

spec/fixtures/proxy.rb

@@ -0,0 +1,9 @@
+require 'webrick'
+require 'webrick/httpproxy'
+
+proxy = WEBrick::HTTPProxyServer.new Port: (ARGV[0].nil? ? 32768 : ARGV[0])
+
+trap 'INT'  do proxy.shutdown end
+trap 'TERM' do proxy.shutdown end
+
+proxy.start

spec/integration/proxy_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+require 'open3'
+require 'puppetfile-resolver/resolver'
+require 'puppetfile-resolver/puppetfile'
+
+describe 'Proxy Tests' do
+  let(:content) do <<-PUPFILE
+    forge 'https://forge.puppet.com'
+
+    mod 'powershell',
+      :git => 'https://github.com/puppetlabs/puppetlabs-powershell',
+      :tag => 'v4.0.0'
+
+    mod 'puppetlabs/stdlib', '6.3.0'
+    PUPFILE
+  end
+  let(:puppetfile) { ::PuppetfileResolver::Puppetfile::Parser::R10KEval.parse(content) }
+  let(:resolver_config) do
+    PuppetfileResolver::SpecSearchers::Configuration.new.tap do |obj|
+      obj.git.proxy = 'http://localhost:32768'
+      obj.forge.proxy = 'http://localhost:32768'
+    end
+  end
+
+  context 'with an invalid proxy server' do
+    it 'should not resolve a complete Puppetfile' do
+      resolver = PuppetfileResolver::Resolver.new(puppetfile)
+      result = resolver.resolve({
+        allow_missing_modules: true,
+        spec_searcher_configuration: resolver_config,
+      })
+
+      expect(result.specifications).to include('powershell')
+      expect(result.specifications['powershell']).to be_a(PuppetfileResolver::Models::MissingModuleSpecification)
+
+      expect(result.specifications).to include('stdlib')
+      expect(result.specifications['stdlib']).to be_a(PuppetfileResolver::Models::MissingModuleSpecification)
+    end
+  end
+
+  context 'with a valid proxy server' do
+    def start_proxy_server(start_options = ['--timeout=5'])
+      cmd = "ruby \"#{File.join(FIXTURES_DIR, 'proxy.rb')}\" 32768"
+
+      stdin, stdout, stderr, wait_thr = Open3.popen3(cmd)
+      # Wait for the Proxy server to indicate it started
+      line = nil
+      begin
+        line = stderr.readline
+      end until line =~ /#start/
+      stdout.close
+      stdin.close
+      [wait_thr, stderr]
+    end
+
+    before(:each) do
+      @server_thr, @server_pipe = start_proxy_server
+    end
+
+    after(:each) do
+      begin
+        Process.kill("KILL", @server_thr[:pid])
+        Process.wait(@server_thr[:pid])
+      rescue
+        # The server process may not exist and checking in a cross platform way in ruby is difficult
+        # Instead just swallow any errors
+      end
+
+      begin
+        @server_pipe.close
+      rescue
+        # The server process may not exist and checking in a cross platform way in ruby is difficult
+        # Instead just swallow any errors
+      end
+    end
+
+    it 'should resolve a complete Puppetfile' do
+      resolver = PuppetfileResolver::Resolver.new(puppetfile)
+      result = resolver.resolve({
+        allow_missing_modules: false,
+        spec_searcher_configuration: resolver_config,
+      })
+
+      expect(result.specifications).to include('powershell')
+      expect(result.specifications['powershell'].version.to_s).to eq('4.0.0')
+
+      expect(result.specifications).to include('stdlib')
+      expect(result.specifications['stdlib'].version.to_s).to eq('6.3.0')
+    end
+  end
+end