(PUP-12058) Add task to generate man (as markdown) reference

Gemfile

@@ -75,6 +75,8 @@ end
 group(:documentation, optional: true) do
   gem 'gettext-setup', '~> 1.0', require: false, platforms: [:ruby]
   gem 'ronn', '~> 0.7.3', require: false, platforms: [:ruby]
+  gem 'puppet-strings', require: false, platforms: [:ruby]
+  gem 'pandoc-ruby', require: false, platforms: [:ruby]
 end
 
 if File.exist? "#{__FILE__}.local"

rakelib/generate_references.rake

@@ -1,12 +1,20 @@
 require 'tempfile'
 
 OUTPUT_DIR = 'references'
+MANDIR = File.join(OUTPUT_DIR, 'man')
+
 CONFIGURATION_ERB = File.join(__dir__, 'references/configuration.erb')
 CONFIGURATION_MD  = File.join(OUTPUT_DIR, 'configuration.md')
 METAPARAMETER_ERB = File.join(__dir__, 'references/metaparameter.erb')
 METAPARAMETER_MD  = File.join(OUTPUT_DIR, 'metaparameter.md')
 REPORT_ERB        = File.join(__dir__, 'references/report.erb')
 REPORT_MD         = File.join(OUTPUT_DIR, 'report.md')
+FUNCTIONS_TEMPLATE_ERB = File.join(__dir__, 'references/functions_template.erb')
+FUNCTION_ERB      = File.join(__dir__, 'references/function.erb')
+FUNCTION_MD       = File.join(OUTPUT_DIR, 'function.md')
+MAN_OVERVIEW_ERB  = File.join(__dir__, 'references/man/overview.erb')
+MAN_OVERVIEW_MD   = File.join(MANDIR, "overview.md")
+MAN_ERB           = File.join(__dir__, 'references/man.erb')
 
 def render_erb(erb_file, variables)
   # Create a binding so only the variables we specify will be visible
@@ -56,4 +64,138 @@ namespace :references do
     body = puppet_doc('report')
     generate_reference('report', REPORT_ERB, body, REPORT_MD)
   end
+
+  desc "Generate function reference"
+  task :function do
+    # Locate puppet-strings
+    begin
+      require 'puppet-strings'
+      require 'puppet-strings/version'
+    rescue LoadError
+      abort("Run `bundle config set with documentation` and `bundle update` to install the `puppet-strings` gem.")
+    end
+
+    strings_data = {}
+    Tempfile.create do |tmpfile|
+      puts "Running puppet strings #{PuppetStrings::VERSION}"
+      PuppetStrings.generate(['lib/puppet/{functions,parser/functions}/**/*.rb'], json: true, path: tmpfile.path)
+      strings_data = JSON.load_file(tmpfile.path)
+    end
+
+    # Based on https://github.com/puppetlabs/puppet-docs/blob/1a13be3fc6981baa8a96ff832ab090abc986830e/lib/puppet_references/puppet/functions.rb#L24-L56
+    functions = strings_data['puppet_functions']
+
+    # Deal with the duplicate 3.x and 4.x functions
+    # 1. Figure out which functions are duplicated.
+    names = functions.map { |func| func['name'] }
+    duplicates = names.uniq.select { |name| names.count(name) > 1 }
+    # 2. Reject the 3.x version of any dupes.
+    functions = functions.reject do |func|
+      duplicates.include?(func['name']) && func['type'] != 'ruby4x'
+    end
+
+    # renders the list of functions
+    body = render_erb(FUNCTIONS_TEMPLATE_ERB, functions: functions)
+
+    # This substitution could potentially make things a bit brittle, but it has to be done because the jump
+    # From H2s to H4s is causing issues with the DITA-OT, which sees this as a rule violation. If it
+    # Does become an issue, we should return to this and figure out a better way to generate the functions doc.
+    body.gsub!(/#####\s(.*?:)/,'**\1**').gsub!(/####\s/,'### ').chomp!
+
+    # renders the preamble and list of functions
+    generate_reference('function', FUNCTION_ERB, body, FUNCTION_MD)
+  end
+
+  desc "Generate man as markdown references"
+  task :man do
+    FileUtils.mkdir_p(MANDIR)
+
+    begin
+      require 'pandoc-ruby'
+    rescue LoadError
+      abort("Run `bundle config set with documentation` and `bundle update` to install the `pandoc-ruby` gem.")
+    end
+
+    pandoc = %x{which pandoc}.chomp
+    unless File.executable?(pandoc)
+      abort("Please install the `pandoc` package.")
+    end
+
+    sha = %x{git rev-parse HEAD}.chomp
+    now = Time.now
+
+    # This is based on https://github.com/puppetlabs/puppet-docs/blob/1a13be3fc6981baa8a96ff832ab090abc986830e/lib/puppet_references/puppet/man.rb#L24-L108
+    core_apps = %w(
+      agent
+      apply
+      lookup
+      module
+      resource
+    )
+    occasional_apps = %w(
+      config
+      describe
+      device
+      doc
+      epp
+      generate
+      help
+      node
+      parser
+      plugin
+      script
+      ssl
+    )
+    weird_apps = %w(
+      catalog
+      facts
+      filebucket
+      report
+    )
+
+    variables = {
+      sha: sha,
+      now: now,
+      title: 'Puppet Man Pages',
+      core_apps: core_apps,
+      occasional_apps: occasional_apps,
+      weird_apps: weird_apps
+    }
+
+    content = render_erb(MAN_OVERVIEW_ERB, variables)
+    File.write(MAN_OVERVIEW_MD, content)
+    puts "Generated #{MAN_OVERVIEW_MD}"
+
+    # Generate manpages in roff
+    Rake::Task[:gen_manpages].invoke
+
+    # Convert the roff formatted man pages to markdown, based on
+    # https://github.com/puppetlabs/puppet-docs/blob/1a13be3fc6981baa8a96ff832ab090abc986830e/lib/puppet_references/puppet/man.rb#L119-L128
+    files = Pathname.glob(File.join(__dir__, '../man/man8/*.8'))
+    files.each do |f|
+      next if File.basename(f) == "puppet.8"
+
+      app = File.basename(f).delete_prefix('puppet-').delete_suffix(".8")
+
+      body =
+        PandocRuby.convert([f], from: :man, to: :markdown)
+        .gsub(/#(.*?)\n/, '##\1')
+        .gsub(/:\s\s\s\n\n```\{=html\}\n<!--\s-->\n```/, '')
+        .gsub(/\n:\s\s\s\s/, '')
+        .chomp
+
+      variables = {
+        sha: sha,
+        now: now,
+        title: "Man Page: puppet #{app}",
+        canonical: "/puppet/latest/man/#{app}.html",
+        body: body
+      }
+
+      content = render_erb(MAN_ERB, variables)
+      output = File.join(MANDIR, "#{app}.md")
+      File.write(output, content)
+      puts "Generated #{output}"
+    end
+  end
 end

rakelib/manpages.rake

@@ -9,6 +9,8 @@ task :gen_manpages do
   bins  = Dir.glob(%w{bin/*})
   non_face_applications = helpface.legacy_applications
   faces = Puppet::Face.faces.map(&:to_s)
+  # exclude puppet-strings
+  faces.delete('strings')
   apps = non_face_applications + faces
 
   ronn_args = '--manual="Puppet manual" --organization="Puppet, Inc." --roff'

rakelib/references/function.erb

@@ -0,0 +1,37 @@
+---
+layout: default
+built_from_commit: <%= sha %>
+title: Built-in function reference
+canonical: "/puppet/latest/function.html"
+toc_levels: 2
+toc: columns
+---
+
+# Built-in function reference
+
+> **NOTE:** This page was generated from the Puppet source code on <%= now %>
+
+
+
+This page is a list of Puppet's built-in functions, with descriptions of what they do and how to use them.
+
+Functions are plugins you can call during catalog compilation. A call to any function is an expression that resolves to a value. For more information on how to call functions, see [the language reference page about function calls.](lang_functions.dita) 
+
+Many of these function descriptions include auto-detected _signatures,_ which are short reminders of the function's allowed arguments. These signatures aren't identical to the syntax you use to call the function; instead, they resemble a parameter list from a Puppet [class](lang_classes.dita), [defined resource type](lang_defined_types.dita), [function](lang_write_functions_in_puppet.dita), or [lambda](lang_lambdas.dita). The syntax of a signature is:
+
+```
+<FUNCTION NAME>(<DATA TYPE> <ARGUMENT NAME>, ...)
+```
+
+The `<DATA TYPE>` is a [Puppet data type value](lang_data_type.dita), like `String` or `Optional[Array[String]]`. The `<ARGUMENT NAME>` is a descriptive name chosen by the function's author to indicate what the argument is used for.
+
+* Any arguments with an `Optional` data type can be omitted from the function call.
+* Arguments that start with an asterisk (like `*$values`) can be repeated any number of times.
+* Arguments that start with an ampersand (like `&$block`) aren't normal arguments; they represent a code block, provided with [Puppet's lambda syntax.](lang_lambdas.dita)
+
+## `undef` values in Puppet 6
+
+In Puppet 6, many Puppet types were moved out of the Puppet codebase, and into modules on the Puppet Forge. The new functions handle `undef` values more strictly than their stdlib counterparts. In Puppet 6, code that relies on `undef` values being implicitly treated as other types will return an evaluation error. For more information on which types were moved into modules, see the [Puppet 6 release notes](https://puppet.com/docs/puppet/6.0/release_notes_puppet.html#select-types-moved-to-modules).
+
+
+<%= body %>

rakelib/references/functions_template.erb

@@ -0,0 +1,47 @@
+<% functions.sort{|a,b| a['name'] <=> b['name'] }.each do |func| -%>
+## `<%= func['name'] %>`
+
+<%= func['docstring']['text'] %>
+
+<% func_signatures = func['signatures']
+multiple_signatures = func_signatures.count > 1
+if func_signatures
+func['signatures'].each_with_index do |signature, index| -%>
+
+<% if multiple_signatures -%>
+Signature <%= index+1 %>
+
+<% end -%>
+`<%= signature['signature'] %>`
+<% has_parameters = signature.dig('docstring', 'tags')&.detect {|tag| tag['tag_name'] == 'param' && tag['text'] != '' && tag['text'] != nil } || false
+if has_parameters -%>
+
+### Parameters
+
+<% signature['docstring']['tags'].select {|tag| tag['tag_name'] == 'param' && tag['text'] != '' && tag['text'] != nil}.each do |param| -%>
+
+* `<%= param['name'] %>` --- <%= param['text'] %>
+<% end # each param
+
+return_types = signature['docstring']['tags'].detect {|tag| tag['tag_name'] == 'return'}
+if return_types -%>
+
+Return type(s): <%= return_types['types'].map {|t| "`#{t}`"}.join(', ') %>. <%= return_types['text'] %>
+<% end # if return_types
+has_examples = signature['docstring']['tags'].detect {|tag| tag['tag_name'] == 'example' && tag['text'] != '' && tag['text'] != nil }
+if has_examples %>
+
+### Examples
+
+<% signature['docstring']['tags'].select {|tag| tag['tag_name'] == 'example' && tag['text'] != '' && tag['text'] != nil}.each do |example| -%>
+<%= example['name'] %>
+
+<%= example['text'] %>
+
+<% end # each example
+end # if has_examples
+end-%>
+<% end # each signature
+end -%>
+
+<% end -%>

rakelib/references/man.erb

@@ -0,0 +1,12 @@
+---
+layout: default
+built_from_commit: <%= sha %>
+title: '<%= title %>'
+canonical: "<%= canonical %>"
+---
+
+# <%= title %>
+
+> **NOTE:** This page was generated from the Puppet source code on <%= now %>
+
+<%= body %>

rakelib/references/man/overview.erb

@@ -0,0 +1,47 @@
+---
+layout: default
+built_from_commit: <%= sha %>
+title: <%= title %>
+canonical: "/puppet/latest/man/overview.html"
+---
+
+# <%= title %>
+
+> **NOTE:** This page was generated from the Puppet source code on <%= now %>
+
+
+
+Puppet's command line tools consist of a single `puppet` binary with many subcommands. The following subcommands are available in this version of Puppet:
+
+Core Tools
+-----
+
+These subcommands form the core of Puppet's tool set, and every user should understand what they do.
+
+<% core_apps.each do |app| -%>
+- [puppet <%= app %>](<%= app %>.md)
+<% end -%>
+
+
+> Note: The `puppet cert` command is available only in Puppet versions prior to 6.0. For 6.0 and later, use the [`puppetserver cert`command](https://puppet.com/docs/puppet/6/puppet_server_ca_cli.html).
+
+Secondary subcommands
+-----
+
+Many or most users need to use these subcommands at some point, but they aren't needed for daily use the way the core tools are.
+
+<% occasional_apps.each do |app| -%>
+- [puppet <%= app %>](<%= app %>.md)
+<% end -%>
+
+
+Niche subcommands
+-----
+
+Most users can ignore these subcommands. They're only useful for certain niche workflows, and most of them are interfaces to Puppet's internal subsystems.
+
+<% weird_apps.each do |app| -%>
+- [puppet <%= app %>](<%= app %>.md)
+<% end -%>
+
+