go/.../analysisutil: ExtractDoc unifies Analyzer.Doc and package doc

This CL defines an internal function, ExtractDoc, which extracts
documentation from the analyzer's package doc. This allows a single
doc comment to serve as both the package doc comment (which is
served by sites such as pkg.go.dev) and the analyzer documentation
(which is accessible through the command-line tool), appropriately
formatted for both media.

Also, change all our analyzers with nontrivial documentation
to use it.

The chosen syntax permits a single doc comment to document multiple
analyzers and looks good in the pkgsite HTML rendering and the
go vet help output. The HTML heading anchors are predictable.

For now this is internal, but we might want to publish it.
(After a proposal.)

Updates golang/go#58950
See golang/go#57906

Change-Id: Ifc0f48e54c3e42bc598649a7139e178a1a653c13
Reviewed-on: https://go-review.googlesource.com/c/tools/+/474935
Run-TryBot: Alan Donovan <[email protected]>
Reviewed-by: Hyang-Ah Hana Kim <[email protected]>
gopls-CI: kokoro <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>

Author: adonovan

go/analysis/passes/assign/assign.go

@@ -2,13 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package assign defines an Analyzer that detects useless assignments.
 package assign
 
 // TODO(adonovan): check also for assignments to struct fields inside
 // methods that are on T instead of *T.
 
 import (
+	_ "embed"
 	"fmt"
 	"go/ast"
 	"go/token"
@@ -21,15 +21,12 @@ import (
 	"golang.org/x/tools/go/ast/inspector"
 )
 
-const Doc = `check for useless assignments
-
-This checker reports assignments of the form x = x or a[i] = a[i].
-These are almost always useless, and even when they aren't they are
-usually a mistake.`
+//go:embed doc.go
+var doc string
 
 var Analyzer = &analysis.Analyzer{
 	Name:     "assign",
-	Doc:      Doc,
+	Doc:      analysisutil.MustExtractDoc(doc, "assign"),
 	URL:      "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/assign",
 	Requires: []*analysis.Analyzer{inspect.Analyzer},
 	Run:      run,

go/analysis/passes/assign/doc.go

@@ -0,0 +1,14 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package assign defines an Analyzer that detects useless assignments.
+//
+// # Analyzer assign
+//
+// assign: check for useless assignments
+//
+// This checker reports assignments of the form x = x or a[i] = a[i].
+// These are almost always useless, and even when they aren't they are
+// usually a mistake.
+package assign

go/analysis/passes/atomic/atomic.go

@@ -2,11 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package atomic defines an Analyzer that checks for common mistakes
-// using the sync/atomic package.
 package atomic
 
 import (
+	_ "embed"
 	"go/ast"
 	"go/token"
 	"go/types"
@@ -17,17 +16,12 @@ import (
 	"golang.org/x/tools/go/ast/inspector"
 )
 
-const Doc = `check for common mistakes using the sync/atomic package
-
-The atomic checker looks for assignment statements of the form:
-
-	x = atomic.AddUint64(&x, 1)
-
-which are not atomic.`
+//go:embed doc.go
+var doc string
 
 var Analyzer = &analysis.Analyzer{
 	Name:             "atomic",
-	Doc:              Doc,
+	Doc:              analysisutil.MustExtractDoc(doc, "atomic"),
 	URL:              "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/atomic",
 	Requires:         []*analysis.Analyzer{inspect.Analyzer},
 	RunDespiteErrors: true,

go/analysis/passes/atomic/doc.go

@@ -0,0 +1,17 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package atomic defines an Analyzer that checks for common mistakes
+// using the sync/atomic package.
+//
+// # Analyzer atomic
+//
+// atomic: check for common mistakes using the sync/atomic package
+//
+// The atomic checker looks for assignment statements of the form:
+//
+//	x = atomic.AddUint64(&x, 1)
+//
+// which are not atomic.
+package atomic

go/analysis/passes/ifaceassert/doc.go

@@ -0,0 +1,24 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package ifaceassert defines an Analyzer that flags
+// impossible interface-interface type assertions.
+//
+// # Analyzer ifaceassert
+//
+// ifaceassert: detect impossible interface-to-interface type assertions
+//
+// This checker flags type assertions v.(T) and corresponding type-switch cases
+// in which the static type V of v is an interface that cannot possibly implement
+// the target interface T. This occurs when V and T contain methods with the same
+// name but different signatures. Example:
+//
+//	var v interface {
+//		Read()
+//	}
+//	_ = v.(io.Reader)
+//
+// The Read method in v has a different signature than the Read method in
+// io.Reader, so this assertion cannot succeed.
+package ifaceassert

go/analysis/passes/ifaceassert/ifaceassert.go

@@ -2,38 +2,25 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package ifaceassert defines an Analyzer that flags
-// impossible interface-interface type assertions.
 package ifaceassert
 
 import (
+	_ "embed"
 	"go/ast"
 	"go/types"
 
 	"golang.org/x/tools/go/analysis"
 	"golang.org/x/tools/go/analysis/passes/inspect"
+	"golang.org/x/tools/go/analysis/passes/internal/analysisutil"
 	"golang.org/x/tools/go/ast/inspector"
 )
 
-const Doc = `detect impossible interface-to-interface type assertions
-
-This checker flags type assertions v.(T) and corresponding type-switch cases
-in which the static type V of v is an interface that cannot possibly implement
-the target interface T. This occurs when V and T contain methods with the same
-name but different signatures. Example:
-
-	var v interface {
-		Read()
-	}
-	_ = v.(io.Reader)
-
-The Read method in v has a different signature than the Read method in
-io.Reader, so this assertion cannot succeed.
-`
+//go:embed doc.go
+var doc string
 
 var Analyzer = &analysis.Analyzer{
 	Name:     "ifaceassert",
-	Doc:      Doc,
+	Doc:      analysisutil.MustExtractDoc(doc, "ifaceassert"),
 	URL:      "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/ifaceassert",
 	Requires: []*analysis.Analyzer{inspect.Analyzer},
 	Run:      run,

go/analysis/passes/internal/analysisutil/extractdoc.go

@@ -0,0 +1,113 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package analysisutil
+
+import (
+	"fmt"
+	"go/parser"
+	"go/token"
+	"strings"
+)
+
+// MustExtractDoc is like [ExtractDoc] but it panics on error.
+//
+// To use, define a doc.go file such as:
+//
+//	// Package halting defines an analyzer of program termination.
+//	//
+//	// # Analyzer halting
+//	//
+//	// halting: reports whether execution will halt.
+//	//
+//	// The halting analyzer reports a diagnostic for functions
+//	// that run forever. To suppress the diagnostics, try inserting
+//	// a 'break' statement into each loop.
+//	package halting
+//
+//	import _ "embed"
+//
+//	//go:embed doc.go
+//	var doc string
+//
+// And declare your analyzer as:
+//
+//	var Analyzer = &analysis.Analyzer{
+//		Name:             "halting",
+//		Doc:              analysisutil.MustExtractDoc(doc, "halting"),
+//		...
+//	}
+func MustExtractDoc(content, name string) string {
+	doc, err := ExtractDoc(content, name)
+	if err != nil {
+		panic(err)
+	}
+	return doc
+}
+
+// ExtractDoc extracts a section of a package doc comment from the
+// provided contents of an analyzer package's doc.go file.
+//
+// A section is a portion of the comment between one heading and
+// the next, using this form:
+//
+//	# Analyzer NAME
+//
+//	NAME: SUMMARY
+//
+//	Full description...
+//
+// where NAME matches the name argument, and SUMMARY is a brief
+// verb-phrase that describes the analyzer. The following lines, up
+// until the next heading or the end of the comment, contain the full
+// description. ExtractDoc returns the portion following the colon,
+// which is the form expected by Analyzer.Doc.
+//
+// Example:
+//
+//	# Analyzer printf
+//
+//	printf: checks consistency of calls to printf
+//
+//	The printf analyzer checks consistency of calls to printf.
+//	Here is the complete description...
+//
+// This notation allows a single doc comment to provide documentation
+// for multiple analyzers, each in its own section.
+// The HTML anchors generated for each heading are predictable.
+//
+// It returns an error if the content was not a valid Go source file
+// containing a package doc comment with a heading of the required
+// form.
+//
+// This machinery enables the package documentation (typically
+// accessible via the web at https://pkg.go.dev/) and the command
+// documentation (typically printed to a terminal) to be derived from
+// the same source and formatted appropriately.
+func ExtractDoc(content, name string) (string, error) {
+	if content == "" {
+		return "", fmt.Errorf("empty Go source file")
+	}
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "", content, parser.ParseComments|parser.PackageClauseOnly)
+	if err != nil {
+		return "", fmt.Errorf("not a Go source file")
+	}
+	if f.Doc == nil {
+		return "", fmt.Errorf("Go source file has no package doc comment")
+	}
+	for _, section := range strings.Split(f.Doc.Text(), "\n# ") {
+		if body := strings.TrimPrefix(section, "Analyzer "+name); body != section &&
+			body != "" &&
+			body[0] == '\r' || body[0] == '\n' {
+			body = strings.TrimSpace(body)
+			rest := strings.TrimPrefix(body, name+":")
+			if rest == body {
+				return "", fmt.Errorf("'Analyzer %s' heading not followed by '%s: summary...' line", name, name)
+			}
+			return strings.TrimSpace(rest), nil
+		}
+	}
+	return "", fmt.Errorf("package doc comment contains no 'Analyzer %s' heading", name)
+}

go/analysis/passes/internal/analysisutil/extractdoc_test.go

@@ -0,0 +1,80 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package analysisutil_test
+
+import (
+	"testing"
+
+	"golang.org/x/tools/go/analysis/passes/internal/analysisutil"
+)
+
+func TestExtractDoc(t *testing.T) {
+	const multi = `// Copyright
+
+//+build tag
+
+// Package foo
+//
+// # Irrelevant heading
+//
+// This is irrelevant doc.
+//
+// # Analyzer nocolon
+//
+// This one has the wrong form for this line.
+//
+// # Analyzer food
+//
+// food: reports dining opportunities
+//
+// This is the doc for analyzer 'food'.
+//
+// # Analyzer foo
+//
+// foo: reports diagnostics
+//
+// This is the doc for analyzer 'foo'.
+//
+// # Analyzer bar
+//
+// bar: reports drinking opportunities
+//
+// This is the doc for analyzer 'bar'.
+package blah
+
+var x = syntax error
+`
+
+	for _, test := range []struct {
+		content, name string
+		want          string // doc or "error: %w" string
+	}{
+		{"", "foo",
+			"error: empty Go source file"},
+		{"//foo", "foo",
+			"error: not a Go source file"},
+		{"//foo\npackage foo", "foo",
+			"error: package doc comment contains no 'Analyzer foo' heading"},
+		{multi, "foo",
+			"reports diagnostics\n\nThis is the doc for analyzer 'foo'."},
+		{multi, "bar",
+			"reports drinking opportunities\n\nThis is the doc for analyzer 'bar'."},
+		{multi, "food",
+			"reports dining opportunities\n\nThis is the doc for analyzer 'food'."},
+		{multi, "nope",
+			"error: package doc comment contains no 'Analyzer nope' heading"},
+		{multi, "nocolon",
+			"error: 'Analyzer nocolon' heading not followed by 'nocolon: summary...' line"},
+	} {
+		got, err := analysisutil.ExtractDoc(test.content, test.name)
+		if err != nil {
+			got = "error: " + err.Error()
+		}
+		if test.want != got {
+			t.Errorf("ExtractDoc(%q) returned <<%s>>, want <<%s>>, given input <<%s>>",
+				test.name, got, test.want, test.content)
+		}
+	}
+}