go mod support for new operators (#1001)

* commands/.../print_deps.go: print deps based on dep manager type

* release.sh: update after go mod changes

* pkg/scaffold/project/tools.go: k8s, openapi, CRD code generator runtime dependencies

* *.md: update docs dep -> go mod, Gopkg.* -> go.*

* internal/pkg/scaffold/*go_mod.go: go, ansible, helm `go.mod` scaffolds

* test/e2e/memcached_test.go: go mod gets local changes instead of copying

* internal/util/projutil/project_util.go: functions for detecting operator type

* CHANGELOG.md: added go module support

* set GO111MODULE=on in ansible and helm e2e

* bump to go 1.12

* Gopkg.lock: revendor

Author: Eric Stroczynski

.travis.yml

@@ -7,7 +7,7 @@ cache:
     - $HOME/.cache/go-build
 
 go:
-- 1.10.3
+- 1.12.x
 
 # The `x_base_steps` top-level key is unknown to travis,
 # so we can use it to create a bunch of common build step

CHANGELOG.md

@@ -5,6 +5,7 @@
 - New option for [`operator-sdk build --image-builder`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#build), which can be used to specify which image builder to use. Adds support for [buildah](https://github.com/containers/buildah/). ([#1311](https://github.com/operator-framework/operator-sdk/pull/1311))
 - Manager is now configured with a new `DynamicRESTMapper`, which accounts for the fact that the default `RESTMapper`, which only checks resource types at startup, can't handle the case of first creating a CRD and then an instance of that CRD. ([#1329](https://github.com/operator-framework/operator-sdk/pull/1329))
 - Unify CLI debug logging under a global `--verbose` flag ([#1361](https://github.com/operator-framework/operator-sdk/pull/1361))
+- [Go module](https://github.com/golang/go/wiki/Modules) support by default for new Go operators and during Ansible and Helm operator migration. The dependency manager used for a new operator can be explicitly specified for new operators through the `--dep-manager` flag, available in [`operator-sdk new`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#new) and [`operator-sdk migrate`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#migrate). `dep` is still available through `--dep-manager=dep`. ([#1001](https://github.com/operator-framework/operator-sdk/pull/1001))
 
 ### Changed
 

Gopkg.lock

@@ -40,7 +40,7 @@ The following workflow is for a new **Helm** operator:
 
 - [dep][dep_tool] version v0.5.0+.
 - [git][git_tool]
-- [go][go_tool] version v1.10+.
+- [go][go_tool] version v1.12+.
 - [docker][docker_tool] version 17.03+.
 - [kubectl][kubectl_tool] version v1.11.3+.
 - Access to a Kubernetes v1.11.3+ cluster.
@@ -72,6 +72,7 @@ Create and deploy an app-operator using the SDK CLI:
 $ mkdir -p $GOPATH/src/github.com/example-inc/
 # Create a new app-operator project
 $ cd $GOPATH/src/github.com/example-inc/
+$ export GO111MODULE=on
 $ operator-sdk new app-operator
 $ cd app-operator
 

README.md

@@ -184,6 +184,7 @@ func buildFunc(cmd *cobra.Command, args []string) error {
 			PackagePath: filepath.Join(projutil.CheckAndGetProjectGoPkg(), scaffold.ManagerDir),
 			Args:        goTrimFlags,
 			Env:         goBuildEnv,
+			GoMod:       projutil.IsDepManagerGoMod(),
 		}
 		if err := projutil.GoBuild(opts); err != nil {
 			return fmt.Errorf("failed to build operator binary: (%v)", err)
@@ -218,6 +219,7 @@ func buildFunc(cmd *cobra.Command, args []string) error {
 					PackagePath: testLocationBuild + "/...",
 					Args:        append(goTrimFlags, "-c"),
 					Env:         goBuildEnv,
+					GoMod:       projutil.IsDepManagerGoMod(),
 				},
 			}
 			if err := projutil.GoTest(opts); err != nil {

cmd/operator-sdk/build/cmd.go

@@ -44,7 +44,7 @@ func NewCmd() *cobra.Command {
 		RunE:  migrateRun,
 	}
 
-	newCmd.Flags().StringVar(&depManager, "dep-manager", "dep", `Dependency manager the new project will use (choices: "dep")`)
+	newCmd.Flags().StringVar(&depManager, "dep-manager", "modules", `Dependency manager the new project will use (choices: "dep", "modules")`)
 	newCmd.Flags().StringVar(&headerFile, "header-file", "", "Path to file containing headers for generated Go files. Copied to hack/boilerplate.go.txt")
 
 	return newCmd
@@ -179,8 +179,13 @@ func scaffoldHelmDepManager(s *scaffold.Scaffold, cfg *input.Config) error {
 	switch m := projutil.DepManagerType(depManager); m {
 	case projutil.DepManagerDep:
 		files = append(files, &helm.GopkgToml{})
+	case projutil.DepManagerGoMod:
+		if err := goModCheck(); err != nil {
+			return err
+		}
+		files = append(files, &helm.GoMod{}, &scaffold.Tools{})
 	default:
-		return projutil.ErrInvalidDepManager
+		return projutil.ErrInvalidDepManager(depManager)
 	}
 	return s.Execute(cfg, files...)
 }
@@ -190,8 +195,22 @@ func scaffoldAnsibleDepManager(s *scaffold.Scaffold, cfg *input.Config) error {
 	switch m := projutil.DepManagerType(depManager); m {
 	case projutil.DepManagerDep:
 		files = append(files, &ansible.GopkgToml{})
+	case projutil.DepManagerGoMod:
+		if err := goModCheck(); err != nil {
+			return err
+		}
+		files = append(files, &ansible.GoMod{}, &scaffold.Tools{})
 	default:
-		return projutil.ErrInvalidDepManager
+		return projutil.ErrInvalidDepManager(depManager)
 	}
 	return s.Execute(cfg, files...)
 }
+
+func goModCheck() error {
+	goModOn, err := projutil.GoModOn()
+	if err == nil && !goModOn {
+		log.Fatal(`Dependency manager "modules" has been selected but go modules are not active. ` +
+			`Activate modules then run "operator-sdk migrate".`)
+	}
+	return err
+}

cmd/operator-sdk/migrate/cmd.go

@@ -54,7 +54,7 @@ generates a skeletal app-operator application in $GOPATH/src/github.com/example.
 	newCmd.Flags().StringVar(&apiVersion, "api-version", "", "Kubernetes apiVersion and has a format of $GROUP_NAME/$VERSION (e.g app.example.com/v1alpha1) - used with \"ansible\" or \"helm\" types")
 	newCmd.Flags().StringVar(&kind, "kind", "", "Kubernetes CustomResourceDefintion kind. (e.g AppService) - used with \"ansible\" or \"helm\" types")
 	newCmd.Flags().StringVar(&operatorType, "type", "go", "Type of operator to initialize (choices: \"go\", \"ansible\" or \"helm\")")
-	newCmd.Flags().StringVar(&depManager, "dep-manager", "dep", `Dependency manager the new project will use (choices: "dep")`)
+	newCmd.Flags().StringVar(&depManager, "dep-manager", "modules", `Dependency manager the new project will use (choices: "dep", "modules")`)
 	newCmd.Flags().BoolVar(&skipGit, "skip-git-init", false, "Do not init the directory as a git repository")
 	newCmd.Flags().StringVar(&headerFile, "header-file", "", "Path to file containing headers for generated Go files. Copied to hack/boilerplate.go.txt")
 	newCmd.Flags().BoolVar(&generatePlaybook, "generate-playbook", false, "Generate a playbook skeleton. (Only used for --type ansible)")
@@ -166,8 +166,16 @@ func doGoScaffold() error {
 	switch m := projutil.DepManagerType(depManager); m {
 	case projutil.DepManagerDep:
 		err = s.Execute(cfg, &scaffold.GopkgToml{})
+	case projutil.DepManagerGoMod:
+		if goModOn, merr := projutil.GoModOn(); merr != nil {
+			return merr
+		} else if !goModOn {
+			log.Fatalf(`Dependency manager "%s" has been selected but go modules are not active. `+
+				`Activate modules then run "operator-sdk new %s".`, m, projectName)
+		}
+		err = s.Execute(cfg, &scaffold.GoMod{}, &scaffold.Tools{})
 	default:
-		err = projutil.ErrInvalidDepManager
+		err = projutil.ErrNoDepManager
 	}
 	if err != nil {
 		return fmt.Errorf("dependency manager file scaffold failed: (%v)", err)
@@ -392,8 +400,13 @@ func getDeps() error {
 		if err := execProjCmd("dep", "ensure", "-v"); err != nil {
 			return err
 		}
+	case projutil.DepManagerGoMod:
+		log.Info("Running go mod ...")
+		if err := execProjCmd("go", "mod", "vendor", "-v"); err != nil {
+			return err
+		}
 	default:
-		return projutil.ErrInvalidDepManager
+		return projutil.ErrInvalidDepManager(depManager)
 	}
 	log.Info("Done getting dependencies")
 	return nil

cmd/operator-sdk/new/cmd.go

@@ -33,15 +33,16 @@ func NewCmd() *cobra.Command {
 		Short: "Print Golang packages and versions required to run the operator",
 		Long: `The operator-sdk print-deps command prints all Golang packages and versions expected
 by this version of the Operator SDK. Versions for these packages should match
-those in an operators' Gopkg.toml file.
+those in an operators' go.mod or Gopkg.toml file, depending on the dependency
+manager chosen when initializing or migrating a project.
 
 print-deps prints in columnar format by default. Use the --as-file flag to
-print in Gopkg.toml file format.
+print in go.mod or Gopkg.toml file format.
 `,
 		RunE: printDepsFunc,
 	}
 
-	printDepsCmd.Flags().BoolVar(&asFile, "as-file", false, "Print dependencies in Gopkg.toml file format.")
+	printDepsCmd.Flags().BoolVar(&asFile, "as-file", false, "Print dependencies in go.mod or Gopkg.toml file format, depending on the dependency manager chosen when initializing or migrating a project")
 
 	return printDepsCmd
 }
@@ -72,14 +73,17 @@ func printDeps(asFile bool) error {
 		if isDep {
 			return ansible.PrintDepGopkgTOML(asFile)
 		}
+		return ansible.PrintGoMod(asFile)
 	case projutil.IsOperatorHelm():
 		if isDep {
 			return helm.PrintDepGopkgTOML(asFile)
 		}
+		return helm.PrintGoMod(asFile)
 	case projutil.IsOperatorGo():
 		if isDep {
 			return scaffold.PrintDepGopkgTOML(asFile)
 		}
+		return scaffold.PrintGoMod(asFile)
 	}
 
 	return projutil.ErrUnknownOperatorType{}

cmd/operator-sdk/printdeps/cmd.go

@@ -213,6 +213,7 @@ func testLocalGoFunc(cmd *cobra.Command, args []string) error {
 			PackagePath: args[0] + "/...",
 			Env:         append(os.Environ(), fmt.Sprintf("%v=%v", test.TestNamespaceEnv, tlConfig.namespace)),
 			Dir:         projutil.MustGetwd(),
+			GoMod:       projutil.IsDepManagerGoMod(),
 		},
 		TestBinaryArgs: testArgs,
 	}

cmd/operator-sdk/test/local.go

@@ -185,6 +185,7 @@ func buildLocal(outputBinName string) error {
 		BinName:     outputBinName,
 		PackagePath: filepath.Join(projutil.CheckAndGetProjectGoPkg(), scaffold.ManagerDir),
 		Args:        args,
+		GoMod:       projutil.IsDepManagerGoMod(),
 	}
 	return projutil.GoBuild(opts)
 }

cmd/operator-sdk/up/local.go

@@ -12,7 +12,7 @@ powered by Ansible using tools and libraries provided by the Operator SDK.
 - [ansible-runner][ansible_runner_tool] version v1.1.0+
 - [ansible-runner-http][ansible_runner_http_plugin] version v1.0.0+
 - [dep][dep_tool] version v0.5.0+. (Optional if you aren't installing from source)
-- [go][go_tool] version v1.10+. (Optional if you aren't installing from source)
+- [go][go_tool] version v1.12+. (Optional if you aren't installing from source)
 - Access to a Kubernetes v.1.9.0+ cluster.
 
 **Note**: This guide uses [minikube][minikube_tool] version v0.25.0+ as the

doc/ansible/user-guide.md

@@ -5,7 +5,7 @@ This document explains how to setup your dev environment.
 ## Prerequisites
 - [dep][dep_tool] version v0.5.0+
 - [git][git_tool]
-- [go][go_tool] version v1.10+
+- [go][go_tool] version v1.12+
 
 ## Download Operator SDK
 

doc/dev/developer_guide.md

@@ -171,7 +171,7 @@ Create a new branch to push release commits:
 $ git checkout -b release-v1.3.0
 ```
 
-Commit changes to the following six files:
+Commit changes to the following files:
 
 - `version/version.go`: update `Version` to `v1.3.0`.
 - `internal/pkg/scaffold/gopkgtoml.go`, under the `[[constraint]]` for `github.com/operator-framework/operator-sdk`:
@@ -181,6 +181,12 @@ Commit changes to the following six files:
 - `internal/pkg/scaffold/gopkgtoml_test.go`: same as for `internal/pkg/scaffold/gopkgtoml.go`.
 - `internal/pkg/scaffold/ansible/gopkgtoml.go`: same as for `internal/pkg/scaffold/gopkgtoml.go`.
 - `internal/pkg/scaffold/helm/gopkgtoml.go`: same as for `internal/pkg/scaffold/gopkgtoml.go`.
+- `internal/pkg/scaffold/go_mod.go`, in the `replace` block for `github.com/operator-framework/operator-sdk`:
+  - Add the following `replace` entry: `github.com/operator-framework/operator-sdk => github.com/operator-framework/operator-sdk v1.3.0`.
+  - If an entry already exists, change the version to `v1.3.0`.
+- `internal/pkg/scaffold/go_mod_test.go`: same as for `internal/pkg/scaffold/go_mod.go`.
+- `internal/pkg/scaffold/helm/go_mod.go`: same as for `internal/pkg/scaffold/go_mod.go`.
+- `internal/pkg/scaffold/ansible/go_mod.go`: same as for `internal/pkg/scaffold/go_mod.go`.
 - `CHANGELOG.md`: update the `## Unreleased` header to `## v1.3.0`.
 
 Create a new PR for `release-v1.3.0`.

doc/dev/release.md

@@ -79,7 +79,7 @@ The Go, Ansible, and Helm tests then differ in what tests they run.
     6. Create a memcached CR and wait for it to be ready.
     7. Create a configmap that the memcached-operator is configured to delete using a finalizer.
     8. Delete memcached CR and verify that the finalizer deleted the configmap.
-    9. Run `operator-sdk migrate` to add go source to the operator.
+    9. Run `operator-sdk migrate` to add go source to the operator (see this [note][deps_mgmt] on dependency management first).
     10. Run `operator-sdk build` to compile the new binary and build a new image.
     11. Re-run steps 4-8 to test the migrated operator.
 
@@ -97,7 +97,7 @@ The Go, Ansible, and Helm tests then differ in what tests they run.
     7. Scale up the dependent deployment and verify the operator reconciles it back down.
     8. Scale up the CR and verify the dependent deployment scales up accordingly.
     9. Delete nginx CR and verify that finalizer (which writes a message in the operator logs) ran.
-    10. Run `operator-sdk migrate` to add go source to the operator.
+    10. Run `operator-sdk migrate` to add go source to the operator (see this [note][deps_mgmt] on dependency management first).
     11. Run `operator-sdk build` to compile the new binary and build a new image.
     12. Re-run steps 4-9 to test the migrated operator.
 
@@ -122,3 +122,4 @@ The markdown test does not create a new cluster and runs in a barebones Travis V
 [helm-base]: ../../../hack/image/helm/scaffold-helm-image.go
 [marker-github]: https://github.com/crawford/marker
 [marker-local]: ../../../hack/ci/marker
+[deps_mgmt]: ../../user-guide.md#a-note-on-dependency-management

doc/dev/testing/travis-build.md

@@ -9,7 +9,7 @@ powered by Helm using tools and libraries provided by the Operator SDK.
 - [docker][docker_tool] version 17.03+.
 - [kubectl][kubectl_tool] version v1.11.3+.
 - [dep][dep_tool] version v0.5.0+. (Optional if you aren't installing from source)
-- [go][go_tool] version v1.10+. (Optional if you aren't installing from source)
+- [go][go_tool] version v1.12+. (Optional if you aren't installing from source)
 - Access to a Kubernetes v1.11.3+ cluster.
 
 **Note**: This guide uses [minikube][minikube_tool] version v0.25.0+ as the

doc/helm/user-guide.md

@@ -10,9 +10,10 @@ The `operator-sdk` CLI generates a number of packages for each project. The foll
 | pkg/controller | This pkg contains the controller implementations. Users are expected to edit the `pkg/controller/<kind>/<kind>_controller.go` to define the controller's reconcile logic for handling a resource type of the specified `kind`. |
 | build | Contains the `Dockerfile` and build scripts used to build the operator. |
 | deploy | Contains various YAML manifests for registering CRDs, setting up [RBAC][RBAC], and deploying the operator as a Deployment.
-| Gopkg.toml Gopkg.lock | The [Go Dep][dep] manifests that describe the external dependencies of this operator. |
-| vendor | The golang [vendor][Vendor] folder that contains the local copies of the external dependencies that satisfy the imports of this project. [Go Dep][dep] manages the vendor directly. |
+| (Gopkg.toml Gopkg.lock) or (go.mod go.sum) | The [Go mod][go_mod] or [Go Dep][dep] manifests that describe the external dependencies of this operator, depending on the dependency manager chosen when initializing or migrating a project. |
+| vendor | The golang [vendor][Vendor] folder that contains the local copies of the external dependencies that satisfy the imports of this project. [Go Dep][dep]/[Go modules][go_mod] manages the vendor directly. |
 
 [RBAC]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/
 [Vendor]: https://golang.org/cmd/go/#hdr-Vendor_Directories
+[go_mod]: https://github.com/golang/go/wiki/Modules
 [dep]: https://github.com/golang/dep

doc/project_layout.md

@@ -96,10 +96,12 @@ Prints the most recent Golang packages and versions required by operators. Print
 
 ### Flags
 
-* `--as-file` - Print packages and versions in Gopkg.toml format.
+* `--as-file` - Print packages and versions in go.mod or Gopkg.toml format, depending on the dependency manager chosen when initializing or migrating a project.
 
 ### Example
 
+With dependency manager `dep`:
+
 ```console
 $ operator-sdk print-deps --as-file
 required = [
@@ -118,6 +120,19 @@ required = [
 ...
 ```
 
+With dependency manager `modules`, i.e. go mod:
+
+```console
+$ operator-sdk print-deps --as-file
+module github.com/example-inc/memcached-operator
+
+require (
+	contrib.go.opencensus.io/exporter/ocagent v0.4.9 // indirect
+	github.com/Azure/go-autorest v11.5.2+incompatible // indirect
+	github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30 // indirect
+	github.com/coreos/prometheus-operator v0.26.0 // indirect
+```
+
 ## generate
 
 ### k8s
@@ -218,20 +233,21 @@ you will need to rename it before running migrate or manually add it to your Doc
 
 #### Flags
 
-* `--dep-manager` string - Dependency manager the migrated project will use (choices: "dep")
+* `--dep-manager` string - Dependency manager the migrated project will use (choices: "dep", "modules") (default "modules")
 
 ### Example
 
 ```console
 $ operator-sdk migrate
-2019/01/10 15:02:45 No playbook was found, so not including it in the new Dockerfile
-2019/01/10 15:02:45 renamed Dockerfile to build/Dockerfile.sdkold and replaced with newer version
-2019/01/10 15:02:45 Compare the new Dockerfile to your old one and manually migrate any customizations
+INFO[0000] No playbook was found, so not including it in the new Dockerfile
+INFO[0000] Renamed Dockerfile to build/Dockerfile.sdkold and replaced with newer version. Compare the new Dockerfile to your old one and manually migrate any customizations
+INFO[0000] Created go.mod
 INFO[0000] Created cmd/manager/main.go
-INFO[0000] Created Gopkg.toml
 INFO[0000] Created build/Dockerfile
 INFO[0000] Created bin/entrypoint
 INFO[0000] Created bin/user_setup
+INFO[0000] Created library/k8s_status.py
+INFO[0000] Created bin/ao-logs
 ```
 
 ## new
@@ -253,7 +269,7 @@ Scaffolds a new operator project.
 * `--helm-chart` string - Initialize helm operator with existing helm chart (`<URL>`, `<repo>/<name>`, or local path)
 * `--helm-chart-repo` string - Chart repository URL for the requested helm chart
 * `--helm-chart-version` string - Specific version of the helm chart (default is latest version)
-* `--dep-manager` string - Dependency manager the new project will use (choices: "dep")
+* `--dep-manager` string - Dependency manager the new project will use (choices: "dep", "modules") (default "modules")
 * `-h, --help` - help for new
 
 ### Example