diff --git a/.travis.yml b/.travis.yml index a098afb9706..ecafce9ca51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,62 +30,10 @@ install: - go get -t -d ./... script: -# Unit test and verify formatting -- go test ./... -- go vet ./... - make install -# Create example operator -- cd $GOPATH/src/github.com/example-inc -- operator-sdk new memcached-operator --api-version=cache.example.com/v1alpha1 --kind=Memcached -- cd memcached-operator -- rm -rf vendor/github.com/operator-framework/operator-sdk/pkg -- ln -s ${TRAVIS_BUILD_DIR}/pkg vendor/github.com/operator-framework/operator-sdk/pkg -- curl https://raw.githubusercontent.com/operator-framework/operator-sdk/master/example/memcached-operator/handler.go.tmpl -o pkg/stub/handler.go -- head -n -6 pkg/apis/cache/v1alpha1/types.go > tmp.txt -- mv tmp.txt pkg/apis/cache/v1alpha1/types.go -- echo 'type MemcachedSpec struct { Size int32 `json:"size"`}' >> pkg/apis/cache/v1alpha1/types.go -- echo 'type MemcachedStatus struct {Nodes []string `json:"nodes"`}' >> pkg/apis/cache/v1alpha1/types.go -- operator-sdk generate k8s -- operator-sdk build quay.io/example/memcached-operator:v0.0.1 -- | - sed -ie 's/imagePullPolicy: Always/imagePullPolicy: Never/g' deploy/operator.yaml -# Run the operator as a pod -- kubectl create -f deploy/rbac.yaml -- kubectl create -f deploy/crd.yaml -- kubectl create -f deploy/operator.yaml -# Sleep until operator is ready, TODO poll api in retry loop -- sleep 10s -- kubectl get deployment -- kubectl get pods -# Create memcached cr -- | - echo "apiVersion: \"cache.example.com/v1alpha1\"" > deploy/cr.yaml -- | - echo "kind: \"Memcached\"" >> deploy/cr.yaml -- echo "metadata:" >> deploy/cr.yaml -- | - echo " name: \"example-memcached\"" >> deploy/cr.yaml -- echo "spec:" >> deploy/cr.yaml -- | - echo " size: 3" >> deploy/cr.yaml -- kubectl apply -f deploy/cr.yaml -# Sleep until 3 pods are up, TODO poll api in retry loop -- sleep 10s -- kubectl get deployment -- kubectl get pods -- kubectl get memcached/example-memcached -o yaml -# Expand to size to 4 -- | - sed -ie 's/size: 3/size: 4/g' deploy/cr.yaml -- kubectl apply -f deploy/cr.yaml -# Sleep until 4 pods are up, TODO poll api in retry loop -- sleep 10s -- kubectl get deployment -- kubectl get pods -- kubectl get memcached/example-memcached -o yaml -# Cleanup -- kubectl delete -f deploy/cr.yaml -- kubectl delete -f deploy/operator.yaml +- go test ./pkg/... +- go vet ./... +- go test ./test/e2e/... after_success: - echo 'Build succeeded, operator was generated, memcached operator is running on minikube, and unit/integration tests pass' diff --git a/Gopkg.lock b/Gopkg.lock index d65a1d88907..c43a3defb1f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -357,6 +357,18 @@ revision = "acf347b865f29325eb61f4cd2df11e86e073a5ee" version = "kubernetes-1.9.3" +[[projects]] + name = "k8s.io/apiextensions-apiserver" + packages = [ + "pkg/apis/apiextensions", + "pkg/apis/apiextensions/v1beta1", + "pkg/client/clientset/clientset", + "pkg/client/clientset/clientset/scheme", + "pkg/client/clientset/clientset/typed/apiextensions/v1beta1" + ] + revision = "f1425805c0335c1f5b23e70f0154747b3df5a16a" + version = "kubernetes-1.9.3" + [[projects]] name = "k8s.io/apimachinery" packages = [ @@ -470,6 +482,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "e39a3b50eecf50ee2f3c6ce8a36306abeea762a41fab1117f0c5e2a038b72fb4" + inputs-digest = "d3740f4348ec1b11ffbcb71d7323496f157bc3d15ee151a8ea43b52c04351c24" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 700254fcd47..b785868cd93 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -14,6 +14,10 @@ name = "k8s.io/apimachinery" version = "kubernetes-1.9.3" +[[override]] + name = "k8s.io/apiextensions-apiserver" + version = "kubernetes-1.9.3" + [[override]] name = "k8s.io/client-go" version = "kubernetes-1.9.3" diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000000..9daeafb9864 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +test diff --git a/test/e2e/e2eutil/check_util.go b/test/e2e/e2eutil/check_util.go new file mode 100644 index 00000000000..a7528001eae --- /dev/null +++ b/test/e2e/e2eutil/check_util.go @@ -0,0 +1,36 @@ +package e2eutil + +import ( + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +var retryInterval = time.Second * 5 + +func DeploymentReplicaCheck(t *testing.T, kubeclient *kubernetes.Clientset, namespace, name string, replicas, retries int) error { + err := Retry(retryInterval, retries, func() (done bool, err error) { + deployment, err := kubeclient.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{IncludeUninitialized: true}) + if err != nil { + if apierrors.IsNotFound(err) { + t.Logf("Waiting for availability of %s deployment\n", name) + return false, nil + } + return false, err + } + + if int(deployment.Status.AvailableReplicas) == replicas { + return true, nil + } + t.Logf("Waiting for full availability of %s deployment (%d/%d)\n", name, deployment.Status.AvailableReplicas, replicas) + return false, nil + }) + if err != nil { + return err + } + t.Logf("Deployment available (%d/%d)\n", replicas, replicas) + return nil +} diff --git a/test/e2e/e2eutil/create_util.go b/test/e2e/e2eutil/create_util.go new file mode 100644 index 00000000000..8dd1d3c52af --- /dev/null +++ b/test/e2e/e2eutil/create_util.go @@ -0,0 +1,111 @@ +package e2eutil + +import ( + "strings" + "testing" + + y2j "github.com/ghodss/yaml" + yaml "gopkg.in/yaml.v2" + apps "k8s.io/api/apps/v1" + "k8s.io/api/rbac/v1beta1" + crd "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + extensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + extensions_scheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" +) + +func GetCRClient(t *testing.T, config *rest.Config, yamlCR []byte) *rest.RESTClient { + // get new RESTClient for custom resources + crConfig := config + m := make(map[interface{}]interface{}) + err := yaml.Unmarshal(yamlCR, &m) + groupVersion := strings.Split(m["apiVersion"].(string), "/") + crGV := schema.GroupVersion{Group: groupVersion[0], Version: groupVersion[1]} + crConfig.GroupVersion = &crGV + crConfig.APIPath = "/apis" + crConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} + + if crConfig.UserAgent == "" { + crConfig.UserAgent = rest.DefaultKubernetesUserAgent() + } + crRESTClient, err := rest.RESTClientFor(crConfig) + if err != nil { + t.Fatal(err) + } + return crRESTClient +} + +func createCRFromYAML(t *testing.T, yamlFile []byte, kubeconfig *rest.Config, namespace, resourceName string) error { + client := GetCRClient(t, kubeconfig, yamlFile) + jsonDat, err := y2j.YAMLToJSON(yamlFile) + err = client.Post(). + Namespace(namespace). + Resource(resourceName). + Body(jsonDat). + Do(). + Error() + return err +} + +func createCRDFromYAML(t *testing.T, yamlFile []byte, extensionsClient *extensions.Clientset) error { + decode := extensions_scheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode(yamlFile, nil, nil) + + if err != nil { + t.Log("Failed to deserialize CustomResourceDefinition") + t.Fatal(err) + } + switch o := obj.(type) { + case *crd.CustomResourceDefinition: + _, err = extensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(o) + return err + } + return nil +} + +func CreateFromYAML(t *testing.T, yamlFile []byte, kubeclient *kubernetes.Clientset, kubeconfig *rest.Config, namespace string) error { + m := make(map[interface{}]interface{}) + err := yaml.Unmarshal(yamlFile, &m) + kind := m["kind"].(string) + switch kind { + case "Role": + case "RoleBinding": + case "Deployment": + case "CustomResourceDefinition": + extensionclient, err := extensions.NewForConfig(kubeconfig) + if err != nil { + t.Fatal(err) + } + return createCRDFromYAML(t, yamlFile, extensionclient) + // we assume that all custom resources are from operator-sdk and thus follow + // a common naming convention + default: + return createCRFromYAML(t, yamlFile, kubeconfig, namespace, strings.ToLower(kind)+"s") + } + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode(yamlFile, nil, nil) + + if err != nil { + t.Log("Unable to deserialize resource; is it a custom resource?") + t.Fatal(err) + } + + switch o := obj.(type) { + case *v1beta1.Role: + _, err = kubeclient.RbacV1beta1().Roles(namespace).Create(o) + return err + case *v1beta1.RoleBinding: + _, err = kubeclient.RbacV1beta1().RoleBindings(namespace).Create(o) + return err + case *apps.Deployment: + _, err = kubeclient.AppsV1().Deployments(namespace).Create(o) + return err + default: + t.Fatalf("unknown type: %s", o) + } + return nil +} diff --git a/test/e2e/e2eutil/retry_util.go b/test/e2e/e2eutil/retry_util.go new file mode 100644 index 00000000000..fafa3719806 --- /dev/null +++ b/test/e2e/e2eutil/retry_util.go @@ -0,0 +1,62 @@ +// Copyright 2018 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2eutil + +import ( + "fmt" + "time" +) + +type RetryError struct { + n int +} + +func (e *RetryError) Error() string { + return fmt.Sprintf("still failing after %d retries", e.n) +} + +func IsRetryFailure(err error) bool { + _, ok := err.(*RetryError) + return ok +} + +type ConditionFunc func() (bool, error) + +// Retry retries f every interval until after maxRetries. +// The interval won't be affected by how long f takes. +// For example, if interval is 3s, f takes 1s, another f will be called 2s later. +// However, if f takes longer than interval, it will be delayed. +func Retry(interval time.Duration, maxRetries int, f ConditionFunc) error { + if maxRetries <= 0 { + return fmt.Errorf("maxRetries (%d) should be > 0", maxRetries) + } + tick := time.NewTicker(interval) + defer tick.Stop() + + for i := 0; ; i++ { + ok, err := f() + if err != nil { + return err + } + if ok { + return nil + } + if i == maxRetries { + break + } + <-tick.C + } + return &RetryError{maxRetries} +} diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go new file mode 100644 index 00000000000..116c48a070d --- /dev/null +++ b/test/e2e/main_test.go @@ -0,0 +1,10 @@ +package e2e + +import ( + "testing" +) + +func TestMain(m *testing.M) { + // TODO: create a setup step for the framework here + m.Run() +} diff --git a/test/e2e/memcached_test.go b/test/e2e/memcached_test.go new file mode 100644 index 00000000000..06700650101 --- /dev/null +++ b/test/e2e/memcached_test.go @@ -0,0 +1,226 @@ +package e2e + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/operator-framework/operator-sdk/test/e2e/e2eutil" + core "k8s.io/api/core/v1" + extensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +func TestMemcached(t *testing.T) { + os.Chdir(os.Getenv("GOPATH") + "/src/github.com/example-inc") + t.Log("Creating new operator project") + cmdOut, err := exec.Command("operator-sdk", + "new", + "memcached-operator", + "--api-version=cache.example.com/v1alpha1", + "--kind=Memcached").CombinedOutput() + if err != nil { + t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) + } + os.Chdir("memcached-operator") + os.RemoveAll("vendor/github.com/operator-framework/operator-sdk/pkg") + os.Symlink(os.Getenv("TRAVIS_BUILD_DIR")+"/pkg", "vendor/github.com/operator-framework/operator-sdk/pkg") + handler, err := os.Create("pkg/stub/handler.go") + if err != nil { + t.Fatal(err) + } + defer handler.Close() + resp, err := http.Get("https://raw.githubusercontent.com/operator-framework/operator-sdk/master/example/memcached-operator/handler.go.tmpl") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + _, err = io.Copy(handler, resp.Body) + if err != nil { + t.Fatal(err) + } + gotypes, err := ioutil.ReadFile("pkg/apis/cache/v1alpha1/types.go") + if err != nil { + t.Fatal(err) + } + lines := bytes.Split(gotypes, []byte("\n")) + lines = lines[:len(lines)-7] + lines = append(lines, []byte("type MemcachedSpec struct { Size int32 `json:\"size\"`}")) + lines = append(lines, []byte("type MemcachedStatus struct {Nodes []string `json:\"nodes\"`}\n")) + os.Remove("pkg/apis/cache/v1alpha1/types.go") + err = ioutil.WriteFile("pkg/apis/cache/v1alpha1/types.go", bytes.Join(lines, []byte("\n")), os.FileMode(int(0664))) + if err != nil { + t.Fatal(err) + } + + t.Log("Generating k8s") + cmdOut, err = exec.Command("operator-sdk", + "generate", + "k8s").CombinedOutput() + if err != nil { + t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) + } + + t.Log("Building operator docker image") + cmdOut, err = exec.Command("operator-sdk", + "build", + "quay.io/example/memcached-operator:v0.0.1").CombinedOutput() + if err != nil { + t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) + } + + opYAML, err := ioutil.ReadFile("deploy/operator.yaml") + if err != nil { + t.Fatal(err) + } + opYAML = bytes.Replace(opYAML, []byte("imagePullPolicy: Always"), []byte("imagePullPolicy: Never"), 1) + err = ioutil.WriteFile("deploy/operator.yaml", opYAML, os.FileMode(int(0664))) + if err != nil { + t.Fatal(err) + } + + namespace := "memcached" + kubeconfigPath := filepath.Join( + os.Getenv("HOME"), ".kube", "config", + ) + kubeconfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + t.Fatal(err) + } + kubeclient, err := kubernetes.NewForConfig(kubeconfig) + if err != nil { + t.Fatal(err) + } + + // create namespace + namespaceObj := &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + _, err = kubeclient.CoreV1().Namespaces().Create(namespaceObj) + if err != nil { + t.Fatal(err) + } + + // create rbac + dat, err := ioutil.ReadFile("deploy/rbac.yaml") + splitDat := bytes.Split(dat, []byte("\n---\n")) + for _, thing := range splitDat { + err = e2eutil.CreateFromYAML(t, thing, kubeclient, kubeconfig, namespace) + if err != nil { + t.Fatal(err) + } + } + t.Log("Created rbac") + + // create crd + yamlCRD, err := ioutil.ReadFile("deploy/crd.yaml") + err = e2eutil.CreateFromYAML(t, yamlCRD, kubeclient, kubeconfig, namespace) + if err != nil { + t.Fatal(err) + } + t.Log("Created crd") + + // create operator + dat, err = ioutil.ReadFile("deploy/operator.yaml") + err = e2eutil.CreateFromYAML(t, dat, kubeclient, kubeconfig, namespace) + if err != nil { + t.Fatal(err) + } + t.Log("Created operator") + + err = e2eutil.DeploymentReplicaCheck(t, kubeclient, namespace, "memcached-operator", 1, 6) + if err != nil { + t.Fatal(err) + } + + // create example-memcached yaml file + file, err := os.OpenFile("deploy/cr.yaml", os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + t.Fatal(err) + } + defer file.Close() + + _, err = file.WriteString("apiVersion: \"cache.example.com/v1alpha1\"\nkind: \"Memcached\"\nmetadata:\n name: \"example-memcached\"\nspec:\n size: 3") + if err != nil { + t.Fatal(err) + } + + file.Close() + + yamlCR, err := ioutil.ReadFile("deploy/cr.yaml") + memcachedClient := e2eutil.GetCRClient(t, kubeconfig, yamlCR) + e2eutil.CreateFromYAML(t, yamlCR, kubeclient, kubeconfig, namespace) + + err = e2eutil.DeploymentReplicaCheck(t, kubeclient, namespace, "example-memcached", 3, 6) + if err != nil { + t.Fatal(err) + } + + // update CR size to 4 + err = memcachedClient.Patch(types.JSONPatchType). + Namespace(namespace). + Resource("memcacheds"). + Name("example-memcached"). + Body([]byte("[{\"op\": \"replace\", \"path\": \"/spec/size\", \"value\": 4}]")). + Do(). + Error() + if err != nil { + t.Fatal(err) + } + + err = e2eutil.DeploymentReplicaCheck(t, kubeclient, namespace, "example-memcached", 4, 6) + if err != nil { + t.Fatal(err) + } + + // clean everything up + err = memcachedClient.Delete(). + Namespace(namespace). + Resource("memcacheds"). + Name("example-memcached"). + Body([]byte("{\"propagationPolicy\":\"Foreground\"}")). + Do(). + Error() + if err != nil { + t.Log("Failed to delete example-memcached CR") + t.Fatal(err) + } + err = kubeclient.AppsV1().Deployments(namespace). + Delete("memcached-operator", metav1.NewDeleteOptions(0)) + if err != nil { + t.Log("Failed to delete memcached-operator deployment") + t.Fatal(err) + } + err = kubeclient.RbacV1beta1().Roles(namespace).Delete("memcached-operator", metav1.NewDeleteOptions(0)) + if err != nil { + t.Log("Failed to delete memcached-operator Role") + t.Fatal(err) + } + err = kubeclient.RbacV1beta1().RoleBindings(namespace).Delete("default-account-memcached-operator", metav1.NewDeleteOptions(0)) + if err != nil { + t.Log("Failed to delete memcached-operator RoleBinding") + t.Fatal(err) + } + extensionclient, err := extensions.NewForConfig(kubeconfig) + if err != nil { + t.Fatal(err) + } + err = extensionclient.ApiextensionsV1beta1().CustomResourceDefinitions().Delete("memcacheds.cache.example.com", metav1.NewDeleteOptions(0)) + if err != nil { + t.Log("Failed to delete memcached CRD") + t.Fatal(err) + } + err = kubeclient.CoreV1().Namespaces().Delete(namespace, metav1.NewDeleteOptions(0)) + if err != nil { + t.Log("Failed to delete memcached namespace") + t.Fatal(err) + } + + os.RemoveAll(os.Getenv("GOPATH") + "/src/github.com/example-inc/memcached-operator") +}