Merge pull request #48 from yangcao77/testOverrideFn-1

use api override and merging utils to get flatterned devfile

Author: yangcao77

devfile.yaml

@@ -4,13 +4,24 @@ metadata:
   version: 1.0.0
   attributes:
     alpha.build-dockerfile: /relative/path/to/Dockerfile
+parent:
+  uri: https://raw.githubusercontent.com/odo-devfiles/registry/master/devfiles/nodejs/devfile.yaml
+  commands:
+  - id: install
+    exec:
+      component: runtime
+      commandLine: npm install
+      workingDir: /project-starter
+      group:
+        kind: build
+        isDefault: true
 starterProjects:
-- name: nodejs-starter
+- name: nodejs-starter2
   git:
     remotes:
       origin: https://github.com/odo-devfiles/nodejs-ex.git
 components:
-- name: runtime
+- name: runtime2
   attributes:
     tool: console-import
     import:
@@ -23,7 +34,7 @@ components:
     memoryLimit: 1024Mi
     mountSources: true
     sourceMapping: /project
-- name: runtime2
+- name: runtime3
   attributes:
     tool: odo
     cli:
@@ -36,7 +47,7 @@ components:
     memoryLimit: 1024Mi
     mountSources: true
     sourceMapping: /project
-- name: runtime3
+- name: runtime4
   attributes:
     tool: workspace-operator
   container:
@@ -50,39 +61,39 @@ components:
 commands:
 - exec:
     commandLine: npm install
-    component: runtime
+    component: runtime2
     group:
       isDefault: true
       kind: build
     workingDir: /project
-  id: install
+  id: install2
   attributes:
     tool: odo
     mandatory: false
 - exec:
     commandLine: npm start
-    component: runtime
+    component: runtime2
     group:
       isDefault: true
       kind: run
     workingDir: /project
-  id: run
+  id: run2
   attributes:
     tool: odo
     mandatory: true
 - exec:
     commandLine: npm run debug
-    component: runtime
+    component: runtime2
     group:
       isDefault: true
       kind: debug
     workingDir: /project
-  id: debug
+  id: debug2
 - exec:
     commandLine: npm test
-    component: runtime
+    component: runtime2
     group:
       isDefault: true
       kind: test
     workingDir: /project
-  id: test
+  id: test2

go.mod

@@ -3,7 +3,7 @@ module github.com/devfile/library
 go 1.13
 
 require (
-	github.com/devfile/api v0.0.0-20201126204309-ec222215253e
+	github.com/devfile/api v0.0.0-20201211221100-a68230324c7e
 	github.com/fatih/color v1.7.0
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/gobwas/glob v0.2.3

go.sum

@@ -44,8 +44,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/devfile/api v0.0.0-20201126204309-ec222215253e h1:NO6mS8ktVFjkVIKZKF2JjfL3ZxXw4VAtKIZWyp9e2kQ=
-github.com/devfile/api v0.0.0-20201126204309-ec222215253e/go.mod h1:/aDiwWjDEW/fY1/Ig8umVtmneAXKZImnLvWqzMlcfrY=
+github.com/devfile/api v0.0.0-20201211221100-a68230324c7e h1:Jy3C3ul05YvL4bJpAVhFwPZD8neOJUBZy7GuCcjc8nc=
+github.com/devfile/api v0.0.0-20201211221100-a68230324c7e/go.mod h1:/aDiwWjDEW/fY1/Ig8umVtmneAXKZImnLvWqzMlcfrY=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -182,7 +182,9 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -235,6 +237,7 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE=
 github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=

main.go

@@ -11,7 +11,7 @@ import (
 )
 
 func main() {
-	devfile, err := ParseDevfile("devfile.yaml")
+	devfile, err := ParseDevfile("./devfile.yaml")
 	if err != nil {
 		fmt.Println(err)
 	} else {
@@ -21,6 +21,29 @@ func main() {
 			fmt.Printf("schema version: %s\n", d.SchemaVersion)
 		}
 
+		components, e := devfile.Data.GetComponents(common.DevfileOptions{})
+		if e != nil {
+			fmt.Printf("err: %v\n", err)
+		}
+		fmt.Printf("All component: \n")
+		for _, component := range components {
+			fmt.Printf("%s\n", component.Name)
+		}
+
+		fmt.Printf("All Exec commands: \n")
+		commands, e := devfile.Data.GetCommands(common.DevfileOptions{})
+		if e != nil {
+			fmt.Printf("err: %v\n", err)
+		}
+		for _, command := range commands {
+			if command.Exec != nil {
+				fmt.Printf("command %s is with kind: %s", command.Id, command.Exec.Group.Kind)
+				fmt.Printf("workingDir is: %s\n", command.Exec.WorkingDir)
+			}
+		}
+
+		fmt.Println("=========================================================")
+
 		compOptions := common.DevfileOptions{
 			Filter: map[string]interface{}{
 				"tool": "console-import",
@@ -30,14 +53,14 @@ func main() {
 			},
 		}
 
-		components, e := devfile.Data.GetComponents(compOptions)
+		components, e = devfile.Data.GetComponents(compOptions)
 		if e != nil {
 			fmt.Printf("err: %v\n", err)
 		}
-
+		fmt.Printf("Container components applied filter: \n")
 		for _, component := range components {
 			if component.Container != nil {
-				fmt.Printf("component container: %s\n", component.Name)
+				fmt.Printf("%s\n", component.Name)
 			}
 		}
 
@@ -47,13 +70,15 @@ func main() {
 			},
 		}
 
-		commands, e := devfile.Data.GetCommands(cmdOptions)
+		fmt.Printf("Exec commands applied filter: \n")
+		commands, e = devfile.Data.GetCommands(cmdOptions)
 		if e != nil {
 			fmt.Printf("err: %v\n", err)
 		}
 		for _, command := range commands {
 			if command.Exec != nil {
-				fmt.Printf("exec command kind: %s\n", command.Exec.Group.Kind)
+				fmt.Printf("command %s is with kind: %s", command.Id, command.Exec.Group.Kind)
+				fmt.Printf("workingDir is: %s\n", command.Exec.WorkingDir)
 			}
 		}
 

pkg/devfile/parser/context/context.go

@@ -67,7 +67,6 @@ func (d *DevfileCtx) Populate() (err error) {
 		return err
 	}
 	klog.V(4).Infof("absolute devfile path: '%s'", d.absPath)
-
 	// Read and save devfile content
 	if err := d.SetDevfileContent(); err != nil {
 		return err

pkg/devfile/parser/data/interface.go

@@ -47,6 +47,10 @@ type DevfileData interface {
 	DeleteVolume(name string) error
 	GetVolumeMountPath(name string) (string, error)
 
+	// workspace related methods
+	GetDevfileWorkspace() *v1.DevWorkspaceTemplateSpecContent
+	SetDevfileWorkspace(content v1.DevWorkspaceTemplateSpecContent)
+
 	//utils
 	GetDevfileContainerComponents(common.DevfileOptions) ([]v1.Component, error)
 	GetDevfileVolumeComponents(common.DevfileOptions) ([]v1.Component, error)

pkg/devfile/parser/data/v2/common/options.go

@@ -3,7 +3,7 @@ package common
 import (
 	"reflect"
 
-	"github.com/devfile/api/pkg/attributes"
+	apiAttributes "github.com/devfile/api/pkg/attributes"
 )
 
 // DevfileOptions provides options for Devfile operations
@@ -13,13 +13,14 @@ type DevfileOptions struct {
 }
 
 // FilterDevfileObject filters devfile attributes with the given options
-func FilterDevfileObject(attributes attributes.Attributes, options DevfileOptions) (bool, error) {
+func FilterDevfileObject(attributes apiAttributes.Attributes, options DevfileOptions) (bool, error) {
 	var err error
 	filterIn := true
 	for key, value := range options.Filter {
 		currentFilterIn := false
 		attrValue := attributes.Get(key, &err)
-		if err != nil {
+		var keyNotFoundErr = &apiAttributes.KeyNotFoundError{Key: key}
+		if err != nil && err.Error() != keyNotFoundErr.Error() {
 			return false, err
 		} else if reflect.DeepEqual(attrValue, value) {
 			currentFilterIn = true

pkg/devfile/parser/data/v2/components.go

@@ -60,36 +60,16 @@ func (d *DevfileV2) GetDevfileVolumeComponents(options common.DevfileOptions) ([
 // if a component is already defined, error out
 func (d *DevfileV2) AddComponents(components []v1.Component) error {
 
-	// different map for volume and container component as a volume and a container with same name
-	// can exist in devfile
-	containerMap := make(map[string]bool)
-	volumeMap := make(map[string]bool)
+	componentMap := make(map[string]bool)
 
 	for _, component := range d.Components {
-		if component.Volume != nil {
-			volumeMap[component.Name] = true
-		}
-		if component.Container != nil {
-			containerMap[component.Name] = true
-		}
+		componentMap[component.Name] = true
 	}
-
 	for _, component := range components {
-
-		if component.Volume != nil {
-			if _, ok := volumeMap[component.Name]; !ok {
-				d.Components = append(d.Components, component)
-			} else {
-				return &common.FieldAlreadyExistError{Name: component.Name, Field: "component"}
-			}
-		}
-
-		if component.Container != nil {
-			if _, ok := containerMap[component.Name]; !ok {
-				d.Components = append(d.Components, component)
-			} else {
-				return &common.FieldAlreadyExistError{Name: component.Name, Field: "component"}
-			}
+		if _, ok := componentMap[component.Name]; !ok {
+			d.Components = append(d.Components, component)
+		} else {
+			return &common.FieldAlreadyExistError{Name: component.Name, Field: "component"}
 		}
 	}
 	return nil

pkg/devfile/parser/data/v2/components_test.go

@@ -35,12 +35,6 @@ func TestDevfile200_AddComponent(t *testing.T) {
 				},
 			},
 			newComponents: []v1.Component{
-				{
-					Name: "component2",
-					ComponentUnion: v1.ComponentUnion{
-						Container: &v1.ContainerComponent{},
-					},
-				},
 				{
 					Name: "component3",
 					ComponentUnion: v1.ComponentUnion{

pkg/devfile/parser/data/v2/workspace.go

@@ -0,0 +1,16 @@
+package v2
+
+import (
+	v1 "github.com/devfile/api/pkg/apis/workspaces/v1alpha2"
+)
+
+// GetDevfileWorkspace returns the workspace content for the devfile
+func (d *DevfileV2) GetDevfileWorkspace() *v1.DevWorkspaceTemplateSpecContent {
+
+	return &d.DevWorkspaceTemplateSpecContent
+}
+
+// SetDevfileWorkspace sets the workspace content
+func (d *DevfileV2) SetDevfileWorkspace(content v1.DevWorkspaceTemplateSpecContent) {
+	d.DevWorkspaceTemplateSpecContent = content
+}

pkg/devfile/parser/data/v2/workspace_test.go

@@ -0,0 +1,74 @@
+package v2
+
+import (
+	"reflect"
+	"testing"
+
+	v1 "github.com/devfile/api/pkg/apis/workspaces/v1alpha2"
+)
+
+func TestDevfile200_SetDevfileWorkspace(t *testing.T) {
+
+	type args struct {
+		name string
+	}
+	tests := []struct {
+		name              string
+		workspace         v1.DevWorkspaceTemplateSpecContent
+		devfilev2         *DevfileV2
+		expectedDevfilev2 *DevfileV2
+	}{
+		{
+			name: "set workspace",
+			devfilev2: &DevfileV2{
+				v1.Devfile{},
+			},
+			workspace: v1.DevWorkspaceTemplateSpecContent{
+				Components: []v1.Component{
+					{
+						Name: "component1",
+						ComponentUnion: v1.ComponentUnion{
+							Container: &v1.ContainerComponent{},
+						},
+					},
+					{
+						Name: "component2",
+						ComponentUnion: v1.ComponentUnion{
+							Volume: &v1.VolumeComponent{},
+						},
+					},
+				},
+			},
+			expectedDevfilev2: &DevfileV2{
+				v1.Devfile{
+					DevWorkspaceTemplateSpec: v1.DevWorkspaceTemplateSpec{
+						DevWorkspaceTemplateSpecContent: v1.DevWorkspaceTemplateSpecContent{
+							Components: []v1.Component{
+								{
+									Name: "component1",
+									ComponentUnion: v1.ComponentUnion{
+										Container: &v1.ContainerComponent{},
+									},
+								},
+								{
+									Name: "component2",
+									ComponentUnion: v1.ComponentUnion{
+										Volume: &v1.VolumeComponent{},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.devfilev2.SetDevfileWorkspace(tt.workspace)
+			if !reflect.DeepEqual(tt.devfilev2, tt.expectedDevfilev2) {
+				t.Errorf("TestDevfile200_SetDevfileWorkspace() expected %v, got %v", tt.expectedDevfilev2, tt.devfilev2)
+			}
+		})
+	}
+}