Decoding issue for complex type

[valentinwidmer]

I'm receiving a strange error message while trying to parse the following HCL structure:

user_namespaces_object = {
  ns1 = {
    accountid    = 46464
    assigned_psp = "default"
    git_token_secret_name = ""
    git_repository        = ""
    flux_custom_path = ""
    blackhole_port_fis       = ""
    extra_labels            = {}
  }
  ns2 = {
    accountid    = 46464
    assigned_psp = "default"
    git_token_secret_name = ""
    git_repository        = ""
    flux_custom_path = ""
    blackhole_port_fis       = ""
    extra_labels            = {}
  }
  ns3 = {
    accountid    = 46464
    assigned_psp = "default"
    git_token_secret_name = ""
    git_repository        = ""
    flux_custom_path = ""
    blackhole_port_fis       = ""
    extra_labels            = {}
  }
}

I would like to convert this HCL structure into a Go struct:

type Root struct {
	UserNamespacesObject []Namespace `hcl:"user_namespaces_object"`
}

type Namespace struct {
	AccountID                       int                               `hcl:"accountid" json:"accountid"`
	AssignedPSP                     string                            `hcl:"assigned_psp" json:"assigned_psp"`
	GitTokenSecretName              string                            `hcl:"git_token_secret_name" json:"git_token_secret_name"`
	GitRepository                   string                            `hcl:"git_repository" json:"git_repository"`
	FluxCustomPath                  string                            `hcl:"flux_custom_path" json:"flux_custom_path"`
	BlackholePortFIS                string                            `hcl:"blackhole_port_fis" json:"blackhole_port_fis"`
	ExtraLabels                     map[string]string                 `hcl:"extra_labels" json:"extra_labels"`
}

func handler(request json.RawMessage) (events.APIGatewayProxyResponse, error) {

	parser := hclparse.NewParser()
	file, diagnostics := parser.ParseHCL([]byte(input), "test")

	if diagnostics.HasErrors() {
		panic(diagnostics)
	}

	var root Root
	diags := gohcl.DecodeBody(file.Body, nil, &root)
	if diags.HasErrors() {
		return events.APIGatewayProxyResponse{
			Body:       diags.Error(),
			StatusCode: 400,
		}, err
	}

	fmt.Println(root)
}

While running the code I receive the following error message:

"errorMessage": "unsuitable DecodeExpression target: no cty.Type for main.Namespace (no cty field tags)", "errorType": "string", "stackTrace": [{"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/errors.go", "line": 39, "label": "lambdaPanicResponse"}, {"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/invoke_loop.go", "line": 102, "label": "callBytesHandlerFunc.func1"}, {"path": "runtime/panic.go", "line": 785, "label": "gopanic"}, {"path": "github.com/hashicorp/hcl/v2@v2.23.0/gohcl/decode.go", "line": 296, "label": "DecodeExpression"}, {"path": "github.com/hashicorp/hcl/v2@v2.23.0/gohcl/decode.go", "line": 127, "label": "decodeBodyToStruct"}, {"path": "github.com/hashicorp/hcl/v2@v2.23.0/gohcl/decode.go", "line": 46, "label": "decodeBodyToValue"}, {"path": "github.com/hashicorp/hcl/v2@v2.23.0/gohcl/decode.go", "line": 39, "label": "DecodeBody"}, {"path": "hello-world/main.go", "line": 63, "label": "handler"}, {"path": "reflect/value.go", "line": 581, "label": "Value.call"}, {"path": "reflect/value.go", "line": 365, "label": "Value.Call"}, {"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/handler.go", "line": 253, "label": "reflectHandler.func1"}, {"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/handler.go", "line": 194, "label": "bytesHandlerFunc.Invoke"}, {"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/invoke_loop.go", "line": 105, "label": "callBytesHandlerFunc"}, {"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/invoke_loop.go", "line": 73, "label": "handleInvoke"}, {"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/invoke_loop.go", "line": 37, "label": "startRuntimeAPILoop"}, {"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/entry.go", "line": 103, "label": "start"}, {"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/entry.go", "line": 66, "label": "StartWithOptions"}, {"path": "github.com/aws/aws-lambda-go@v1.36.1/lambda/entry.go", "line": 42, "label": "Start"}, {"path": "hello-world/main.go", "line": 157, "label": "main"}, {"path": "runtime/proc.go", "line": 272, "label": "main"}, {"path": "runtime/asm_amd64.s", "line": 1700, "label": "goexit"}]}

[apparentlymart]

Hi @valentinwidmer,

HCL's own decoder only deals with HCL blocks and arguments, before delegating to an upstream library called "cty" to deal with the values of those arguments.

In your example you only have a single HCL argument called user_namespaces_object, and so HCL's struct tags apply only to that one field.

For dealing with the result of this expression -- which is a cty.Value from the upstream library I mentioned -- your Namespace field would first need to be a map from string to Namespace rather than a slice (to match what you're assigning to it) and then your Namespace type would need to use the struct tags expected by gocty.FromCtyValue, since that's what HCL is calling internally.

For the relatively simple fields in your Namespace type I think that just means swapping the hcl: prefix for the cty: prefix, since gocty will infer everything else based on the field types.

Your ExtraLabels field might suffer from zclconf/go-cty#17 . I've not prioritized fixing that because in practice there aren't many applications using the gocty package rather than using cty's main API directly, but I expect I could fix it if it were a blocker for you here. (The symptom of this problem I would expect for you is that it would complain that the field is supposed to be a map but you've assigned an object to it.)

[valentinwidmer]

Thanks for the explanation @apparentlymart !

Actually the extra label field contains an object itself. Further there are more complex structures nested further down (map which contains map of objects) that I did not add as an example.

Overall I just want to append a new namespace to this map. Are you aware of another library/solution which could solve my problem?

[genelet]

Hi, valentinwidmer:

Two comments:

1. you might meant to have

type Root struct {
	UserNamespacesObject map[string]*Namespace `hcl:"user_namespaces_object"`
}

because of the keys "ns1", "ns2" and "ns3" in your data.

2. if continue -> contain #1 is correct, you should express the map as the following:

  user_namespaces_object ns1 {
    accountid    = 46464
    assigned_psp = "default"
    git_token_secret_name = ""
    git_repository        = ""
    flux_custom_path = ""
    blackhole_port_fis       = ""
    extra_labels            = {}
  }
  user_namespaces_object ns2  {
    accountid    = 46464
    assigned_psp = "default"
    git_token_secret_name = ""
    git_repository        = ""
    flux_custom_path = ""
    blackhole_port_fis       = ""
    extra_labels            = {}
  }
  user_namespaces_object ns3  {
    accountid    = 46464
    assigned_psp = "default"
    git_token_secret_name = ""
    git_repository        = ""
    flux_custom_path = ""
    blackhole_port_fis       = ""
    extra_labels            = {}
  }

then you can parse using "hcl/hclsyntax", instead of "hcl/gohcl" which is a high-level parser based on hclsyntax but may be too simplified to match your use cases.

However, "hclsyntax" returns low-level data structures, called "attributes" and "blocks". Here is another high-level parser based on "hclsyntax", "github.com/genelet/determined/dethcl":

package main

import (
      "log"
      "github.com/genelet/determined/dethcl"
)

type Root struct {
	UserNamespacesObject map[string]*Namespace `hcl:"user_namespaces_object"`
}

type Namespace struct {
	AccountID          int               `hcl:"accountid" json:"accountid"`
	AssignedPSP        string            `hcl:"assigned_psp" json:"assigned_psp"`
	GitTokenSecretName string            `hcl:"git_token_secret_name" json:"git_token_secret_name"`
	GitRepository      string            `hcl:"git_repository" json:"git_repository"`
	FluxCustomPath     string            `hcl:"flux_custom_path" json:"flux_custom_path"`
	BlackholePortFIS   string            `hcl:"blackhole_port_fis" json:"blackhole_port_fis"`
	ExtraLabels        map[string]string `hcl:"extra_labels" json:"extra_labels"`
}

func main() {
	bs := []byte(`
user_namespaces_object ns1 {
    accountid    = 46464
    assigned_psp = "default"
    git_token_secret_name = ""
    git_repository        = ""
    flux_custom_path = ""
    blackhole_port_fis       = ""
    extra_labels            = {}
  }
  user_namespaces_object ns2  {
    accountid    = 46464
    assigned_psp = "default"
    git_token_secret_name = ""
    git_repository        = ""
    flux_custom_path = ""
    blackhole_port_fis       = ""
    extra_labels            = {}
  }
  user_namespaces_object ns3  {
    accountid    = 46464
    assigned_psp = "default"
    git_token_secret_name = ""
    git_repository        = ""
    flux_custom_path = ""
    blackhole_port_fis       = ""
    extra_labels            = {}
  }
  `)

    report := &Root{}
    err := dethcl.Unmarshal(bs, report)
    if err != nil {
        panic(err)
    }

    log.Printf("%#v", report.UserNamespacesObject["ns1"])
    log.Printf("%#v", report.UserNamespacesObject["ns2"])
    log.Printf("%#v", report.UserNamespacesObject["ns3"])
}

it will work.