json-output: Add output type to JSON format

alisdair · alisdair · commit 80e87d066c12 · 2022-04-27T13:20:03.000-04:00
Previously the supported JSON plan and state formats included only
serialized output values, which was a lossy serialization of the
Terraform type system. This commit adds a type field in the usual cty
JSON format, which allows reconstitution of the original value.

For example, previously a list(string) and a set(string) containing the
same values were indistinguishable. This change serializes these as
follows:

{
  "value": ["a","b","c"],
  "type": ["list","string"]
}

and:

{
  "value": ["a","b","c"],
  "type": ["set","string"]
}
diff --git a/internal/command/jsonplan/plan.go b/internal/command/jsonplan/plan.go
@@ -106,6 +106,7 @@ type change struct {
 
 type output struct {
 	Sensitive bool            `json:"sensitive"`
+	Type      json.RawMessage `json:"type,omitempty"`
 	Value     json.RawMessage `json:"value,omitempty"`
 }
 
diff --git a/internal/command/jsonplan/values.go b/internal/command/jsonplan/values.go
@@ -57,7 +57,7 @@ func marshalPlannedOutputs(changes *plans.Changes) (map[string]output, error) {
 			continue
 		}
 
-		var after []byte
+		var after, afterType []byte
 		changeV, err := oc.Decode()
 		if err != nil {
 			return ret, err
@@ -68,14 +68,20 @@ func marshalPlannedOutputs(changes *plans.Changes) (map[string]output, error) {
 		changeV.After, _ = changeV.After.UnmarkDeep()
 
 		if changeV.After != cty.NilVal && changeV.After.IsWhollyKnown() {
-			after, err = ctyjson.Marshal(changeV.After, changeV.After.Type())
+			ty := changeV.After.Type()
+			after, err = ctyjson.Marshal(changeV.After, ty)
+			if err != nil {
+				return ret, err
+			}
+			afterType, err = ctyjson.MarshalType(ty)
 			if err != nil {
 				return ret, err
 			}
 		}
 
 		ret[oc.Addr.OutputValue.Name] = output{
 			Value:     json.RawMessage(after),
+			Type:      json.RawMessage(afterType),
 			Sensitive: oc.Sensitive,
 		}
 	}
diff --git a/internal/command/jsonplan/values_test.go b/internal/command/jsonplan/values_test.go
@@ -137,6 +137,7 @@ func TestMarshalPlannedOutputs(t *testing.T) {
 			map[string]output{
 				"bar": {
 					Sensitive: false,
+					Type:      json.RawMessage(`"string"`),
 					Value:     json.RawMessage(`"after"`),
 				},
 			},
diff --git a/internal/command/jsonstate/state.go b/internal/command/jsonstate/state.go
@@ -38,6 +38,7 @@ type stateValues struct {
 type output struct {
 	Sensitive bool            `json:"sensitive"`
 	Value     json.RawMessage `json:"value,omitempty"`
+	Type      json.RawMessage `json:"type,omitempty"`
 }
 
 // module is the representation of a module in state. This can be the root module
@@ -180,12 +181,18 @@ func marshalOutputs(outputs map[string]*states.OutputValue) (map[string]output,
 
 	ret := make(map[string]output)
 	for k, v := range outputs {
-		ov, err := ctyjson.Marshal(v.Value, v.Value.Type())
+		ty := v.Value.Type()
+		ov, err := ctyjson.Marshal(v.Value, ty)
+		if err != nil {
+			return ret, err
+		}
+		ot, err := ctyjson.MarshalType(ty)
 		if err != nil {
 			return ret, err
 		}
 		ret[k] = output{
 			Value:     ov,
+			Type:      ot,
 			Sensitive: v.Sensitive,
 		}
 	}
diff --git a/internal/command/jsonstate/state_test.go b/internal/command/jsonstate/state_test.go
@@ -36,6 +36,7 @@ func TestMarshalOutputs(t *testing.T) {
 				"test": {
 					Sensitive: true,
 					Value:     json.RawMessage(`"sekret"`),
+					Type:      json.RawMessage(`"string"`),
 				},
 			},
 			false,
@@ -51,6 +52,39 @@ func TestMarshalOutputs(t *testing.T) {
 				"test": {
 					Sensitive: false,
 					Value:     json.RawMessage(`"not_so_sekret"`),
+					Type:      json.RawMessage(`"string"`),
+				},
+			},
+			false,
+		},
+		{
+			map[string]*states.OutputValue{
+				"mapstring": {
+					Sensitive: false,
+					Value: cty.MapVal(map[string]cty.Value{
+						"beep": cty.StringVal("boop"),
+					}),
+				},
+				"setnumber": {
+					Sensitive: false,
+					Value: cty.SetVal([]cty.Value{
+						cty.NumberIntVal(3),
+						cty.NumberIntVal(5),
+						cty.NumberIntVal(7),
+						cty.NumberIntVal(11),
+					}),
+				},
+			},
+			map[string]output{
+				"mapstring": {
+					Sensitive: false,
+					Value:     json.RawMessage(`{"beep":"boop"}`),
+					Type:      json.RawMessage(`["map","string"]`),
+				},
+				"setnumber": {
+					Sensitive: false,
+					Value:     json.RawMessage(`[3,5,7,11]`),
+					Type:      json.RawMessage(`["set","number"]`),
 				},
 			},
 			false,
@@ -67,10 +101,8 @@ func TestMarshalOutputs(t *testing.T) {
 		} else if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		eq := reflect.DeepEqual(got, test.Want)
-		if !eq {
-			// printing the output isn't terribly useful, but it does help indicate which test case failed
-			t.Fatalf("wrong result:\nGot: %#v\nWant: %#v\n", got, test.Want)
+		if !cmp.Equal(test.Want, got) {
+			t.Fatalf("wrong result:\n%s", cmp.Diff(test.Want, got))
 		}
 	}
 }
diff --git a/internal/command/testdata/show-json-sensitive/output.json b/internal/command/testdata/show-json-sensitive/output.json
@@ -9,6 +9,7 @@
         "outputs": {
             "test": {
                 "sensitive": true,
+                "type": "string",
                 "value": "bar"
             }
         },
@@ -71,6 +72,7 @@
             "outputs": {
                 "test": {
                     "sensitive": true,
+                    "type": "string",
                     "value": "bar"
                 }
             },
diff --git a/internal/command/testdata/show-json-state/modules/output.json b/internal/command/testdata/show-json-state/modules/output.json
@@ -5,6 +5,7 @@
         "outputs": {
             "test": {
                 "sensitive": false,
+                "type": "string",
                 "value": "baz"
             }
         },
diff --git a/internal/command/testdata/show-json/basic-create/output.json b/internal/command/testdata/show-json/basic-create/output.json
@@ -9,6 +9,7 @@
         "outputs": {
             "test": {
                 "sensitive": false,
+                "type": "string",
                 "value": "bar"
             }
         },
@@ -62,6 +63,7 @@
             "outputs": {
                 "test": {
                     "sensitive": false,
+                    "type": "string",
                     "value": "bar"
                 }
             },
diff --git a/internal/command/testdata/show-json/basic-delete/output.json b/internal/command/testdata/show-json/basic-delete/output.json
@@ -9,6 +9,7 @@
         "outputs": {
             "test": {
                 "sensitive": false,
+                "type": "string",
                 "value": "bar"
             }
         },
@@ -94,6 +95,7 @@
             "outputs": {
                 "test": {
                     "sensitive": false,
+                    "type": "string",
                     "value": "bar"
                 }
             },
diff --git a/internal/command/testdata/show-json/basic-update/output.json b/internal/command/testdata/show-json/basic-update/output.json
@@ -9,6 +9,7 @@
         "outputs": {
             "test": {
                 "sensitive": false,
+                "type": "string",
                 "value": "bar"
             }
         },
@@ -73,6 +74,7 @@
             "outputs": {
                 "test": {
                     "sensitive": false,
+                    "type": "string",
                     "value": "bar"
                 }
             },
diff --git a/internal/command/testdata/show-json/conditions/output-refresh-only.json b/internal/command/testdata/show-json/conditions/output-refresh-only.json
@@ -13,6 +13,7 @@
     "outputs": {
       "foo_id": {
         "sensitive": false,
+        "type": "string",
         "value": "placeholder"
       }
     },
@@ -37,6 +38,7 @@
       "outputs": {
         "foo_id": {
           "sensitive": false,
+          "type": "string",
           "value": "placeholder"
         }
       },
diff --git a/internal/command/testdata/show-json/modules/output.json b/internal/command/testdata/show-json/modules/output.json
@@ -4,6 +4,7 @@
         "outputs": {
             "test": {
                 "sensitive": false,
+                "type": "string",
                 "value": "baz"
             }
         },
@@ -79,6 +80,7 @@
             "outputs": {
                 "test": {
                     "sensitive": false,
+                    "type": "string",
                     "value": "baz"
                 }
             },
diff --git a/internal/command/testdata/show-json/multi-resource-update/output.json b/internal/command/testdata/show-json/multi-resource-update/output.json
@@ -10,6 +10,7 @@
         "outputs": {
             "test": {
                 "sensitive": false,
+                "type": "string",
                 "value": "bar"
             }
         },
@@ -113,6 +114,7 @@
             "outputs": {
                 "test": {
                     "sensitive": false,
+                    "type": "string",
                     "value": "bar"
                 }
             },
diff --git a/internal/command/testdata/show-json/provider-version-no-config/output.json b/internal/command/testdata/show-json/provider-version-no-config/output.json
@@ -9,6 +9,7 @@
         "outputs": {
             "test": {
                 "sensitive": false,
+                "type": "string",
                 "value": "bar"
             }
         },
@@ -62,6 +63,7 @@
             "outputs": {
                 "test": {
                     "sensitive": false,
+                    "type": "string",
                     "value": "bar"
                 }
             },
diff --git a/internal/command/testdata/show-json/provider-version/output.json b/internal/command/testdata/show-json/provider-version/output.json
@@ -9,6 +9,7 @@
         "outputs": {
             "test": {
                 "sensitive": false,
+                "type": "string",
                 "value": "bar"
             }
         },
@@ -62,6 +63,7 @@
             "outputs": {
                 "test": {
                     "sensitive": false,
+                    "type": "string",
                     "value": "bar"
                 }
             },
diff --git a/internal/command/testdata/show-json/sensitive-values/output.json b/internal/command/testdata/show-json/sensitive-values/output.json
@@ -9,6 +9,7 @@
         "outputs": {
             "test": {
                 "sensitive": true,
+                "type": "string",
                 "value": "boop"
             }
         },
@@ -74,6 +75,7 @@
             "outputs": {
                 "test": {
                     "sensitive": true,
+                    "type": "string",
                     "value": "boop"
                 }
             },
diff --git a/website/docs/internals/json-format.mdx b/website/docs/internals/json-format.mdx
@@ -217,6 +217,7 @@ The following example illustrates the structure of a `<values-representation>`:
   "outputs": {
     "private_ip": {
       "value": "192.168.3.2",
+      "type": "string",
       "sensitive": false
     }
   },
@@ -307,6 +308,8 @@ The following example illustrates the structure of a `<values-representation>`:
 
 The translation of attribute and output values is the same intuitive mapping from HCL types to JSON types used by Terraform's [`jsonencode`](/language/functions/jsonencode) function. This mapping does lose some information: lists, sets, and tuples all lower to JSON arrays while maps and objects both lower to JSON objects. Unknown values and null values are both treated as absent or null.
 
+Output values include a `"type"` field, which is a [serialization of the value's type](https://pkg.go.dev/github.com/zclconf/go-cty/cty#Type.MarshalJSON). For primitive types this is a string value, such as `"number"` or `"bool"`. Complex types are represented as a nested JSON array, such as `["map","string"]` or `["object",{"a":"number"]]`. This can be used to reconstruct the output value with the correct type.
+
 Only the "current" object for each resource instance is described. "Deposed" objects are not reflected in this structure at all; in plan representations, you can refer to the change representations for further details.
 
 The intent of this structure is to give a caller access to a similar level of detail as is available to expressions within the configuration itself. This common representation is not suitable for all use-cases because it loses information compared to the data structures it is built from. For more complex needs, use the more elaborate changes and configuration representations.

Original file line number	Diff line number	Diff line change
`@@ -106,6 +106,7 @@ type change struct {`
`106`	`106`
`107`	`107`	`type output struct {`
`108`	`108`	Sensitive bool `json:"sensitive"`
	`109`	+ Type json.RawMessage `json:"type,omitempty"`
`109`	`110`	Value json.RawMessage `json:"value,omitempty"`
`110`	`111`	`}`
`111`	`112`
Original file line number	Diff line number	Diff line change
`@@ -57,7 +57,7 @@ func marshalPlannedOutputs(changes *plans.Changes) (map[string]output, error) {`
`57`	`57`	`continue`
`58`	`58`	`}`
`59`	`59`
`60`		`- var after []byte`
	`60`	`+ var after, afterType []byte`
`61`	`61`	`changeV, err := oc.Decode()`
`62`	`62`	`if err != nil {`
`63`	`63`	`return ret, err`
`@@ -68,14 +68,20 @@ func marshalPlannedOutputs(changes *plans.Changes) (map[string]output, error) {`
`68`	`68`	`changeV.After, _ = changeV.After.UnmarkDeep()`
`69`	`69`
`70`	`70`	`if changeV.After != cty.NilVal && changeV.After.IsWhollyKnown() {`
`71`		`- after, err = ctyjson.Marshal(changeV.After, changeV.After.Type())`
	`71`	`+ ty := changeV.After.Type()`
	`72`	`+ after, err = ctyjson.Marshal(changeV.After, ty)`
	`73`	`+ if err != nil {`
	`74`	`+ return ret, err`
	`75`	`+ }`
	`76`	`+ afterType, err = ctyjson.MarshalType(ty)`
`72`	`77`	`if err != nil {`
`73`	`78`	`return ret, err`
`74`	`79`	`}`
`75`	`80`	`}`
`76`	`81`
`77`	`82`	`ret[oc.Addr.OutputValue.Name] = output{`
`78`	`83`	`Value: json.RawMessage(after),`
	`84`	`+ Type: json.RawMessage(afterType),`
`79`	`85`	`Sensitive: oc.Sensitive,`
`80`	`86`	`}`
`81`	`87`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@`
`5`	`5`	`"outputs": {`
`6`	`6`	`"test": {`
`7`	`7`	`"sensitive": false,`
	`8`	`+ "type": "string",`
`8`	`9`	`"value": "baz"`
`9`	`10`	`}`
`10`	`11`	`},`