jsonplan and jsonstate: include sensitive_values in state representations (#28889)

* jsonplan and jsonstate: include sensitive_values in state representations

A sensitive_values field has been added to the resource in state and planned values which is a map of all sensitive attributes with the values set to true.

It wasn't entirely clear to me if the values in state would suffice, or if we also need to consult the schema - I believe that this is sufficient for state files written since v0.15, and if that's incorrect or insufficient, I'll add in the provider schema check as well.

I also updated the documentation, and, since we've considered this before, bumped the FormatVersions for both jsonstate and jsonplan.

Author: mildwonkey

internal/command/jsonplan/plan.go

@@ -22,7 +22,7 @@ import (
 // FormatVersion represents the version of the json format and will be
 // incremented for any change to this format that requires changes to a
 // consuming parser.
-const FormatVersion = "0.1"
+const FormatVersion = "0.2"
 
 // Plan is the top-level representation of the json format of a plan. It includes
 // the complete config and current state.
@@ -259,8 +259,8 @@ func (p *plan) marshalResourceDrift(oldState, newState *states.State, schemas *t
 				} else {
 					newVal = cty.NullVal(ty)
 				}
-				oldSensitive := sensitiveAsBool(oldVal)
-				newSensitive := sensitiveAsBool(newVal)
+				oldSensitive := jsonstate.SensitiveAsBool(oldVal)
+				newSensitive := jsonstate.SensitiveAsBool(newVal)
 				oldVal, _ = oldVal.UnmarkDeep()
 				newVal, _ = newVal.UnmarkDeep()
 
@@ -367,7 +367,7 @@ func (p *plan) marshalResourceChanges(changes *plans.Changes, schemas *terraform
 			if schema.ContainsSensitive() {
 				marks = append(marks, schema.ValueMarks(changeV.Before, nil)...)
 			}
-			bs := sensitiveAsBool(changeV.Before.MarkWithPaths(marks))
+			bs := jsonstate.SensitiveAsBool(changeV.Before.MarkWithPaths(marks))
 			beforeSensitive, err = ctyjson.Marshal(bs, bs.Type())
 			if err != nil {
 				return err
@@ -396,7 +396,7 @@ func (p *plan) marshalResourceChanges(changes *plans.Changes, schemas *terraform
 			if schema.ContainsSensitive() {
 				marks = append(marks, schema.ValueMarks(changeV.After, nil)...)
 			}
-			as := sensitiveAsBool(changeV.After.MarkWithPaths(marks))
+			as := jsonstate.SensitiveAsBool(changeV.After.MarkWithPaths(marks))
 			afterSensitive, err = ctyjson.Marshal(as, as.Type())
 			if err != nil {
 				return err
@@ -682,88 +682,6 @@ func unknownAsBool(val cty.Value) cty.Value {
 	}
 }
 
-// recursively iterate through a marked cty.Value, replacing sensitive values
-// with cty.True and non-sensitive values with cty.False.
-//
-// The result also normalizes some types: all sequence types are turned into
-// tuple types and all mapping types are converted to object types, since we
-// assume the result of this is just going to be serialized as JSON (and thus
-// lose those distinctions) anyway.
-//
-// For map/object values, all non-sensitive attribute values will be omitted
-// instead of returning false, as this results in a more compact serialization.
-func sensitiveAsBool(val cty.Value) cty.Value {
-	if val.HasMark("sensitive") {
-		return cty.True
-	}
-
-	ty := val.Type()
-	switch {
-	case val.IsNull(), ty.IsPrimitiveType(), ty.Equals(cty.DynamicPseudoType):
-		return cty.False
-	case ty.IsListType() || ty.IsTupleType() || ty.IsSetType():
-		if !val.IsKnown() {
-			// If the collection is unknown we can't say anything about the
-			// sensitivity of its contents
-			return cty.EmptyTupleVal
-		}
-		length := val.LengthInt()
-		if length == 0 {
-			// If there are no elements then we can't have sensitive values
-			return cty.EmptyTupleVal
-		}
-		vals := make([]cty.Value, 0, length)
-		it := val.ElementIterator()
-		for it.Next() {
-			_, v := it.Element()
-			vals = append(vals, sensitiveAsBool(v))
-		}
-		// The above transform may have changed the types of some of the
-		// elements, so we'll always use a tuple here in case we've now made
-		// different elements have different types. Our ultimate goal is to
-		// marshal to JSON anyway, and all of these sequence types are
-		// indistinguishable in JSON.
-		return cty.TupleVal(vals)
-	case ty.IsMapType() || ty.IsObjectType():
-		if !val.IsKnown() {
-			// If the map/object is unknown we can't say anything about the
-			// sensitivity of its attributes
-			return cty.EmptyObjectVal
-		}
-		var length int
-		switch {
-		case ty.IsMapType():
-			length = val.LengthInt()
-		default:
-			length = len(val.Type().AttributeTypes())
-		}
-		if length == 0 {
-			// If there are no elements then we can't have sensitive values
-			return cty.EmptyObjectVal
-		}
-		vals := make(map[string]cty.Value)
-		it := val.ElementIterator()
-		for it.Next() {
-			k, v := it.Element()
-			s := sensitiveAsBool(v)
-			// Omit all of the "false"s for non-sensitive values for more
-			// compact serialization
-			if !s.RawEquals(cty.False) {
-				vals[k.AsString()] = s
-			}
-		}
-		// The above transform may have changed the types of some of the
-		// elements, so we'll always use an object here in case we've now made
-		// different elements have different types. Our ultimate goal is to
-		// marshal to JSON anyway, and all of these mapping types are
-		// indistinguishable in JSON.
-		return cty.ObjectVal(vals)
-	default:
-		// Should never happen, since the above should cover all types
-		panic(fmt.Sprintf("sensitiveAsBool cannot handle %#v", val))
-	}
-}
-
 func actionString(action string) []string {
 	switch {
 	case action == "NoOp":

internal/command/jsonplan/plan_test.go

@@ -265,219 +265,6 @@ func TestUnknownAsBool(t *testing.T) {
 	}
 }
 
-func TestSensitiveAsBool(t *testing.T) {
-	sensitive := "sensitive"
-	tests := []struct {
-		Input cty.Value
-		Want  cty.Value
-	}{
-		{
-			cty.StringVal("hello"),
-			cty.False,
-		},
-		{
-			cty.NullVal(cty.String),
-			cty.False,
-		},
-		{
-			cty.StringVal("hello").Mark(sensitive),
-			cty.True,
-		},
-		{
-			cty.NullVal(cty.String).Mark(sensitive),
-			cty.True,
-		},
-
-		{
-			cty.NullVal(cty.DynamicPseudoType).Mark(sensitive),
-			cty.True,
-		},
-		{
-			cty.NullVal(cty.Object(map[string]cty.Type{"test": cty.String})),
-			cty.False,
-		},
-		{
-			cty.NullVal(cty.Object(map[string]cty.Type{"test": cty.String})).Mark(sensitive),
-			cty.True,
-		},
-		{
-			cty.DynamicVal,
-			cty.False,
-		},
-		{
-			cty.DynamicVal.Mark(sensitive),
-			cty.True,
-		},
-
-		{
-			cty.ListValEmpty(cty.String),
-			cty.EmptyTupleVal,
-		},
-		{
-			cty.ListValEmpty(cty.String).Mark(sensitive),
-			cty.True,
-		},
-		{
-			cty.ListVal([]cty.Value{
-				cty.StringVal("hello"),
-				cty.StringVal("friend").Mark(sensitive),
-			}),
-			cty.TupleVal([]cty.Value{
-				cty.False,
-				cty.True,
-			}),
-		},
-		{
-			cty.SetValEmpty(cty.String),
-			cty.EmptyTupleVal,
-		},
-		{
-			cty.SetValEmpty(cty.String).Mark(sensitive),
-			cty.True,
-		},
-		{
-			cty.SetVal([]cty.Value{cty.StringVal("hello")}),
-			cty.TupleVal([]cty.Value{cty.False}),
-		},
-		{
-			cty.SetVal([]cty.Value{cty.StringVal("hello").Mark(sensitive)}),
-			cty.True,
-		},
-		{
-			cty.EmptyTupleVal.Mark(sensitive),
-			cty.True,
-		},
-		{
-			cty.TupleVal([]cty.Value{
-				cty.StringVal("hello"),
-				cty.StringVal("friend").Mark(sensitive),
-			}),
-			cty.TupleVal([]cty.Value{
-				cty.False,
-				cty.True,
-			}),
-		},
-		{
-			cty.MapValEmpty(cty.String),
-			cty.EmptyObjectVal,
-		},
-		{
-			cty.MapValEmpty(cty.String).Mark(sensitive),
-			cty.True,
-		},
-		{
-			cty.MapVal(map[string]cty.Value{
-				"greeting": cty.StringVal("hello"),
-				"animal":   cty.StringVal("horse"),
-			}),
-			cty.EmptyObjectVal,
-		},
-		{
-			cty.MapVal(map[string]cty.Value{
-				"greeting": cty.StringVal("hello"),
-				"animal":   cty.StringVal("horse").Mark(sensitive),
-			}),
-			cty.ObjectVal(map[string]cty.Value{
-				"animal": cty.True,
-			}),
-		},
-		{
-			cty.MapVal(map[string]cty.Value{
-				"greeting": cty.StringVal("hello"),
-				"animal":   cty.StringVal("horse").Mark(sensitive),
-			}).Mark(sensitive),
-			cty.True,
-		},
-		{
-			cty.EmptyObjectVal,
-			cty.EmptyObjectVal,
-		},
-		{
-			cty.ObjectVal(map[string]cty.Value{
-				"greeting": cty.StringVal("hello"),
-				"animal":   cty.StringVal("horse"),
-			}),
-			cty.EmptyObjectVal,
-		},
-		{
-			cty.ObjectVal(map[string]cty.Value{
-				"greeting": cty.StringVal("hello"),
-				"animal":   cty.StringVal("horse").Mark(sensitive),
-			}),
-			cty.ObjectVal(map[string]cty.Value{
-				"animal": cty.True,
-			}),
-		},
-		{
-			cty.ObjectVal(map[string]cty.Value{
-				"greeting": cty.StringVal("hello"),
-				"animal":   cty.StringVal("horse").Mark(sensitive),
-			}).Mark(sensitive),
-			cty.True,
-		},
-		{
-			cty.ListVal([]cty.Value{
-				cty.ObjectVal(map[string]cty.Value{
-					"a": cty.UnknownVal(cty.String),
-				}),
-				cty.ObjectVal(map[string]cty.Value{
-					"a": cty.StringVal("known").Mark(sensitive),
-				}),
-			}),
-			cty.TupleVal([]cty.Value{
-				cty.EmptyObjectVal,
-				cty.ObjectVal(map[string]cty.Value{
-					"a": cty.True,
-				}),
-			}),
-		},
-		{
-			cty.ListVal([]cty.Value{
-				cty.MapValEmpty(cty.String),
-				cty.MapVal(map[string]cty.Value{
-					"a": cty.StringVal("known").Mark(sensitive),
-				}),
-				cty.MapVal(map[string]cty.Value{
-					"a": cty.UnknownVal(cty.String),
-				}),
-			}),
-			cty.TupleVal([]cty.Value{
-				cty.EmptyObjectVal,
-				cty.ObjectVal(map[string]cty.Value{
-					"a": cty.True,
-				}),
-				cty.EmptyObjectVal,
-			}),
-		},
-		{
-			cty.ObjectVal(map[string]cty.Value{
-				"list":   cty.UnknownVal(cty.List(cty.String)),
-				"set":    cty.UnknownVal(cty.Set(cty.Bool)),
-				"tuple":  cty.UnknownVal(cty.Tuple([]cty.Type{cty.String, cty.Number})),
-				"map":    cty.UnknownVal(cty.Map(cty.String)),
-				"object": cty.UnknownVal(cty.Object(map[string]cty.Type{"a": cty.String})),
-			}),
-			cty.ObjectVal(map[string]cty.Value{
-				"list":   cty.EmptyTupleVal,
-				"set":    cty.EmptyTupleVal,
-				"tuple":  cty.EmptyTupleVal,
-				"map":    cty.EmptyObjectVal,
-				"object": cty.EmptyObjectVal,
-			}),
-		},
-	}
-
-	for _, test := range tests {
-		got := sensitiveAsBool(test.Input)
-		if !reflect.DeepEqual(got, test.Want) {
-			t.Errorf(
-				"wrong result\ninput: %#v\ngot:   %#v\nwant:  %#v",
-				test.Input, got, test.Want,
-			)
-		}
-	}
-}
-
 func TestEncodePaths(t *testing.T) {
 	tests := map[string]struct {
 		Input cty.PathSet

internal/command/jsonplan/resource.go

@@ -1,6 +1,8 @@
 package jsonplan
 
 import (
+	"encoding/json"
+
 	"github.com/hashicorp/terraform/internal/addrs"
 )
 
@@ -33,6 +35,10 @@ type resource struct {
 	// unknown values are omitted or set to null, making them indistinguishable
 	// from absent values.
 	AttributeValues attributeValues `json:"values,omitempty"`
+
+	// SensitiveValues is similar to AttributeValues, but with all sensitive
+	// values replaced with true, and all non-sensitive leaf values omitted.
+	SensitiveValues json.RawMessage `json:"sensitive_values,omitempty"`
 }
 
 // resourceChange is a description of an individual change action that Terraform