Add support for force pushing with the remote backend

Both differing serials and lineage protections should be bypassed
with the -force flag (in addition to resources).

Compared to other backends we aren’t just shipping over the state
bytes in a simple payload during the persistence phase of the push
command and the force flag added to the Go TFE client needs to be
specified at that time.

To prevent changing every method signature of PersistState of the
remote client I added an optional interface that provides a hook
to flag the Client as operating in a force push context. Changing
the method signature would be more explicit at the cost of not
being used anywhere else currently or the optional interface pattern
could be applied to the state itself so it could be upgraded to
support PersistState(force bool) only when needed.

Prior to this only the resources of the state were checked for
changes not the lineage or the serial. To bring this in line with
documented behavior noted above those attributes also have a “read”
counterpart just like state has. These are now checked along with
state to determine if the state as a whole is unchanged.

Tests were altered to table driven test format and testing was
expanded to include WriteStateForMigration and its interaction
with a ClientForcePusher type.

Author: Lee Trout

backend/remote/backend_state.go

@@ -20,6 +20,7 @@ type remoteClient struct {
 	runID          string
 	stateUploadErr bool
 	workspace      *tfe.Workspace
+	forcePush      bool
 }
 
 // Get the remote state.
@@ -69,6 +70,7 @@ func (r *remoteClient) Put(state []byte) error {
 		Serial:  tfe.Int64(int64(stateFile.Serial)),
 		MD5:     tfe.String(fmt.Sprintf("%x", md5.Sum(state))),
 		State:   tfe.String(base64.StdEncoding.EncodeToString(state)),
+		Force:   tfe.Bool(r.forcePush),
 	}
 
 	// If we have a run ID, make sure to add it to the options
@@ -97,6 +99,12 @@ func (r *remoteClient) Delete() error {
 	return nil
 }
 
+// EnableForcePush to allow the remote client to overwrite state
+// by implementing remote.ClientForcePusher
+func (r *remoteClient) EnableForcePush() {
+	r.forcePush = true
+}
+
 // Lock the remote state.
 func (r *remoteClient) Lock(info *state.LockInfo) (string, error) {
 	ctx := context.Background()

state/remote/remote.go

@@ -15,6 +15,14 @@ type Client interface {
 	Delete() error
 }
 
+// ClientForcePusher is an optional interface that allows a remote
+// state to force push by managing a flag on the client that is
+// toggled on by a call to EnableForcePush.
+type ClientForcePusher interface {
+	Client
+	EnableForcePush()
+}
+
 // ClientLocker is an optional interface that allows a remote state
 // backend to enable state lock/unlock.
 type ClientLocker interface {

state/remote/remote_test.go

@@ -69,6 +69,7 @@ func (c nilClient) Delete() error { return nil }
 type mockClient struct {
 	current []byte
 	log     []mockClientRequest
+	force   bool
 }
 
 type mockClientRequest struct {
@@ -89,7 +90,11 @@ func (c *mockClient) Get() (*Payload, error) {
 }
 
 func (c *mockClient) Put(data []byte) error {
-	c.appendLog("Put", data)
+	if c.force {
+		c.appendLog("Force Put", data)
+	} else {
+		c.appendLog("Put", data)
+	}
 	c.current = data
 	return nil
 }
@@ -100,6 +105,11 @@ func (c *mockClient) Delete() error {
 	return nil
 }
 
+// Implements remote.ClientForcePusher
+func (c *mockClient) EnableForcePush() {
+	c.force = true
+}
+
 func (c *mockClient) appendLog(method string, content []byte) {
 	// For easier test assertions, we actually log the result of decoding
 	// the content JSON rather than the raw bytes. Callers are in principle

state/remote/state.go

@@ -21,10 +21,19 @@ type State struct {
 
 	Client Client
 
-	lineage          string
-	serial           uint64
-	state, readState *states.State
-	disableLocks     bool
+	// We track two pieces of meta data in addition to the state itself:
+	//
+	// lineage - the state's unique ID
+	// serial  - the monotonic counter of "versions" of the state
+	//
+	// Both of these (along with state) have a sister field
+	// that represents the values read in from an existing source.
+	// All three of these values are used to determine if the new
+	// state has changed from an existing state we read in.
+	lineage, readLineage string
+	serial, readSerial   uint64
+	state, readState     *states.State
+	disableLocks         bool
 }
 
 var _ statemgr.Full = (*State)(nil)
@@ -64,8 +73,15 @@ func (s *State) WriteStateForMigration(f *statefile.File, force bool) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	checkFile := statefile.New(s.state, s.lineage, s.serial)
-	if !force {
+	// `force` is passed down from the CLI flag and terminates here. Actual
+	// force pushing with the remote backend happens when Put()'ing the contents
+	// in the backend. If force is specified we skip verifications and hand the
+	// context off to the client to use when persitence operations actually take place.
+	c, isForcePusher := s.Client.(ClientForcePusher)
+	if force && isForcePusher {
+		c.EnableForcePush()
+	} else {
+		checkFile := statefile.New(s.state, s.lineage, s.serial)
 		if err := statemgr.CheckValidImport(f, checkFile); err != nil {
 			return err
 		}
@@ -113,7 +129,12 @@ func (s *State) refreshState() error {
 	s.lineage = stateFile.Lineage
 	s.serial = stateFile.Serial
 	s.state = stateFile.State
-	s.readState = s.state.DeepCopy() // our states must be separate instances so we can track changes
+
+	// Properties from the remote must be separate so we can
+	// track changes as lineage, serial and/or state are mutated
+	s.readLineage = stateFile.Lineage
+	s.readSerial = stateFile.Serial
+	s.readState = s.state.DeepCopy()
 	return nil
 }
 
@@ -123,8 +144,11 @@ func (s *State) PersistState() error {
 	defer s.mu.Unlock()
 
 	if s.readState != nil {
-		if statefile.StatesMarshalEqual(s.state, s.readState) {
-			// If the state hasn't changed at all then we have nothing to do.
+		lineageUnchanged := s.readLineage != "" && s.lineage == s.readLineage
+		serialUnchanged := s.readSerial != 0 && s.serial == s.readSerial
+		stateUnchanged := statefile.StatesMarshalEqual(s.state, s.readState)
+		if stateUnchanged && lineageUnchanged && serialUnchanged {
+			// If the state, lineage or serial haven't changed at all then we have nothing to do.
 			return nil
 		}
 		s.serial++
@@ -161,7 +185,13 @@ func (s *State) PersistState() error {
 
 	// After we've successfully persisted, what we just wrote is our new
 	// reference state until someone calls RefreshState again.
+	// We've potentially overwritten (via force) the state, lineage
+	// and / or serial (and serial was incremented) so we copy over all
+	// three fields so everything matches the new state and a subsequent
+	// operation would correctly detect no changes to the lineage, serial or state.
 	s.readState = s.state.DeepCopy()
+	s.readLineage = s.lineage
+	s.readSerial = s.serial
 	return nil
 }