[backend] kubernetes: fix secret size limitation

By now kubernetes backend could hold up to defaultETCDSize gzipped data
(which is 1-1.5Mb). This doesn't scale for larger states.

This commit implements spliting data across multiple secrets bound by
the same Secret labels. This practically removes etcd value size
limitation and allows backend to scale across multiple secrets.

This also takes care of cases when state needs to be shrinked. In such
case code will cleanup unneeded secrets

Signed-off-by: Dinar Valeev <dinar.valeev@absa.africa>

Author: k0da

internal/backend/remote-state/kubernetes/backend_state.go

@@ -94,7 +94,8 @@ func (b *Backend) StateMgr(name string) (statemgr.Full, error) {
 			return nil, err
 		}
 
-		secretName, err := c.createSecretName()
+		// get base secret name
+		secretName, err := c.createSecretName(0)
 		if err != nil {
 			return nil, err
 		}

internal/backend/remote-state/kubernetes/client.go

@@ -30,6 +30,7 @@ const (
 	tfstateWorkspaceKey       = "tfstateWorkspace"
 	tfstateLockInfoAnnotation = "app.terraform.io/lock-info"
 	managedByKey              = "app.kubernetes.io/managed-by"
+	etcdDefaultSize           = 1048576
 )
 
 type RemoteClient struct {
@@ -42,28 +43,27 @@ type RemoteClient struct {
 }
 
 func (c *RemoteClient) Get() (payload *remote.Payload, err error) {
-	secretName, err := c.createSecretName()
+	secretList, err := c.getSecrets()
 	if err != nil {
 		return nil, err
 	}
-	secret, err := c.kubernetesSecretClient.Get(secretName, metav1.GetOptions{})
-	if err != nil {
-		if k8serrors.IsNotFound(err) {
-			return nil, nil
-		}
-		return nil, err
-	}
 
-	secretData := getSecretData(secret)
-	stateRaw, ok := secretData[tfstateKey]
-	if !ok {
-		// The secret exists but there is no state in it
+	if len(secretList.Items) == 0 {
 		return nil, nil
 	}
 
-	stateRawString := stateRaw.(string)
+	var data []string
+	for _, secret := range secretList.Items {
+		secretData := getSecretData(&secret)
+		stateRaw, ok := secretData[tfstateKey]
+		if !ok {
+			// The secret exists but there is no state in it
+			return nil, nil
+		}
+		data = append(data, stateRaw.(string))
+	}
 
-	state, err := uncompressState(stateRawString)
+	state, err := uncompressState(data)
 	if err != nil {
 		return nil, err
 	}
@@ -77,8 +77,18 @@ func (c *RemoteClient) Get() (payload *remote.Payload, err error) {
 	return p, nil
 }
 
+func multiSecret(size []byte) bool {
+	// by default etcd can hold up to 1.5Mib data for secret
+	return len(size) > etcdDefaultSize
+}
+
+func (c *RemoteClient) getSecrets() (*unstructured.UnstructuredList, error) {
+	ls := metav1.SetAsLabelSelector(c.getLabels())
+	return c.kubernetesSecretClient.List(metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(ls)})
+}
+
 func (c *RemoteClient) Put(data []byte) error {
-	secretName, err := c.createSecretName()
+	secretName, err := c.createSecretName(0)
 	if err != nil {
 		return err
 	}
@@ -88,46 +98,90 @@ func (c *RemoteClient) Put(data []byte) error {
 		return err
 	}
 
-	secret, err := c.getSecret(secretName)
+	parts := split(payload, etcdDefaultSize)
+	existingSecrets, err := c.getSecrets()
 	if err != nil {
-		if !k8serrors.IsNotFound(err) {
-			return err
-		}
+		return err
+	}
 
-		secret = &unstructured.Unstructured{
-			Object: map[string]interface{}{
-				"metadata": metav1.ObjectMeta{
-					Name:        secretName,
-					Namespace:   c.namespace,
-					Labels:      c.getLabels(),
-					Annotations: map[string]string{"encoding": "gzip"},
-				},
-			},
+	for idx, data := range parts {
+		secretName, err := c.createSecretName(idx)
+		if err != nil {
+			return err
 		}
 
-		secret, err = c.kubernetesSecretClient.Create(secret, metav1.CreateOptions{})
+		secret, err := c.getSecret(secretName)
 		if err != nil {
-			return err
+			if !k8serrors.IsNotFound(err) {
+				return err
+			}
+
+			secret = c.formatSecret(secretName)
+
+			secret, err = c.kubernetesSecretClient.Create(secret, metav1.CreateOptions{})
+			if err != nil {
+				return err
+			}
 		}
+
+		setState(secret, data)
+		_, err = c.kubernetesSecretClient.Update(secret, metav1.UpdateOptions{})
 	}
 
-	setState(secret, payload)
-	_, err = c.kubernetesSecretClient.Update(secret, metav1.UpdateOptions{})
+	// in case new state requires less secrets, cleanup old secrets
+	secretNum := len(existingSecrets.Items)
+	newSecretNum := len(parts)
+	for i := newSecretNum; i <= secretNum; i++ {
+		c.deleteSecret(fmt.Sprintf("%s-part%d", secretName, i))
+	}
 	return err
 }
 
+func (c *RemoteClient) formatSecret(name string) *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"metadata": metav1.ObjectMeta{
+				Name:        name,
+				Namespace:   c.namespace,
+				Labels:      c.getLabels(),
+				Annotations: map[string]string{"encoding": "gzip"},
+			},
+		},
+	}
+}
+
+func split(buf []byte, size int) [][]byte {
+	var chunk []byte
+	chunks := make([][]byte, 0, len(buf)/size+1)
+	for len(buf) >= size {
+		chunk, buf = buf[:size], buf[size:]
+		chunks = append(chunks, chunk)
+	}
+	if len(buf) > 0 {
+		chunks = append(chunks, buf[:len(buf)])
+	}
+	return chunks
+}
+
 // Delete the state secret
 func (c *RemoteClient) Delete() error {
-	secretName, err := c.createSecretName()
+	secretList, err := c.getSecrets()
 	if err != nil {
 		return err
 	}
 
-	err = c.deleteSecret(secretName)
-	if err != nil {
-		if !k8serrors.IsNotFound(err) {
+	for i, _ := range secretList.Items {
+		secretName, err := c.createSecretName(i)
+		if err != nil {
 			return err
 		}
+
+		err = c.deleteSecret(secretName)
+		if err != nil {
+			if !k8serrors.IsNotFound(err) {
+				return err
+			}
+		}
 	}
 
 	leaseName, err := c.createLeaseName()
@@ -317,9 +371,13 @@ func (c *RemoteClient) deleteLease(name string) error {
 	return c.kubernetesLeaseClient.Delete(name, delOps)
 }
 
-func (c *RemoteClient) createSecretName() (string, error) {
+func (c *RemoteClient) createSecretName(idx int) (string, error) {
 	secretName := strings.Join([]string{tfstateKey, c.workspace, c.nameSuffix}, "-")
 
+	if idx > 0 {
+		secretName = fmt.Sprintf("%s-part%d", secretName, idx)
+	}
+
 	errs := validation.IsDNS1123Subdomain(secretName)
 	if len(errs) > 0 {
 		k8sInfo := `
@@ -333,7 +391,7 @@ The workspace name and key must adhere to Kubernetes naming conventions.`
 }
 
 func (c *RemoteClient) createLeaseName() (string, error) {
-	n, err := c.createSecretName()
+	n, err := c.createSecretName(0)
 	if err != nil {
 		return "", err
 	}
@@ -352,14 +410,18 @@ func compressState(data []byte) ([]byte, error) {
 	return b.Bytes(), nil
 }
 
-func uncompressState(data string) ([]byte, error) {
-	decode, err := base64.StdEncoding.DecodeString(data)
-	if err != nil {
-		return nil, err
+func uncompressState(data []string) ([]byte, error) {
+	var rawData []byte
+	for _, chunk := range data {
+		decode, err := base64.StdEncoding.DecodeString(chunk)
+		if err != nil {
+			return nil, err
+		}
+		rawData = append(rawData, decode...)
 	}
 
 	b := new(bytes.Buffer)
-	gz, err := gzip.NewReader(bytes.NewReader(decode))
+	gz, err := gzip.NewReader(bytes.NewReader(rawData))
 	if err != nil {
 		return nil, err
 	}