Merge pull request #1104 from amazeeio/relaxed-scheduling

Introduce relaxed backup job scheduling feature

Author: Kidswiss

cmd/operator/main.go

@@ -83,6 +83,7 @@ var (
 			&cli.StringFlag{Destination: &cfg.Config.PodExecRoleName, Name: "podexecrolename", EnvVars: []string{"BACKUP_PODEXECROLENAME"}, Value: "pod-executor", Usage: "set the role name that should be used for pod command execution"},
 
 			&cli.BoolFlag{Destination: &cfg.Config.EnableLeaderElection, Name: "enable-leader-election", EnvVars: []string{"BACKUP_ENABLE_LEADER_ELECTION"}, Value: true, DefaultText: "enabled", Usage: "enable leader election within the operator Pod"},
+			&cli.BoolFlag{Destination: &cfg.Config.EnableRelaxedScheduling, Name: "enable-relaxed-scheduling", EnvVars: []string{"BACKUP_ENABLE_RELAXED_SCHEDULING"}, Value: false, DefaultText: "disabled", Usage: "enable relaxed scheduling of backup jobs relying on the Kubernetes scheduler"},
 			&cli.BoolFlag{Destination: &cfg.Config.SkipWithoutAnnotation, Name: "skip-pvcs-without-annotation", EnvVars: []string{"BACKUP_SKIP_WITHOUT_ANNOTATION"}, Value: false, DefaultText: "disabled", Usage: "skip selecting PVCs that don't have the BACKUP_ANNOTATION"},
 			&cli.StringFlag{Destination: &cfg.Config.BackupCheckSchedule, Name: "checkschedule", EnvVars: []string{"BACKUP_CHECKSCHEDULE"}, Value: "0 0 * * 0", Usage: "the default check schedule"},
 			&cli.StringFlag{Destination: &cfg.Config.OperatorNamespace, Name: "operator-namespace", EnvVars: []string{"BACKUP_OPERATOR_NAMESPACE"}, Required: true, Usage: "set the namespace in which the K8up operator itself runs"},

docs/modules/ROOT/examples/usage/k8up.txt

@@ -1,5 +1,5 @@
 NAME:
-   k8up - A new cli application
+   k8up - Kubernetes and OpenShift Backup Operator
 
 USAGE:
    k8up [global options] command [command options] [arguments...]
@@ -19,4 +19,4 @@ GLOBAL OPTIONS:
    --version, -v           print the version (default: false)
 
 COPYRIGHT:
-   (c) 2021 VSHN AG
+   (c) K8up Authors

docs/modules/ROOT/examples/usage/operator.txt

@@ -47,8 +47,10 @@ OPTIONS:
    --podexecaccountname value, --serviceaccount value  set the service account name that should be used for the pod command execution (default: "pod-executor") [$BACKUP_PODEXECACCOUNTNAME]
    --podexecrolename value                             set the role name that should be used for pod command execution (default: "pod-executor") [$BACKUP_PODEXECROLENAME]
    --enable-leader-election                            enable leader election within the operator Pod (default: enabled) [$BACKUP_ENABLE_LEADER_ELECTION]
+   --enable-relaxed-scheduling                         enable relaxed scheduling of backup jobs relying on the Kubernetes scheduler (default: disabled) [$BACKUP_ENABLE_RELAXED_SCHEDULING]
    --skip-pvcs-without-annotation                      skip selecting PVCs that don't have the BACKUP_ANNOTATION (default: disabled) [$BACKUP_SKIP_WITHOUT_ANNOTATION]
    --checkschedule value                               the default check schedule (default: "0 0 * * 0") [$BACKUP_CHECKSCHEDULE]
    --operator-namespace value                          set the namespace in which the K8up operator itself runs [$BACKUP_OPERATOR_NAMESPACE]
+   --insecure-allow-podexec-spdy-fallback              enable fallback to SPDY connections for data streaming used by application aware backups. Might need to be enabled if the cluster has Kubernetes version 1.30 or lower. K8up uses WebSockets by default. CAUTION: Has been observed to cause silent data corruption in some network setups, use at own risk! (default: false) [$INSECURE_ALLOW_PODEXEC_SPDY_FALLBACK]
    --vardir value                                      the var data dir for read/write k8up data or temp file in the backup pod (default: "/k8up") [$VAR_DIR]
    --help, -h                                          show help (default: false)

docs/modules/ROOT/examples/usage/restic.txt

@@ -18,6 +18,7 @@ OPTIONS:
    --fileExtensionAnnotation value                                                Defines the file extension to use for STDOUT backups. [$FILEEXTENSION_ANNOTATION]
    --backucontainerannotation value                                               set the annotation name that specify the backup container inside the Pod (default: "k8up.io/backupcommand-container") [$BACKUP_CONTAINERANNOTATION]
    --skipPreBackup                                                                If the job should skip the backup command and only backup volumes. (default: false) [$SKIP_PREBACKUP]
+   --skipSnapshotSync                                                             If set, skip synchronizing Snapshot custom resources to the cluster after backup or prune operations. Webhook notifications are still sent. (default: false) [$BACKUP_SKIP_SNAPSHOT_SYNC]
    --promURL value                                                                Sets the URL of a prometheus push gateway to report metrics. [$PROM_URL]
    --clusterName value                                                            Sets the Kubernetes cluster name for grouping metrics in push gateway [$CLUSTER_NAME]
    --webhookURL value, --statsURL value                                           Sets the URL of a server which will retrieve a webhook after the action completes. [$STATS_URL]
@@ -69,4 +70,5 @@ OPTIONS:
    --caCert value                                                                 The certificate authority file path [$CA_CERT_FILE]
    --clientCert value                                                             The client certificate file path [$CLIENT_CERT_FILE]
    --clientKey value                                                              The client private key file path [$CLIENT_KEY_FILE]
+   --insecure-allow-podexec-spdy-fallback                                         enable fallback to SPDY connections for data streaming used by application aware backups. Might need to be enabled if the cluster has Kubernetes version 1.30 or lower. K8up uses WebSockets by default. CAUTION: Has been observed to cause silent data corruption in some network setups, use at own risk! (default: false) [$INSECURE_ALLOW_PODEXEC_SPDY_FALLBACK]
    --help, -h                                                                     show help (default: false)

docs/modules/ROOT/nav.adoc

@@ -38,7 +38,7 @@
 * xref:explanations/system-requirements.adoc[System Requirements]
 * xref:explanations/what-has-changed-in-v1.adoc[Changes in K8up v1.0]
 * xref:explanations/what-has-changed-in-v2.adoc[Changes in K8up v2.0]
-* xref:explanations/rwo.adoc[]
+* xref:explanations/backup-pod-scheduling.adoc[Backup Pod Scheduling]
 
 .About
 * xref:about/roadmap.adoc[Roadmap]

docs/modules/ROOT/pages/explanations/backup-pod-scheduling.adoc

@@ -0,0 +1,25 @@
+= Backup Pod Scheduling
+
+K8up has two scheduling modes for backup pods. Both scheduling modes respect the `k8up.io/hostname` annotation on a PVC (see: xref:references/annotations.adoc[Annotations]).
+
+== Classic Scheduling (default)
+
+K8s does not prevent mounting an RWO PVC to multiple pods, if they are scheduled on the same host.
+K8up uses this fact to provide the ability to back up RWO PVCs.
+
+For a given backup in a namespace K8up will list all the PVCs.
+The PVCs are then grouped depending on their type:
+
+* all RWX PVCs are grouped together
+* RWO PVCs are grouped by k8s node where they are currently mounted
+
+K8up will then deploy backup jobs according to the grouping, a single job for all RWX PVCs and a job for each K8s node.
+The jobs themselves loop over the mounted PVCs and do a file backup via restic.
+
+== Relaxed Scheduling
+
+In the relaxed scheduling mode, the backup pod scheduling is handled by the Kubernetes scheduler. This mode is intended for the use with storage provisioners that handle node affinity and topology well for their provisioned volumes.
+
+For a given backup in a namespace K8up will list all the PVCs and schedule one backup job for each PVC. K8up will not set a node selector for the backup pods unless explicitly requested through the `k8up.io/hostname` annotation on a PVC. This allows the Kubernetes scheduler to more freely select an appropriate node to run the backup on.
+
+Relaxed scheduling can be enabled via the operator flag `--enable-relaxed-scheduling` or environment variable `BACKUP_ENABLE_RELAXED_SCHEDULING` (see: xref:references/operator-config-reference.adoc[Operator Configuration]).

docs/modules/ROOT/pages/explanations/backup.adoc

@@ -21,9 +21,9 @@ It does not work in some cases though.
 More precise, it does not work when files are kept open for a long period of time, like databases do.
 It does also not work for content that is not stored in the cluster, like a managed database that is offered by a service provider.
 
-NOTE: If the PVC has the `RWO` access mode, the backup `Pod` needs to be scheduled onto the same node, on which the `Pod` (which uses the respective `PVC`) runs.
+NOTE: If the PVC has the `RWO` access mode, the backup `Pod` needs to be scheduled onto the same node, on which the `Pod` (which uses the respective `PVC`) runs. This happens explicitly or implicitly depending on the backup pod scheduling mode (see: xref:explanations/backup-pod-scheduling.adoc[Backup Pod Scheduling]).
 
-Read xref:references/annotations.adoc[] to learn more about how the backup process can be influenced.
+Read xref:references/annotations.adoc[] to learn more about how the backup process can be influenced and xref:explanations/backup-pod-scheduling.adoc[Backup Pod Scheduling] to learn more about the two backup pod scheduling modes.
 
 == Application-Aware Backups
 

docs/modules/ROOT/pages/explanations/rwo.adoc

@@ -36,4 +36,10 @@ See xref:references/operator-config-reference.adoc[Operator Configuration refere
 |A string which is valid pod name.
 |`Pod`
 |`BACKUP_CONTAINERANNOTATION`
+
+|`k8up.io/hostname`
+|If defined, the backup pod will be scheduled on this node
+|A string which is a valid node name.
+|`PersistentVolumeClaim`
+|n/a
 |===

docs/modules/ROOT/pages/references/annotations.adoc

@@ -246,13 +246,13 @@ This is for advanced use-cases only. Please only set this if you know what you'r
 Value must be positive integer if given.
 | *`keepJobs`* __integer__ | KeepJobs amount of jobs to keep for later analysis.
 
-
 Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
 | *`failedJobsHistoryLimit`* __integer__ | FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
 KeepJobs is used property is not specified.
 | *`successfulJobsHistoryLimit`* __integer__ | SuccessfulJobsHistoryLimit amount of successful jobs to keep for later analysis.
 KeepJobs is used property is not specified.
 | *`promURL`* __string__ | PromURL sets a prometheus push URL where the backup container send metrics to
+| *`clusterName`* __string__ | ClusterName sets the kubernetes cluster name to send to pushgateway for grouping metrics
 | *`statsURL`* __string__ | StatsURL sets an arbitrary URL where the restic container posts metrics and
 information about the snapshots to. This is in addition to the prometheus
 pushgateway.
@@ -290,13 +290,13 @@ This is for advanced use-cases only. Please only set this if you know what you'r
 Value must be positive integer if given.
 | *`keepJobs`* __integer__ | KeepJobs amount of jobs to keep for later analysis.
 
-
 Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
 | *`failedJobsHistoryLimit`* __integer__ | FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
 KeepJobs is used property is not specified.
 | *`successfulJobsHistoryLimit`* __integer__ | SuccessfulJobsHistoryLimit amount of successful jobs to keep for later analysis.
 KeepJobs is used property is not specified.
 | *`promURL`* __string__ | PromURL sets a prometheus push URL where the backup container send metrics to
+| *`clusterName`* __string__ | ClusterName sets the kubernetes cluster name to send to pushgateway for grouping metrics
 | *`statsURL`* __string__ | StatsURL sets an arbitrary URL where the restic container posts metrics and
 information about the snapshots to. This is in addition to the prometheus
 pushgateway.
@@ -371,9 +371,9 @@ This is for advanced use-cases only. Please only set this if you know what you'r
 | *`activeDeadlineSeconds`* __integer__ | ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be continuously active before the system tries to terminate it.
 Value must be positive integer if given.
 | *`promURL`* __string__ | PromURL sets a prometheus push URL where the backup container send metrics to
+| *`clusterName`* __string__ | ClusterName sets the kubernetes cluster name to send to pushgateway for grouping metrics
 | *`keepJobs`* __integer__ | KeepJobs amount of jobs to keep for later analysis.
 
-
 Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
 | *`failedJobsHistoryLimit`* __integer__ | FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
 KeepJobs is used property is not specified.
@@ -408,9 +408,9 @@ This is for advanced use-cases only. Please only set this if you know what you'r
 | *`activeDeadlineSeconds`* __integer__ | ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be continuously active before the system tries to terminate it.
 Value must be positive integer if given.
 | *`promURL`* __string__ | PromURL sets a prometheus push URL where the backup container send metrics to
+| *`clusterName`* __string__ | ClusterName sets the kubernetes cluster name to send to pushgateway for grouping metrics
 | *`keepJobs`* __integer__ | KeepJobs amount of jobs to keep for later analysis.
 
-
 Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
 | *`failedJobsHistoryLimit`* __integer__ | FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
 KeepJobs is used property is not specified.
@@ -738,7 +738,6 @@ Value must be positive integer if given.
 | *`retention`* __xref:{anchor_prefix}-github.1485827954.workers.dev-k8up-io-k8up-v2-api-v1-retentionpolicy[$$RetentionPolicy$$]__ | Retention sets how many backups should be kept after a forget and prune
 | *`keepJobs`* __integer__ | KeepJobs amount of jobs to keep for later analysis.
 
-
 Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
 | *`failedJobsHistoryLimit`* __integer__ | FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
 KeepJobs is used property is not specified.
@@ -775,7 +774,6 @@ Value must be positive integer if given.
 | *`retention`* __xref:{anchor_prefix}-github.1485827954.workers.dev-k8up-io-k8up-v2-api-v1-retentionpolicy[$$RetentionPolicy$$]__ | Retention sets how many backups should be kept after a forget and prune
 | *`keepJobs`* __integer__ | KeepJobs amount of jobs to keep for later analysis.
 
-
 Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
 | *`failedJobsHistoryLimit`* __integer__ | FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
 KeepJobs is used property is not specified.
@@ -889,16 +887,19 @@ This is for advanced use-cases only. Please only set this if you know what you'r
 Value must be positive integer if given.
 | *`restoreMethod`* __xref:{anchor_prefix}-github.1485827954.workers.dev-k8up-io-k8up-v2-api-v1-restoremethod[$$RestoreMethod$$]__ | 
 | *`restoreFilter`* __string__ | 
+| *`restoreTimeFilter`* __string__ | Simple filter to define a timestamp (prefix, YYYY-MM-DD hh:mm:ss) for snapshot selection instead of latest (or latest if nothing matches)
 | *`snapshot`* __string__ | 
 | *`keepJobs`* __integer__ | KeepJobs amount of jobs to keep for later analysis.
 
-
 Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
 | *`failedJobsHistoryLimit`* __integer__ | FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
 KeepJobs is used property is not specified.
 | *`successfulJobsHistoryLimit`* __integer__ | SuccessfulJobsHistoryLimit amount of successful jobs to keep for later analysis.
 KeepJobs is used property is not specified.
 | *`tags`* __string array__ | Tags is a list of arbitrary tags that get added to the backup via Restic's tagging system
+| *`paths`* __string array__ | Paths is a list of paths that are contained with in a snapshot and can be filtered by
+| *`delete`* __boolean__ | Delete ensures the state after restoring a snapshot is identical to the snapshot
+Deletes files from target if they do not exist in snapshot
 |===
 
 
@@ -931,16 +932,19 @@ This is for advanced use-cases only. Please only set this if you know what you'r
 Value must be positive integer if given.
 | *`restoreMethod`* __xref:{anchor_prefix}-github.1485827954.workers.dev-k8up-io-k8up-v2-api-v1-restoremethod[$$RestoreMethod$$]__ | 
 | *`restoreFilter`* __string__ | 
+| *`restoreTimeFilter`* __string__ | Simple filter to define a timestamp (prefix, YYYY-MM-DD hh:mm:ss) for snapshot selection instead of latest (or latest if nothing matches)
 | *`snapshot`* __string__ | 
 | *`keepJobs`* __integer__ | KeepJobs amount of jobs to keep for later analysis.
 
-
 Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
 | *`failedJobsHistoryLimit`* __integer__ | FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
 KeepJobs is used property is not specified.
 | *`successfulJobsHistoryLimit`* __integer__ | SuccessfulJobsHistoryLimit amount of successful jobs to keep for later analysis.
 KeepJobs is used property is not specified.
 | *`tags`* __string array__ | Tags is a list of arbitrary tags that get added to the backup via Restic's tagging system
+| *`paths`* __string array__ | Paths is a list of paths that are contained with in a snapshot and can be filtered by
+| *`delete`* __boolean__ | Delete ensures the state after restoring a snapshot is identical to the snapshot
+Deletes files from target if they do not exist in snapshot
 |===
 
 
@@ -1153,7 +1157,6 @@ ScheduleSpec defines the schedules for the various job types.
 | *`backend`* __xref:{anchor_prefix}-github.1485827954.workers.dev-k8up-io-k8up-v2-api-v1-backend[$$Backend$$]__ | 
 | *`keepJobs`* __integer__ | KeepJobs amount of jobs to keep for later analysis.
 
-
 Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively.
 | *`failedJobsHistoryLimit`* __integer__ | FailedJobsHistoryLimit amount of failed jobs to keep for later analysis.
 KeepJobs is used property is not specified.