diff --git a/Cargo.lock b/Cargo.lock index de2de2b8..103ac139 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1755,6 +1755,7 @@ dependencies = [ "futures", "http", "http-body-util", + "json-patch", "k8s-openapi", "kopiur-api", "kopiur-kopia", diff --git a/crates/api/src/common/mod.rs b/crates/api/src/common/mod.rs index e0fd48da..81fbc96d 100644 --- a/crates/api/src/common/mod.rs +++ b/crates/api/src/common/mod.rs @@ -1273,7 +1273,30 @@ pub struct ReplicationManualRunStatus { #[serde(default, skip_serializing_if = "Option::is_none")] pub phase: Option, /// RFC3339 instant the run reached a terminal phase. - #[serde(default, skip_serializing_if = "Option::is_none")] + // Deliberately serialized EVEN WHEN `None` — no `skip_serializing_if`, unlike + // every other optional in this struct. A non-terminal phase must emit + // `"completedAt": null` so the merge-patch the controller sends CLEARS the + // previous run's stamp. On receiving that null the apiserver either deletes + // the key (plain RFC-7386) or stores the null verbatim — a nullable CRD + // field on k8s 1.33 was observed doing the latter — and BOTH converge here: + // an absent key decodes to `None`, an explicit null decodes to `None`, and + // re-serializing either yields `null` again. + // + // Omitting the key instead leaves the old instant standing, which is both a + // lie in `kubectl get -o yaml` and — because the replication controllers + // build the noop guard's `current` by re-serializing this very struct — a + // guard that never converges: `desired` would omit the key while `current` + // carries the stale value, so every queued pass re-fires a PATCH the + // apiserver no-ops (#394). + // + // This depends on `patch_status` sending `kube::api::Patch::Merge` + // (`crates/controller/src/io/apply.rs`). Under `Patch::Apply` an explicit + // null does NOT clear the field, and this contract silently breaks. + // + // Kept a plain comment rather than rustdoc on purpose: doc comments become + // the CRD `description` (`kubectl explain`, docs/field-reference.md), and + // serde mechanics are not the user's business. + #[serde(default)] pub completed_at: Option, } diff --git a/crates/api/src/common/mover.rs b/crates/api/src/common/mover.rs index 15d9f4c2..ee59b066 100644 --- a/crates/api/src/common/mover.rs +++ b/crates/api/src/common/mover.rs @@ -134,6 +134,36 @@ pub struct MoverDefaults { /// their pods self-GC instead of lingering (ADR-0005 §12). pub const DEFAULT_JOB_TTL_SECONDS: i64 = 3600; +/// Per-side throttle overrides for a `kopia snapshot migrate` run, which reads +/// from a SOURCE repository and writes into a DESTINATION repository under two +/// separate kopia connections. +/// +/// `snapshot migrate` has **no speed flags of its own** — the only lever is +/// `kopia repository throttle set` on each side's connection (kopia persists the +/// limits in that connection's client config, and the migrate honors them when it +/// reopens it). So a cap here is expressed per side, and each side overrides +/// **that side's repository's** `moverDefaults.throttle` field by field: a field +/// set here wins, a field left unset falls back to the repository default, and +/// all four [`Throttle`] knobs are available on each side independently. +/// +/// Not `Copy` (neither is [`Throttle`]). +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Default, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct MigrateThrottle { + /// Caps for the SOURCE (read) side, overriding the source repository's + /// `moverDefaults.throttle` field by field. Applied with `repository + /// throttle set` on the migrate's read-only source connection — accepted + /// there, so a read-only source is throttled like any other. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// Caps for the DESTINATION (write) side, overriding the destination + /// repository's `moverDefaults.throttle` field by field. Applied with + /// `repository throttle set` on the destination connection the migrate + /// writes through. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub destination: Option, +} + /// Repository-wide throttling for a mover's kopia connection; each `None` leaves kopia's current limit. #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Default, JsonSchema)] #[serde(rename_all = "camelCase")] diff --git a/crates/api/src/common/tests.rs b/crates/api/src/common/tests.rs index 10c3dc34..c3f9fb47 100644 --- a/crates/api/src/common/tests.rs +++ b/crates/api/src/common/tests.rs @@ -1523,7 +1523,33 @@ fn replication_manual_run_status_roundtrips_camel_case() { assert_eq!(json["completedAt"], "2026-06-11T12:01:42Z"); let back: ReplicationManualRunStatus = serde_json::from_value(json).unwrap(); assert_eq!(back, st); - // Absent fields serialize away entirely (no explicit nulls in status). + // The contract (#394): `requestedAt`/`phase` serialize away when absent, but + // `completedAt` ALWAYS emits — as an explicit null when there is no + // completion instant — so the merge-patch CLEARS a previous run's stamp + // instead of leaving it standing over a fresh non-terminal phase. The + // apiserver either deletes the key or stores the null; both decode back to + // `None`, so neither leaves a stale timestamp. let empty = serde_json::to_value(ReplicationManualRunStatus::default()).unwrap(); - assert_eq!(empty, serde_json::json!({})); + assert_eq!(empty, serde_json::json!({ "completedAt": null })); + // …and that explicit null reads back as "no completion instant", not as a + // decode error (the round trip both directions). + let back: ReplicationManualRunStatus = serde_json::from_value(empty).unwrap(); + assert_eq!(back, ReplicationManualRunStatus::default()); + assert!(back.completed_at.is_none()); + + // A non-terminal run keeps its other fields AND clears the stamp. + let running = serde_json::to_value(ReplicationManualRunStatus { + requested_at: Some("2026-06-11T13:00:00Z".into()), + phase: Some(ReplicationManualRunPhase::Running), + completed_at: None, + }) + .unwrap(); + assert_eq!( + running, + serde_json::json!({ + "requestedAt": "2026-06-11T13:00:00Z", + "phase": "Running", + "completedAt": null, + }) + ); } diff --git a/crates/api/src/consts.rs b/crates/api/src/consts.rs index 31dd8a45..ead7794d 100644 --- a/crates/api/src/consts.rs +++ b/crates/api/src/consts.rs @@ -386,6 +386,36 @@ pub const SOURCE_PVC_AVAILABLE_CONDITION: &str = "SourcePvcAvailable"; /// `reason`/Event reason for [`SOURCE_PVC_AVAILABLE_CONDITION`] = `False`. pub const SOURCE_PVC_MISSING_REASON: &str = "SourcePvcMissing"; +/// `Restore` condition recording whether the object this restore's repository +/// is DERIVED from exists. Set `False` (with +/// [`RESTORE_REFERENT_MISSING_REASON`]) when the readiness gate cannot even +/// look up the repository because a referent is absent: the explicit +/// `spec.repository` object itself, or the `source.fromPolicy` `SnapshotPolicy` +/// the repository ref is read from (issue #393). +/// +/// Its own condition rather than the `Ready` one: the registry's coarse +/// [`crate::gates::StructuralGate::trips`] filter keys on condition+polarity, +/// so registering a `Ready=False` row would make every unrelated `Ready=False` +/// reason on a `Snapshot`/`Restore` read as an unknown gate from a newer +/// operator. Only this gate ever writes this condition, so it cannot misfire. +/// +/// Structural ([`crate::gates::RESTORE_REFERENT_MISSING_GATE`]): a referent +/// that is never created never self-heals, and while it is missing the restore +/// sits at `phase: Pending` with its `policy.waitTimeout` window deliberately +/// NOT started — the phase-invisible park shape of #359. +pub const RESTORE_REFERENT_AVAILABLE_CONDITION: &str = "ReferentAvailable"; +/// `reason`/Event reason for [`RESTORE_REFERENT_AVAILABLE_CONDITION`] = +/// `False`. Distinct from [`REPOSITORY_NOT_READY_REASON`] on purpose: that one +/// means "the repository object exists and its backend is unreachable", which +/// would be a lie for a `SnapshotPolicy` that was never applied. +pub const RESTORE_REFERENT_MISSING_REASON: &str = "RestoreReferentMissing"; +/// `reason` for [`RESTORE_REFERENT_AVAILABLE_CONDITION`] = `True`: the referent +/// appeared on a later pass (the clear-side counterpart of +/// [`RESTORE_REFERENT_MISSING_REASON`]; controller-internal remediation copy, +/// like the `Snapshot` gate's `SourcePvcFound`). Written ONLY when the +/// condition already exists — the healthy wire never grows the condition. +pub const RESTORE_REFERENT_FOUND_REASON: &str = "RestoreReferentFound"; + /// `Snapshot` condition: this deletion is HELD by the mass-deletion breaker /// (`Repository`/`ClusterRepository` `spec.deletionProtection.threshold`) /// until acknowledged via [`ALLOW_MASS_DELETION_ANNOTATION`] on the diff --git a/crates/api/src/gates.rs b/crates/api/src/gates.rs index a479f0c0..a79f1908 100644 --- a/crates/api/src/gates.rs +++ b/crates/api/src/gates.rs @@ -528,6 +528,33 @@ pub const SEED_MOVER_TOO_OLD_GATE: StructuralGate = StructuralGate { severity: GateSeverity::Warn, }; +/// A `Restore` whose repository referent does not exist (issue #393): the +/// explicit `spec.repository` object, or the `source.fromPolicy` +/// `SnapshotPolicy` the repository ref is derived from. +/// +/// The readiness gate cannot verify a repository it cannot even look up, so the +/// restore parks at `phase: Pending` and — critically — its +/// `policy.waitTimeout` window is NOT opened (`status.waitStartedAt` stays +/// unstamped) until the referent appears and its repository becomes `Ready`. +/// Before #393 the gate fell through unverified here, so a slow-arriving +/// referent silently spent the window; for a `fromPolicy` source, whose +/// `onMissingSnapshot` defaults to `Continue`, a spent window means an EMPTY +/// volume. +/// +/// WARN, not Fail, for the reason [`POLICY_REPOSITORY_NOT_READY_GATE`] is: +/// referents applied moments apart by GitOps are the ordinary case and resolve +/// themselves within a requeue or two, so this must explain the park rather +/// than independently turn a diagnostic red. What it must never be is INVISIBLE +/// — a restore parked on a `SnapshotPolicy` that was never applied looks +/// identical, by phase, to one that is simply still resolving. +pub const RESTORE_REFERENT_MISSING_GATE: StructuralGate = StructuralGate { + applies_to: GateScope::SnapshotOrRestore, + condition: consts::RESTORE_REFERENT_AVAILABLE_CONDITION, + blocked_status: CONDITION_FALSE, + reason: consts::RESTORE_REFERENT_MISSING_REASON, + severity: GateSeverity::Warn, +}; + /// Every human-actionable structural gate kopiur's reconcilers can park an /// object on. /// @@ -559,6 +586,7 @@ pub const STRUCTURAL_GATES: &[StructuralGate] = &[ SCHEDULE_FANOUT_CAPPED_GATE, POLICY_REPOSITORY_NOT_READY_GATE, SOURCE_PVC_MISSING_GATE, + RESTORE_REFERENT_MISSING_GATE, SEED_SOURCE_NOT_READY_GATE, SEED_SOURCE_AUTH_CONFLICT_GATE, SEEDING_GATE, @@ -893,6 +921,13 @@ mod tests { GateScope::Snapshot, GateSeverity::Fail, ), + ( + consts::RESTORE_REFERENT_AVAILABLE_CONDITION, + CONDITION_FALSE, + consts::RESTORE_REFERENT_MISSING_REASON, + GateScope::SnapshotOrRestore, + GateSeverity::Warn, + ), ( consts::SEEDED_CONDITION, CONDITION_FALSE, @@ -967,6 +1002,47 @@ mod tests { } } + /// The #393 park must be visible to `doctor` on a `Restore` — and must not + /// make any OTHER condition read as a gate. + /// + /// The registry's coarse [`StructuralGate::trips`] filter is + /// reason-agnostic, so a row registered on a condition other reconcilers + /// also write (`Ready`, `Resolved`, …) would make every unrelated reason on + /// that condition report as "a gate from a newer operator". This row's + /// condition is therefore written by exactly one gate and one reason. + #[test] + fn the_restore_referent_gate_is_restore_scoped_and_owns_its_condition() { + let row = STRUCTURAL_GATES + .iter() + .find(|g| g.reason == consts::RESTORE_REFERENT_MISSING_REASON) + .expect("the #393 referent-missing park must be registered"); + // Reachable from a Restore (the only scope whose `covers_restore` is true). + assert!(row.applies_to.covers_restore()); + assert_eq!(row.applies_to, GateScope::SnapshotOrRestore); + // Warn: referents applied moments apart by GitOps resolve themselves. + assert_eq!(row.severity, GateSeverity::Warn); + assert_eq!(row.condition, consts::RESTORE_REFERENT_AVAILABLE_CONDITION); + assert!(row.trips(consts::RESTORE_REFERENT_AVAILABLE_CONDITION, "False")); + assert!(!row.trips(consts::RESTORE_REFERENT_AVAILABLE_CONDITION, "True")); + // Exactly one row owns this condition, so the coarse filter can only + // fire for the one reason this build writes on it. + assert_eq!( + STRUCTURAL_GATES + .iter() + .filter(|g| g.condition == consts::RESTORE_REFERENT_AVAILABLE_CONDITION) + .count(), + 1 + ); + // ...and it is NOT the `Ready` condition, which every Restore writes with + // many reasons — registering there would misreport all of them. + assert_ne!(row.condition, consts::READY_CONDITION); + assert!( + !STRUCTURAL_GATES + .iter() + .any(|g| g.condition == consts::READY_CONDITION) + ); + } + #[test] fn every_scope_classifier_is_consistent() { // Each scope covers exactly one of the four kind families a consumer diff --git a/crates/api/src/maintenance.rs b/crates/api/src/maintenance.rs index d0e3f5c2..e14011d1 100644 --- a/crates/api/src/maintenance.rs +++ b/crates/api/src/maintenance.rs @@ -609,7 +609,24 @@ pub struct ManualRunStatus { #[serde(default, skip_serializing_if = "Option::is_none")] pub phase: Option, /// RFC3339 instant the run reached a terminal phase. - #[serde(default, skip_serializing_if = "Option::is_none")] + // Deliberately serialized EVEN WHEN `None` (no `skip_serializing_if`), for + // the same reason as `common::ReplicationManualRunStatus::completed_at`: a + // non-terminal phase emits `"completedAt": null` so the merge-patch CLEARS + // the previous run's stamp instead of leaving it standing over a fresh + // `Running` (#394). The apiserver answers that null by deleting the key + // (plain RFC-7386) or by storing the null verbatim — a nullable CRD field on + // k8s 1.33 was observed doing the latter — and both decode back to `None`, + // so no stale timestamp survives either way. Maintenance patches `manualRun` + // unconditionally (no noop guard), so here the stake is a truthful status + // rather than a non-converging write loop. + // + // This depends on `patch_status` sending `kube::api::Patch::Merge` + // (`crates/controller/src/io/apply.rs`). Under `Patch::Apply` an explicit + // null does NOT clear the field, and this contract silently breaks. + // + // Kept a plain comment rather than rustdoc on purpose: doc comments become + // the CRD `description` (`kubectl explain`, docs/field-reference.md). + #[serde(default)] pub completed_at: Option, } @@ -677,6 +694,64 @@ mod tests { } } + #[test] + fn manual_run_status_roundtrips_camel_case_and_nulls_a_missing_completion() { + // Parsed the cluster's way (YAML -> serde_json::Value -> typed), which + // is the only path that proves the camelCase wire names land. + let status: MaintenanceStatus = from_yaml( + "manualRun:\n requestedAt: 2026-06-11T12:00:00Z\n mode: full\n phase: Succeeded\n completedAt: 2026-06-11T12:01:42Z\n", + ); + let manual = status.manual_run.expect("manualRun decodes"); + assert_eq!(manual.requested_at.as_deref(), Some("2026-06-11T12:00:00Z")); + assert_eq!(manual.mode, Some(ManualRunMode::Full)); + assert_eq!(manual.phase, Some(ManualRunPhase::Succeeded)); + assert_eq!(manual.completed_at.as_deref(), Some("2026-06-11T12:01:42Z")); + let json = serde_json::to_value(&manual).unwrap(); + assert_eq!(json["requestedAt"], "2026-06-11T12:00:00Z"); + assert_eq!(json["mode"], "full"); + assert_eq!(json["phase"], "Succeeded"); + assert_eq!(json["completedAt"], "2026-06-11T12:01:42Z"); + + // #394: a non-terminal run emits an EXPLICIT null completedAt, so the + // merge-patch clears the previous run's stamp rather than leaving it + // standing over a fresh `Running`. Whether the apiserver deletes the key + // or stores the null, it decodes back to `None`. + let running = serde_json::to_value(ManualRunStatus { + requested_at: Some("2026-06-11T13:00:00Z".into()), + mode: Some(ManualRunMode::Quick), + phase: Some(ManualRunPhase::Running), + completed_at: None, + }) + .unwrap(); + assert_eq!( + running, + serde_json::json!({ + "requestedAt": "2026-06-11T13:00:00Z", + "mode": "quick", + "phase": "Running", + "completedAt": null, + }) + ); + // The explicit null reads back as "no completion instant". + let back: ManualRunStatus = serde_json::from_value(running).unwrap(); + assert!(back.completed_at.is_none()); + assert_eq!( + serde_json::to_value(ManualRunStatus::default()).unwrap(), + serde_json::json!({ "completedAt": null }) + ); + + // A status that never requested a run still has NO manualRun key — + // the parent field keeps its own skip_serializing_if. + let never: MaintenanceStatus = from_yaml("observedGeneration: 3\n"); + assert!(never.manual_run.is_none()); + assert!( + serde_json::to_value(&never) + .unwrap() + .get("manualRun") + .is_none() + ); + } + #[test] fn lease_identity_is_hostname_safe_and_stable() { let (user, host) = kopia_lease_identity("kopiur/media/My_App.x"); diff --git a/crates/api/src/restore.rs b/crates/api/src/restore.rs index 2fc73666..0dd3814f 100644 --- a/crates/api/src/restore.rs +++ b/crates/api/src/restore.rs @@ -426,13 +426,17 @@ pub struct RestoreStatus { pub timing: Option, /// When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which /// the restore could actually proceed (its repository reached `Ready`, and for a - /// `target.populator` a PVC already claims it), NOT when the Restore was created. If the - /// referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate - /// falls through unverified and the anchor can be stamped as early as creation. Stamped - /// once and then honored verbatim, so the window survives controller restarts and Job pod - /// retries; cleared when a populator re-opens resolution for a re-created claim, so that - /// claim gets the full window again. Absent means the window has not opened yet (or no - /// `policy.waitTimeout` is configured, in which case there is no window to anchor). + /// `target.populator` a PVC already claims it), NOT when the Restore was created. The + /// window does NOT open while the referenced `Repository` object or `fromPolicy` + /// `SnapshotPolicy` doesn't exist: the restore parks in `Pending` + /// (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It + /// DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so + /// `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose + /// mover Job already launched. Stamped once and then honored verbatim, so the window + /// survives controller restarts and Job pod retries; cleared when a populator re-opens + /// resolution for a re-created claim, so that claim gets the full window again. Absent + /// means the window has not opened yet (or no `policy.waitTimeout` is configured, in + /// which case there is no window to anchor). #[serde(default, skip_serializing_if = "Option::is_none")] pub wait_started_at: Option, /// Bytes/files restored so far, patched periodically by the mover. diff --git a/crates/api/src/seed.rs b/crates/api/src/seed.rs index 4bd7b4a6..4242bb14 100644 --- a/crates/api/src/seed.rs +++ b/crates/api/src/seed.rs @@ -36,7 +36,7 @@ //! standing in a GitOps manifest forever. use crate::backend::Backend; -use crate::common::{CredentialProjection, FailurePolicy, RepositoryRef}; +use crate::common::{CredentialProjection, FailurePolicy, MigrateThrottle, RepositoryRef}; use crate::snapshot_replication::PolicyCopyMode; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -246,7 +246,10 @@ pub struct SeedSyncOptions { } /// Migrate-mode tuning for `kopia snapshot migrate`. -#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Default, JsonSchema)] +/// +/// Scalars plus an optional throttle sub-object, so `Eq` but NOT `Copy` +/// ([`Throttle`](crate::common::Throttle) isn't). +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Default, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SeedMigrateOptions { /// `--parallel`: snapshots migrated concurrently (kopia default `1` — @@ -267,6 +270,24 @@ pub struct SeedMigrateOptions { /// operator's back. #[serde(default)] pub policies: PolicyCopyMode, + /// Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + /// two repositories under two kopia connections and `snapshot migrate` has + /// no speed flags of its own, so each side is applied as `kopia repository + /// throttle set` on that side's connection: + /// + /// * `source` caps the REPLICA — the repository named by + /// `spec.seed.from.repository`, opened read-only — overriding **its** + /// `moverDefaults.throttle`; + /// * `destination` caps THIS repository, the one being seeded, overriding + /// **its own** `moverDefaults.throttle`. + /// + /// Each side overrides field by field: a knob set here wins, a knob left + /// unset keeps that repository's default. Absent: both sides use their + /// repository's defaults. Applies only while the seed is armed — an + /// ordinary connect to the now-initialized repository is capped by + /// `moverDefaults.throttle` alone. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub throttle: Option, } /// What the last seed attempt did, pinned on `Repository`/`ClusterRepository` @@ -501,6 +522,74 @@ credentialProjection: ); } + #[test] + fn migrate_throttle_parses_per_side_and_round_trips() { + // Two repositories, two kopia connections, two independent caps — so + // the wire shape has to keep them apart. Parsed the cluster's way + // (YAML -> JSON value -> typed), which is also what proves the + // camelCase knob names survive. + let seed: SeedSpec = from_yaml( + r#" +from: + repository: + name: offsite +migrate: + parallel: 4 + throttle: + source: + downloadBytesPerSecond: 20000000 + readOpsPerSecond: 500 + destination: + uploadBytesPerSecond: 10000000 +"#, + ); + let throttle = seed + .migrate + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .expect("per-side migrate throttle"); + let source = throttle.source.as_ref().expect("source side"); + assert_eq!(source.download_bytes_per_second, Some(20_000_000)); + assert_eq!(source.read_ops_per_second, Some(500)); + assert_eq!( + source.upload_bytes_per_second, None, + "an unset knob stays unset rather than defaulting to a cap" + ); + let destination = throttle.destination.as_ref().expect("destination side"); + assert_eq!(destination.upload_bytes_per_second, Some(10_000_000)); + assert_eq!( + destination.download_bytes_per_second, None, + "the SOURCE side's knobs must not leak into the destination" + ); + + let json = serde_json::to_value(&seed).expect("serialize"); + assert_eq!( + json.pointer("/migrate/throttle/source/downloadBytesPerSecond"), + Some(&serde_json::json!(20_000_000)), + "{json}" + ); + assert_eq!( + json.pointer("/migrate/throttle/destination/uploadBytesPerSecond"), + Some(&serde_json::json!(10_000_000)), + "{json}" + ); + let reparsed: SeedSpec = serde_json::from_value(json).expect("reparse"); + assert_eq!(seed, reparsed); + + // Absent on a migrate seed that does not cap anything: the common case + // must not grow a key (and must not show up in a GitOps diff). + let plain: SeedSpec = from_yaml( + r#" +from: + repository: + name: offsite +migrate: + parallel: 2 +"#, + ); + assert!(plain.migrate.expect("migrate options").throttle.is_none()); + } + #[test] fn unknown_source_variant_is_rejected() { // An externally-tagged enum must refuse a key it does not know rather diff --git a/crates/api/src/snapshot_replication.rs b/crates/api/src/snapshot_replication.rs index da4c31a4..4b59912b 100644 --- a/crates/api/src/snapshot_replication.rs +++ b/crates/api/src/snapshot_replication.rs @@ -20,7 +20,8 @@ //! `Snapshot` CRs themselves) removes replicated data. use crate::common::{ - CredentialProjection, CronSpec, MoverSpec, ReplicationManualRunStatus, RepositoryRef, Retention, + CredentialProjection, CronSpec, MigrateThrottle, MoverSpec, ReplicationManualRunStatus, + RepositoryRef, Retention, }; use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; use kube::CustomResource; @@ -146,8 +147,9 @@ pub struct IdentityMatcher { pub source_path: Option, } -/// Tuning for `kopia snapshot migrate`. Pure scalars, so `Eq`. -#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Default, JsonSchema)] +/// Tuning for `kopia snapshot migrate`. Pure scalars + an optional throttle +/// sub-object, so `Eq` but NOT `Copy` (`Throttle` isn't). +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Default, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MigrateOptions { /// `--parallel`: number of snapshots migrated concurrently (kopia default @@ -158,6 +160,14 @@ pub struct MigrateOptions { /// to the destination. Defaults to [`PolicyCopyMode::None`]. #[serde(default)] pub policies: PolicyCopyMode, + /// Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + /// migrate` has no speed flags, so each side is applied as `kopia repository + /// throttle set` on that side's connection; `source` overrides the source + /// repository's `moverDefaults.throttle` and `destination` the destination + /// repository's, field by field (a field left unset keeps that repository's + /// default). Absent: both sides use their repository's defaults. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub throttle: Option, } /// How `kopia snapshot migrate` treats the source's kopia policies. @@ -552,7 +562,7 @@ suspend: false assert!(ids.include[0].source_path.is_none()); assert_eq!(ids.exclude[0].source_path.as_deref(), Some("/scratch/*")); - let migrate = spec.migrate.expect("migrate set"); + let migrate = spec.migrate.clone().expect("migrate set"); assert_eq!(migrate.parallel, Some(4)); assert_eq!(migrate.policies, PolicyCopyMode::CopyOverwrite); diff --git a/crates/api/src/validate/mod.rs b/crates/api/src/validate/mod.rs index 017c6415..2233b0e5 100644 --- a/crates/api/src/validate/mod.rs +++ b/crates/api/src/validate/mod.rs @@ -256,6 +256,34 @@ pub fn require_min(field: &str, value: i64, min: i64) -> Option }) } +/// Validate every SET knob of a [`Throttle`](crate::common::Throttle): a cap is a +/// positive rate, so each present field must be >= 1. `field` names the block's +/// path for the message (e.g. `"SnapshotReplication spec.migrate.throttle.source"`), +/// and each knob's own camelCase name is appended. +/// +/// A `0` is refused rather than read as "unlimited": kopia's own "no limit" is the +/// ABSENT field, so accepting `0` would give one wire value two plausible meanings +/// (uncapped vs. stalled) — and the one it actually has in kopia is a hard stop. +/// Exhaustively destructured, so a fifth knob cannot be added without deciding its +/// bound here. +pub fn validate_throttle(field: &str, throttle: &crate::common::Throttle) -> Vec { + let crate::common::Throttle { + upload_bytes_per_second, + download_bytes_per_second, + read_ops_per_second, + write_ops_per_second, + } = throttle; + [ + ("uploadBytesPerSecond", upload_bytes_per_second), + ("downloadBytesPerSecond", download_bytes_per_second), + ("readOpsPerSecond", read_ops_per_second), + ("writeOpsPerSecond", write_ops_per_second), + ] + .into_iter() + .filter_map(|(knob, value)| value.and_then(|v| require_min(&format!("{field}.{knob}"), v, 1))) + .collect() +} + /// Validate a `MoverSpec`. `context` names the owning resource for the message (e.g. /// `"Restore mover"`). /// diff --git a/crates/api/src/validate/repository.rs b/crates/api/src/validate/repository.rs index c8848b54..6a48aaf8 100644 --- a/crates/api/src/validate/repository.rs +++ b/crates/api/src/validate/repository.rs @@ -1027,10 +1027,28 @@ pub fn validate_snapshot_replication(spec: &SnapshotReplicationSpec) -> Vec= 1" rule the throttle + // knobs carry everywhere, applied to each side independently so a + // message names the side that is wrong. + if let Some(throttle) = migrate.throttle.as_ref() { + for (side, block) in [ + ("source", throttle.source.as_ref()), + ("destination", throttle.destination.as_ref()), + ] { + if let Some(block) = block { + errs.extend(validate_throttle( + &format!("SnapshotReplication spec.migrate.throttle.{side}"), + block, + )); + } + } + } } if let Some(pruning) = &spec.pruning { // Exhaustive: a new pruning mode cannot compile without deciding its @@ -1434,6 +1452,24 @@ pub fn validate_repository_seed( { errs.push(e); } + // Per-side seed caps, the same "a rate is >= 1" rule the throttle + // knobs carry everywhere. Validated per side so the message names + // the side that is wrong: `source` is the REPLICA being read, + // `destination` this repository — and mixing them up is the likely + // authoring mistake. + if let Some(throttle) = seed.migrate.as_ref().and_then(|m| m.throttle.as_ref()) { + for (side, block) in [ + ("source", throttle.source.as_ref()), + ("destination", throttle.destination.as_ref()), + ] { + if let Some(block) = block { + errs.extend(validate_throttle( + &format!("spec.seed.migrate.throttle.{side}"), + block, + )); + } + } + } } } errs diff --git a/crates/api/src/validate/tests.rs b/crates/api/src/validate/tests.rs index aaf2dc6f..1e00f0d6 100644 --- a/crates/api/src/validate/tests.rs +++ b/crates/api/src/validate/tests.rs @@ -5832,6 +5832,7 @@ fn srepl_spec(source: RepositoryRef, dest: RepositoryRef, cron: &str) -> Snapsho #[test] fn srepl_valid_full_spec_has_no_errors() { + use crate::common::{MigrateThrottle, Throttle}; use crate::snapshot_replication::{ IdentityMatcher, IdentitySelection, MigrateOptions, PolicyCopyMode, Pruning, SelectionSpec, }; @@ -5860,6 +5861,17 @@ fn srepl_valid_full_spec_has_no_errors() { spec.migrate = Some(MigrateOptions { parallel: Some(4), policies: PolicyCopyMode::Copy, + throttle: Some(MigrateThrottle { + source: Some(Throttle { + download_bytes_per_second: Some(20 * 1024 * 1024), + ..Default::default() + }), + destination: Some(Throttle { + upload_bytes_per_second: Some(5 * 1024 * 1024), + write_ops_per_second: Some(50), + ..Default::default() + }), + }), }); spec.pruning = Some(Pruning::Retention(Retention { keep_daily: Some(7), @@ -6139,6 +6151,127 @@ fn srepl_migrate_parallel_zero_is_rejected() { assert!(validate_snapshot_replication(&spec).is_empty()); } +#[test] +fn srepl_migrate_throttle_rejects_non_positive_rates_per_side() { + use crate::common::{MigrateThrottle, Throttle}; + use crate::snapshot_replication::MigrateOptions; + let mut spec = srepl_spec( + srepl_ref(RepositoryKind::Repository, "a", None), + srepl_ref(RepositoryKind::Repository, "b", None), + "0 6 * * *", + ); + // A zero on one side and a negative on the other: BOTH must be reported, and + // each message must name the side + the knob (a per-side cap is useless + // guidance if the operator can't tell which connection is wrong). + spec.migrate = Some(MigrateOptions { + throttle: Some(MigrateThrottle { + source: Some(Throttle { + download_bytes_per_second: Some(0), + ..Default::default() + }), + destination: Some(Throttle { + upload_bytes_per_second: Some(-1), + write_ops_per_second: Some(0), + ..Default::default() + }), + }), + ..Default::default() + }); + let fields: Vec = validate_snapshot_replication(&spec) + .iter() + .filter_map(|e| match e { + ValidationError::InvalidFieldValue { field, .. } => Some(field.clone()), + _ => None, + }) + .collect(); + assert_eq!( + fields, + vec![ + "SnapshotReplication spec.migrate.throttle.source.downloadBytesPerSecond", + "SnapshotReplication spec.migrate.throttle.destination.uploadBytesPerSecond", + "SnapshotReplication spec.migrate.throttle.destination.writeOpsPerSecond", + ], + "every offending knob is named, source side first" + ); + + // 1 is the smallest accepted rate; unset knobs are always fine. + spec.migrate = Some(MigrateOptions { + throttle: Some(MigrateThrottle { + source: Some(Throttle { + download_bytes_per_second: Some(1), + ..Default::default() + }), + destination: Some(Throttle { + upload_bytes_per_second: Some(1), + read_ops_per_second: Some(1), + write_ops_per_second: Some(1), + ..Default::default() + }), + }), + ..Default::default() + }); + assert!(validate_snapshot_replication(&spec).is_empty()); + + // An empty MigrateThrottle constrains nothing and is accepted (both sides + // simply inherit their repository's moverDefaults). + spec.migrate = Some(MigrateOptions { + throttle: Some(MigrateThrottle::default()), + ..Default::default() + }); + assert!(validate_snapshot_replication(&spec).is_empty()); +} + +#[test] +fn srepl_migrate_throttle_roundtrips_camel_case_through_the_apiserver_path() { + // YAML → JSON → typed, the way the cluster parses it (serde_yaml 0.9 would + // mis-encode the nested sub-objects). + use crate::snapshot_replication::SnapshotReplicationSpec; + use crate::testutil::from_yaml; + let spec: SnapshotReplicationSpec = from_yaml( + r#" +sourceRef: { name: nas-primary } +destinationRef: { name: offsite } +schedule: { cron: "0 6 * * *" } +migrate: + parallel: 2 + throttle: + source: + downloadBytesPerSecond: 20971520 + readOpsPerSecond: 200 + destination: + uploadBytesPerSecond: 5242880 + writeOpsPerSecond: 50 +"#, + ); + let t = spec + .migrate + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .expect("migrate.throttle set"); + let src = t.source.as_ref().expect("source side set"); + assert_eq!(src.download_bytes_per_second, Some(20 * 1024 * 1024)); + assert_eq!(src.read_ops_per_second, Some(200)); + assert_eq!(src.upload_bytes_per_second, None); + let dst = t.destination.as_ref().expect("destination side set"); + assert_eq!(dst.upload_bytes_per_second, Some(5 * 1024 * 1024)); + assert_eq!(dst.write_ops_per_second, Some(50)); + + // Re-serializing keeps the camelCase wire shape and elides unset knobs. + let json = serde_json::to_value(&spec).expect("serialize"); + assert_eq!( + json["migrate"]["throttle"]["source"]["readOpsPerSecond"], + 200 + ); + assert!( + json["migrate"]["throttle"]["source"] + .get("uploadBytesPerSecond") + .is_none(), + "unset knobs are elided: {json}" + ); + let reparsed: SnapshotReplicationSpec = serde_json::from_value(json).expect("reparse"); + assert_eq!(spec, reparsed); +} + #[test] fn srepl_retention_that_keeps_nothing_is_rejected() { use crate::snapshot_replication::{MirrorSourcePruning, NoPruning, Pruning}; @@ -6628,6 +6761,66 @@ fn seed_numeric_knobs_and_failure_policy_must_be_positive() { ); } +#[test] +fn a_seed_migrate_throttle_is_rejected_per_side_and_names_the_knob() { + // #374: a cap is a positive rate on either side. Both sides are checked + // independently and the message names the side, because "source" is the + // REPLICA and "destination" is this repository — mixing them up is the + // likely authoring mistake, and a message that only named the knob would + // not help at all. + let errs = validate_repository(&repo_with_seed(&format!( + "{MIGRATE_SEED} migrate:\n throttle:\n source: {{ downloadBytesPerSecond: 0 }}\n \ + destination: {{ uploadBytesPerSecond: -1, writeOpsPerSecond: 0 }}\n" + ))); + let fields: Vec = errs + .iter() + .filter_map(|e| match e { + ValidationError::InvalidFieldValue { field, .. } => Some(field.clone()), + _ => None, + }) + .collect(); + assert_eq!( + fields, + vec![ + "spec.seed.migrate.throttle.source.downloadBytesPerSecond", + "spec.seed.migrate.throttle.destination.uploadBytesPerSecond", + "spec.seed.migrate.throttle.destination.writeOpsPerSecond", + ], + "every offending knob is named, source side first: {errs:?}" + ); + + // 1 is the smallest accepted rate, unset knobs are always fine, and an + // empty block constrains nothing (both sides inherit their repository's + // moverDefaults). + assert!( + validate_repository(&repo_with_seed(&format!( + "{MIGRATE_SEED} migrate:\n throttle:\n source: {{ downloadBytesPerSecond: 1 }}\n \ + destination: {{ uploadBytesPerSecond: 1, readOpsPerSecond: 1, writeOpsPerSecond: 1 }}\n" + ))) + .is_empty() + ); + assert!( + validate_repository(&repo_with_seed(&format!( + "{MIGRATE_SEED} migrate: {{ throttle: {{}} }}\n" + ))) + .is_empty() + ); + + // A throttle on a BLOB seed is refused as an inert field by the existing + // mode-pairing rule — `migrate` belongs to `from.repository`, throttle and + // all — rather than silently doing nothing. + let errs = validate_repository(&repo_with_seed(&format!( + "{BLOB_SEED} migrate: {{ throttle: {{ source: {{ downloadBytesPerSecond: 1 }} }} }}\n" + ))); + assert!( + errs.iter().any(|e| matches!( + e, + ValidationError::SeedTuningNotApplicable { field, .. } if field == "migrate" + )), + "{errs:?}" + ); +} + #[test] fn a_cluster_repository_seed_ref_must_not_carry_a_namespace() { // `validate_repository_ref`'s existing rule reaches the seed source too. diff --git a/crates/controller/Cargo.toml b/crates/controller/Cargo.toml index 41ad9f50..bfdbeec2 100644 --- a/crates/controller/Cargo.toml +++ b/crates/controller/Cargo.toml @@ -51,6 +51,9 @@ rand = "0.10" [dev-dependencies] serial_test = { workspace = true } +# Apply RFC-7386 merge patches the way the apiserver does, so the status-patch +# convergence tests can replay a reconcile loop instead of asserting one pass. +json-patch = { workspace = true } # Collect mock request bodies in the leader-election protocol tests. http-body-util = "0.1" # `test-util` enables paused tokio time (`start_paused`) so the renewal-loss diff --git a/crates/controller/src/cluster_repository.rs b/crates/controller/src/cluster_repository.rs index c19fdb07..800d1433 100644 --- a/crates/controller/src/cluster_repository.rs +++ b/crates/controller/src/cluster_repository.rs @@ -1426,7 +1426,11 @@ async fn bootstrap_cluster_via_mover( // Probe-only (#414): see the namespaced twin. probe_style_launch && !catalog_create_due, repo.spec.create.as_ref(), - repo.spec.mover_defaults.as_ref(), + crate::repo_seed::seed_bootstrap_throttle( + seed_armed, + repo.spec.seed.as_ref(), + repo.spec.mover_defaults.as_ref(), + ), cluster, foreign, read_only, @@ -1670,7 +1674,10 @@ fn cluster_bootstrap_work_spec( // #414: this launch is a pure health probe — see the namespaced twin. probe_only: bool, create: Option<&kopiur_api::common::CreateBehavior>, - mover_defaults: Option<&kopiur_api::common::MoverDefaults>, + // The cap for every connection this bootstrap opens to THIS repository, + // resolved by the caller through `repo_seed::seed_bootstrap_throttle` — + // identical rule to the `Repository` twin, so the two kinds cannot drift. + throttle: kopiur_mover::workspec::ThrottleSpec, cluster: Option<&str>, foreign: ForeignSnapshots, read_only: bool, @@ -1744,7 +1751,7 @@ fn cluster_bootstrap_work_spec( options: MoverOptions::default(), // Bootstrap is a connect/create probe, not a data run: kopia defaults. cache: Default::default(), - throttle: io::throttle_spec(mover_defaults), + throttle, } } @@ -1886,10 +1893,13 @@ async fn finalize_cluster_bootstrap( // #380: fold the seed outcome into the SAME conditions array — the // mover-skew guard has already refused a seed-armed success carrying no // outcome, so a present `result.seed` is proof the image understood it. - let seed_fold = result - .seed - .as_ref() - .map(|outcome| crate::repo_seed::seed_success_fold(outcome, &now)); + let seed_fold = result.seed.as_ref().map(|outcome| { + crate::repo_seed::seed_success_fold( + outcome, + repo.status.as_ref().and_then(|s| s.seed.as_ref()), + &now, + ) + }); let conditions = match seed_fold.as_ref() { Some(fold) => io::upsert_condition( &conditions, @@ -2699,7 +2709,7 @@ mod tests { true, false, None, - None, + Default::default(), None, ForeignSnapshots::Fallback, read_only, @@ -2737,6 +2747,7 @@ mod tests { migrate: None, allow_empty_source: true, resume: false, + replica_throttle: Default::default(), }), ); let carried = armed.seed.expect("the seed rides the op"); @@ -2769,7 +2780,7 @@ mod tests { true, false, None, - None, + Default::default(), cluster, ForeignSnapshots::Fallback, read_only, diff --git a/crates/controller/src/consts.rs b/crates/controller/src/consts.rs index 8793e169..9bf2fd02 100644 --- a/crates/controller/src/consts.rs +++ b/crates/controller/src/consts.rs @@ -25,7 +25,8 @@ pub use kopiur_api::consts::{ OP_RESTORE_TARGET, ORIGIN_LABEL, PRIVILEGED_MOVER_NOT_PERMITTED_REASON, PRIVILEGED_MOVERS_ANNOTATION, READY_CONDITION, RECONCILING_CONDITION, REPOSITORIES_READY_CONDITION, REPOSITORY_NOT_READY_REASON, REPOSITORY_READ_ONLY_REASON, - REPOSITORY_UID_LABEL, REPOSITORY_WRITABLE_CONDITION, RUN_MODE_ANNOTATION, + REPOSITORY_UID_LABEL, REPOSITORY_WRITABLE_CONDITION, RESTORE_REFERENT_AVAILABLE_CONDITION, + RESTORE_REFERENT_FOUND_REASON, RESTORE_REFERENT_MISSING_REASON, RUN_MODE_ANNOTATION, RUN_REQUESTED_ANNOTATION, SCHEDULE_FANOUT_CAPPED_CONDITION, SCHEDULE_LABEL, SCHEDULE_RUNNABLE_CONDITION, SKIP_SNAPSHOT_CLEANUP_ANNOTATION, SNAPSHOT_CLEANUP_FINALIZER, SNAPSHOT_ID_LABEL, SOURCE_PVC_AVAILABLE_CONDITION, SOURCE_PVC_MISSING_REASON, @@ -314,6 +315,13 @@ pub const ORPHANED_PRIME_REAPED_REASON: &str = "OrphanedPrimePvcReaped"; /// `action` for the already-bound no-op / orphan-reap Events: to actually restore into the /// claim, delete it and let it be recreated (keeping its `dataSourceRef`). pub const RECREATE_CLAIM_TO_RESTORE_ACTION: &str = "RecreateClaimToRestore"; +/// Event `action` (remediation hint) for +/// [`RESTORE_REFERENT_MISSING_REASON`]: create the referenced object, or repoint +/// the `Restore` at one that exists. Published on the park TRANSITION only (the +/// park itself re-patches an identical status every 15s, which is a server-side +/// no-op) — it is what replaces the error metric + Warning Event the pre-#393 +/// unverified fall-through produced downstream. +pub const CREATE_RESTORE_REFERENT_ACTION: &str = "CreateRestoreReferentOrFixTheReference"; /// `Snapshot` condition tracking kopia-side pin reconciliation (ADR-0005 §13(c)). /// `True`/`False` mirrors `status.pinned` once a SnapshotPin mover Job ran; diff --git a/crates/controller/src/io/repo.rs b/crates/controller/src/io/repo.rs index f616e974..426115e5 100644 --- a/crates/controller/src/io/repo.rs +++ b/crates/controller/src/io/repo.rs @@ -7,7 +7,7 @@ use kopiur_api::backend::Backend; use kopiur_api::common::{ CatalogBounds, DeletionProtectionSpec, Encryption, IdentityDefaults, MoverDefaults, NamespaceDeletePolicy, PhaseLabel, RepositoryKind, RepositoryMode, RepositoryRef, - ScheduleDefaults, + ScheduleDefaults, Throttle, }; use kopiur_api::preflight::{PreflightInputs, UNKNOWN_AGE}; use kopiur_api::repository::{RepositoryHealthStatus, RepositoryPhase, StorageStats}; @@ -36,6 +36,35 @@ pub fn throttle_spec(defaults: Option<&MoverDefaults>) -> kopiur_mover::workspec kopiur_mover::workspec::ThrottleSpec::from_mover_defaults(defaults) } +/// Resolve ONE side's throttle for a two-repository run (a `SnapshotReplication` +/// migrate, a seed): that side's repository `moverDefaults.throttle` overlaid by +/// the CR's own per-side override, **field by field**. +/// +/// Field-wise, not block-wise, because the two layers answer different questions: +/// the repository default says "never let anything hammer this backend harder +/// than X", the CR override says "and this particular run is the one I want to +/// hold back further". Replacing the whole block would silently drop the repo's +/// other caps the moment a CR pinned one knob — the classic partial-override +/// footgun that `resolve_mover` avoids for every other `moverDefaults` field. +/// +/// Both absent ⇒ an empty spec, which the mover reads as "skip `throttle set` +/// entirely" (kopia keeps whatever limits its config already carries). +pub fn merged_throttle( + defaults: Option<&MoverDefaults>, + override_: Option<&Throttle>, +) -> kopiur_mover::workspec::ThrottleSpec { + let base = throttle_spec(defaults); + let Some(o) = override_ else { return base }; + kopiur_mover::workspec::ThrottleSpec { + upload_bytes_per_second: o.upload_bytes_per_second.or(base.upload_bytes_per_second), + download_bytes_per_second: o + .download_bytes_per_second + .or(base.download_bytes_per_second), + read_ops_per_second: o.read_ops_per_second.or(base.read_ops_per_second), + write_ops_per_second: o.write_ops_per_second.or(base.write_ops_per_second), + } +} + /// The credentials a mover Job needs, sourced from a repository's /// `encryption.passwordSecretRef`. The Secret is mounted as env (`envFrom`), so /// here we only need its name; the key resolution is documented for callers that @@ -882,6 +911,64 @@ mod tests { assert_eq!(names, vec!["repo-pw", "s3-keys"]); // password first, order-stable } + // --- merged_throttle (per-side override over that repo's moverDefaults) --- + + fn throttle(up: Option, down: Option, r: Option, w: Option) -> Throttle { + Throttle { + upload_bytes_per_second: up, + download_bytes_per_second: down, + read_ops_per_second: r, + write_ops_per_second: w, + } + } + + fn defaults_with(t: Throttle) -> MoverDefaults { + MoverDefaults { + throttle: Some(t), + ..Default::default() + } + } + + #[test] + fn merged_throttle_is_empty_when_neither_layer_sets_anything() { + assert!(merged_throttle(None, None).is_empty()); + assert!(merged_throttle(Some(&MoverDefaults::default()), None).is_empty()); + // An override present but all-None is still nothing to apply. + assert!(merged_throttle(None, Some(&Throttle::default())).is_empty()); + } + + #[test] + fn merged_throttle_falls_back_to_the_repository_defaults() { + let d = defaults_with(throttle(Some(1), Some(2), Some(3), Some(4))); + let out = merged_throttle(Some(&d), None); + assert_eq!(out.upload_bytes_per_second, Some(1)); + assert_eq!(out.download_bytes_per_second, Some(2)); + assert_eq!(out.read_ops_per_second, Some(3)); + assert_eq!(out.write_ops_per_second, Some(4)); + } + + #[test] + fn merged_throttle_override_wins_field_by_field() { + // The footgun this exists to prevent: pinning ONE knob on the CR must not + // drop the repository's other three. + let d = defaults_with(throttle(Some(1), Some(2), Some(3), Some(4))); + let o = throttle(Some(99), None, None, Some(88)); + let out = merged_throttle(Some(&d), Some(&o)); + assert_eq!(out.upload_bytes_per_second, Some(99), "override wins"); + assert_eq!(out.download_bytes_per_second, Some(2), "default survives"); + assert_eq!(out.read_ops_per_second, Some(3), "default survives"); + assert_eq!(out.write_ops_per_second, Some(88), "override wins"); + } + + #[test] + fn merged_throttle_override_alone_stands_without_defaults() { + let o = throttle(None, Some(7), None, None); + let out = merged_throttle(None, Some(&o)); + assert_eq!(out.download_bytes_per_second, Some(7)); + assert_eq!(out.upload_bytes_per_second, None); + assert!(!out.is_empty()); + } + #[test] fn mover_creds_secrets_is_names_only_projection() { let backend = Backend::Filesystem(FilesystemBackend { diff --git a/crates/controller/src/io/tests.rs b/crates/controller/src/io/tests.rs index 08b40b5e..f8168794 100644 --- a/crates/controller/src/io/tests.rs +++ b/crates/controller/src/io/tests.rs @@ -820,6 +820,99 @@ fn status_patch_noop_ignores_volatile_message_only_when_message_matches() { assert!(status_patch_is_noop(Some(¤t), &desired)); } +/// The stored status a replication has after a run that finished: the previous +/// request, terminal, with its completion stamp. +fn finished_manual_run() -> serde_json::Value { + serde_json::json!({ + "manualRun": { + "requestedAt": "2026-06-11T12:00:00Z", + "phase": "Succeeded", + "completedAt": "2026-06-11T12:01:42Z", + }, + }) +} + +/// The `desired` payload `patch_manual_run` builds for a FOLLOW-UP request that +/// has not started — built from the typed struct exactly as the controllers do. +fn pending_manual_run_patch() -> serde_json::Value { + let manual = kopiur_api::common::ReplicationManualRunStatus { + requested_at: Some("2026-06-11T13:00:00Z".into()), + phase: Some(kopiur_api::common::ReplicationManualRunPhase::Pending), + completed_at: None, + }; + serde_json::json!({ "manualRun": manual }) +} + +#[test] +fn non_terminal_manual_run_patch_carries_an_explicit_null_completed_at() { + // #394, the fix itself at the layer that consumes it: `Value::Index` maps a + // MISSING key to `Null`, so this has to ask for the key with `.get()` — + // present-and-null is the whole point, and absent is the bug. + let desired = pending_manual_run_patch(); + assert_eq!( + desired["manualRun"].get("completedAt"), + Some(&serde_json::Value::Null), + "a non-terminal manualRun must NAME completedAt so the merge-patch \ + clears the previous run's stamp; got {desired}" + ); +} + +#[test] +fn manual_run_patch_converges_after_clearing_a_stale_completed_at() { + // The #394 trap is a LOOP, not one pass, so this replays one: patch, let the + // apiserver apply RFC-7386, rebuild `current` the way the replication + // controllers do (re-serialize the TYPED status off the refreshed object), + // and demand the second pass write nothing. + // + // Without the explicit null, `desired` simply omits `completedAt`; a merge + // patch never removes what it omits, so the stale stamp SURVIVES, the + // rebuilt `current` still differs from `desired`, and every queued pass + // re-fires a PATCH the apiserver no-ops — forever. + let desired = pending_manual_run_patch(); + let mut stored = finished_manual_run(); + assert!( + !status_patch_is_noop(Some(&stored), &desired), + "answering a NEW request must write" + ); + + // The apiserver applies the merge patch (this is `Patch::Merge`). Given the + // explicit null it either DELETES the key — plain RFC-7386, what + // `json_patch::merge` models — or STORES the null verbatim; a nullable CRD + // field on k8s 1.33 was observed doing the latter. Both outcomes are + // replayed below, because the guard has to converge under either. + json_patch::merge(&mut stored, &desired); + assert!( + stored["manualRun"].get("completedAt").is_none(), + "the merge-patch must have cleared the stale stamp; got {stored}" + ); + + // Next reconcile: `current` is the typed status of the refreshed object, + // re-serialized — the exact round trip `patch_manual_run` performs. + let converges = |stored: &serde_json::Value| { + let typed: kopiur_api::common::ReplicationManualRunStatus = + serde_json::from_value(stored["manualRun"].clone()).expect("stored manualRun decodes"); + let current = serde_json::json!({ "manualRun": typed }); + assert!( + status_patch_is_noop(Some(¤t), &desired), + "the guard must CONVERGE once the stamp is cleared; current {current}, \ + desired {desired}" + ); + }; + converges(&stored); + + // The other apiserver outcome: the null is STORED rather than deleted. It + // decodes to `None` just the same (`#[serde(default)]` on an `Option`), so + // the rebuilt `current` is identical and the guard converges here too. + let stored_null = serde_json::json!({ + "manualRun": { + "requestedAt": "2026-06-11T13:00:00Z", + "phase": "Pending", + "completedAt": null, + }, + }); + converges(&stored_null); +} + #[test] fn terminal_gate_only_on_failed_at_current_generation() { use kopiur_api::RepositoryPhase; @@ -3702,6 +3795,17 @@ const GATE_WRITERS: &[(&str, bool, &str, &str)] = &[ crate::consts::SOURCE_PVC_MISSING_REASON, "snapshot::handle_missing_source_pvc (computed polarity)", ), + // `restore::park_on_missing_referent` via + // io::upsert_gate(&RESTORE_REFERENT_MISSING_GATE, ...) — the tri-state + // readiness gate's `Undetermined` arm (#393); cleared (True, reason + // `RestoreReferentFound`) by `restore::proceed_past_gate` / + // `plan::cleared_referent_conditions` once the referent exists. + ( + crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION, + false, + crate::consts::RESTORE_REFERENT_MISSING_REASON, + "restore::park_on_missing_referent", + ), // `repository::park_on_seed_source` + // `cluster_repository::park_cluster_on_seed_source`, both via // io::upsert_gate(&SEED_SOURCE_NOT_READY_GATE, ...); cleared by diff --git a/crates/controller/src/replication_run.rs b/crates/controller/src/replication_run.rs index 3296e070..798cd484 100644 --- a/crates/controller/src/replication_run.rs +++ b/crates/controller/src/replication_run.rs @@ -620,11 +620,17 @@ mod tests { ReplicationManualRunPhase::Pending, ReplicationManualRunPhase::Running, ] { - assert!( - manual_run_status(&req, phase.clone(), now) - .completed_at - .is_none(), - "{phase:?} has not completed" + let st = manual_run_status(&req, phase.clone(), now); + assert!(st.completed_at.is_none(), "{phase:?} has not completed"); + // #394: and it must SERIALIZE as an explicit null, not vanish — the + // merge-patch only clears a previous run's stamp when the key is + // present with a null value (the apiserver then deletes the key or + // stores the null; either way no stale timestamp survives). + let json = serde_json::to_value(&st).unwrap(); + assert_eq!( + json.get("completedAt"), + Some(&serde_json::Value::Null), + "{phase:?} must clear a stale completedAt; got {json}" ); } } diff --git a/crates/controller/src/repo_seed.rs b/crates/controller/src/repo_seed.rs index 67903500..5642ab74 100644 --- a/crates/controller/src/repo_seed.rs +++ b/crates/controller/src/repo_seed.rs @@ -24,7 +24,7 @@ use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector}; use kube::api::Api; use kopiur_api::backend::Backend; -use kopiur_api::common::{RepositoryKind, RepositoryRef}; +use kopiur_api::common::{MoverDefaults, RepositoryKind, RepositoryRef, Throttle}; use kopiur_api::seed::{SeedMode, SeedSource, SeedSpec, SeedStatus}; use kopiur_mover::bootstrap::SeedOutcome; use kopiur_mover::jobs::VolumeMountSpec; @@ -155,10 +155,63 @@ pub(crate) struct SeedSourceRepository<'a> { pub kind: RepositoryKind, /// The source CR's name. pub name: &'a str, - /// The source's resolved backend/encryption/CA surface. + /// The source's resolved backend/encryption/CA surface — and its + /// `moverDefaults`, which is where the REPLICA side's throttle comes from + /// (kopia's limits are per connection, so the replica's cap can only be the + /// replica's own default, never the seeded repository's). pub repo: &'a ResolvedRepository, } +/// **Pure.** The per-CR override for ONE side of a migrate seed's copy, or +/// `None` when this seed caps nothing (blob mode included — `spec.seed.migrate` +/// is refused alongside `from.backend`, so the block cannot exist there). +/// +/// Two named accessors rather than one indexed helper because the two sides are +/// two different repositories with two different failure modes, and a caller +/// that picked the wrong one would produce a run that caps the link it meant to +/// leave alone. +fn seed_source_throttle(seed: &SeedSpec) -> Option<&Throttle> { + seed.migrate + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.source.as_ref()) +} + +/// The DESTINATION-side counterpart of [`seed_source_throttle`]: the override +/// for THIS repository, the one being seeded. +fn seed_destination_throttle(seed: &SeedSpec) -> Option<&Throttle> { + seed.migrate + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.destination.as_ref()) +} + +/// **Pure.** The throttle a bootstrap work spec carries — the cap that lands on +/// every connection the bootstrap mover opens to THIS repository. +/// +/// Ordinarily that is just this repository's own `moverDefaults.throttle`. While +/// a seed is ARMED it also picks up `spec.seed.migrate.throttle.destination`, +/// field by field: a seeding bootstrap's local connect is the write side of the +/// heaviest transfer kopiur performs, and `kopia snapshot migrate` has no speed +/// flags of its own, so the limits persisted into that connection's config are +/// the only cap it can honor. +/// +/// Gated on `armed` deliberately. The destination override describes the SEED +/// RUN, not the repository; leaving it in force on every later connect to the +/// now-initialized repository would cap routine work with a number chosen for a +/// one-time copy. `armed == false` therefore reproduces the pre-#374 value byte +/// for byte. +pub(crate) fn seed_bootstrap_throttle( + armed: bool, + seed: Option<&SeedSpec>, + defaults: Option<&MoverDefaults>, +) -> kopiur_mover::workspec::ThrottleSpec { + io::merged_throttle( + defaults, + seed.filter(|_| armed).and_then(seed_destination_throttle), + ) +} + /// **Pure.** Build the mover's seed payload from `spec.seed`. /// /// `blob_ca_bundle_pem` is the resolved `tls.caBundleRef` content of a BLOB @@ -181,11 +234,21 @@ pub(crate) fn seed_op_for( source: Option<&SeedSourceRepository<'_>>, resume: bool, ) -> Option { - let from = match (&seed.from, source) { - (SeedSource::Backend(backend), _) => SeedConnectSource::Backend(Box::new( - backend_to_repository_connect(backend, blob_ca_bundle_pem), - )), - (SeedSource::Repository(_), Some(src)) => { + // Resolved together with the source it belongs to, so the two can never + // describe different repositories: kopia's limits are per CONNECTION, and + // the replica's cap comes from the REPLICA's own `moverDefaults.throttle` + // (overlaid field-wise by `spec.seed.migrate.throttle.source`) — never from + // the repository being seeded. Blob mode caps its copy through `sync-to`'s + // own speed flags instead, so it carries none. + let (from, replica_throttle) = match (&seed.from, source) { + (SeedSource::Backend(backend), _) => ( + SeedConnectSource::Backend(Box::new(backend_to_repository_connect( + backend, + blob_ca_bundle_pem, + ))), + Default::default(), + ), + (SeedSource::Repository(_), Some(src)) => ( SeedConnectSource::Repository(Box::new(SeedRepositoryConnect { kind: io::repo_kind_str(src.kind).to_string(), name: src.name.to_string(), @@ -194,8 +257,9 @@ pub(crate) fn seed_op_for( &src.repo.backend, src.repo.ca_bundle_pem.clone(), ), - })) - } + })), + io::merged_throttle(src.repo.mover_defaults.as_ref(), seed_source_throttle(seed)), + ), (SeedSource::Repository(_), None) => return None, }; Some(SeedOpSpec { @@ -211,13 +275,14 @@ pub(crate) fn seed_op_for( max_download_speed_bytes_per_second: s.max_download_speed_bytes_per_second, max_upload_speed_bytes_per_second: s.max_upload_speed_bytes_per_second, }), - migrate: seed.migrate.map(|m| SeedMigrateSpec { + migrate: seed.migrate.as_ref().map(|m| SeedMigrateSpec { parallel: m.parallel, latest_only: m.latest_only, policies: crate::snapshot_replication::policy_copy_mode_spec(m.policies), }), allow_empty_source: seed.allow_empty_source, resume, + replica_throttle, }) } @@ -318,14 +383,43 @@ pub(crate) struct SeedSuccess { /// storage, not manifests, so there is no per-snapshot copy count; its /// `snapshotCount` is the SOURCE listing the mover took before the copy, which /// for a byte-for-byte mirror is also what this repository ends up holding). -pub(crate) fn seed_success_fold(outcome: &SeedOutcome, now: &str) -> SeedSuccess { +/// +/// # Why `existing` — the fold must be BYTE-STABLE across re-folds +/// +/// `finalize_bootstrap` re-enters on EVERY reconcile for as long as the +/// finished bootstrap Job lingers (it is deleted only on the probe arm), and it +/// re-reads the same result ConfigMap each time — so a seeding Job's outcome is +/// re-folded, unchanged, hundreds of times. Two properties make that pass a +/// no-op under [`crate::io::status_patch_is_noop`], and both need `existing`: +/// +/// * `seededAt` is stamped ONCE, at the first fold, and reused verbatim +/// afterwards. Re-stamping `now` made every pass a fresh status write, which +/// bumped `resourceVersion`, re-triggered this reconciler through its own +/// primary watch and spun the repository at ~4 reconciles/second until the +/// Job's TTL. (A seed happens exactly once, so "when it completed" is a fact, +/// not a heartbeat — the same rule `seed_marker_patch` already applies to +/// `startedAt`.) +/// * the seed-attempt marker (`startedAt`) is carried FORWARD into the patch. +/// A merge patch would preserve it either way, but the no-op guard compares +/// the whole `seed` value: a partial sub-object can never equal the stored +/// one, so omitting the marker re-writes forever on its own. +pub(crate) fn seed_success_fold( + outcome: &SeedOutcome, + existing: Option<&SeedStatus>, + now: &str, +) -> SeedSuccess { let mode = seed_mode_of(outcome); let mut status = serde_json::json!({ "mode": mode, "source": outcome.source, }); + if let Some(started_at) = existing.and_then(|s| s.started_at.as_deref()) { + status["startedAt"] = serde_json::Value::String(started_at.to_string()); + } if outcome.performed { - status["seededAt"] = serde_json::Value::String(now.to_string()); + // Set once; every later re-fold of the same lingering result reuses it. + let seeded_at = existing.and_then(|s| s.seeded_at.as_deref()).unwrap_or(now); + status["seededAt"] = serde_json::Value::String(seeded_at.to_string()); if let Some(n) = outcome.snapshot_count { status["snapshotCount"] = serde_json::json!(n); } @@ -1125,6 +1219,101 @@ mod tests { } } + /// The #374 seed glue guard: two repositories, two kopia connections, two + /// independently resolved caps — and neither side's knobs may reach the + /// other. A crossed merge is invisible in production until the wrong link + /// gets saturated, which is exactly what this feature exists to prevent. + #[test] + fn seed_throttles_resolve_per_side_from_each_repositorys_own_defaults() { + use kopiur_api::common::{MoverDefaults, Throttle}; + let throttle = |up, down, r, w| Throttle { + upload_bytes_per_second: up, + download_bytes_per_second: down, + read_ops_per_second: r, + write_ops_per_second: w, + }; + // The REPLICA brings its own defaults; so does the repository being + // seeded. The CR overrides ONE knob per side, so a correct merge shows + // the override AND that side's surviving default — and nothing else. + let mut src = resolved_source(s3("source-bucket"), Some("backups")); + src.mover_defaults = Some(MoverDefaults { + throttle: Some(throttle(None, Some(1), Some(2), None)), + ..Default::default() + }); + let local_defaults = MoverDefaults { + throttle: Some(throttle(Some(3), None, None, Some(4))), + ..Default::default() + }; + let seed = seed_spec(serde_json::json!({ + "from": { "repository": { "name": "offsite" } }, + "migrate": { "throttle": { + "source": { "downloadBytesPerSecond": 11 }, + "destination": { "uploadBytesPerSecond": 22 }, + } }, + })); + let source = SeedSourceRepository { + kind: RepositoryKind::Repository, + name: "offsite", + repo: &src, + }; + let op = seed_op_for(&seed, None, Some(&source), false).expect("migrate op"); + // REPLICA side: the CR's download override wins, the replica repo's + // readOps default survives, and NOTHING from the local repository's + // defaults leaks in. + assert_eq!(op.replica_throttle.download_bytes_per_second, Some(11)); + assert_eq!(op.replica_throttle.read_ops_per_second, Some(2)); + assert_eq!( + op.replica_throttle.upload_bytes_per_second, None, + "the SEEDED repository's upload default must not reach the replica" + ); + assert_eq!( + op.replica_throttle.write_ops_per_second, None, + "the SEEDED repository's writeOps default must not reach the replica" + ); + + // LOCAL (destination) side rides the bootstrap work spec's own throttle: + // the CR's upload override wins, this repository's writeOps default + // survives, and the REPLICA's knobs stay out. + let armed = seed_bootstrap_throttle(true, Some(&seed), Some(&local_defaults)); + assert_eq!(armed.upload_bytes_per_second, Some(22)); + assert_eq!(armed.write_ops_per_second, Some(4)); + assert_eq!( + armed.download_bytes_per_second, None, + "the REPLICA's download default must not reach the seeded repository" + ); + assert_eq!(armed.read_ops_per_second, None); + + // NOT armed — every ordinary bootstrap, including every later connect to + // the now-initialized repository: byte-for-byte this repository's own + // defaults, with the seed-run override nowhere in sight. + let unarmed = seed_bootstrap_throttle(false, Some(&seed), Some(&local_defaults)); + assert_eq!(unarmed.upload_bytes_per_second, Some(3)); + assert_eq!(unarmed.write_ops_per_second, Some(4)); + assert!(unarmed.download_bytes_per_second.is_none()); + + // A migrate seed that caps nothing leaves both sides on their + // repositories' defaults (and an all-empty pair skips `throttle set`). + let plain = seed_spec(serde_json::json!({ + "from": { "repository": { "name": "offsite" } }, + })); + let op = seed_op_for(&plain, None, Some(&source), false).expect("migrate op"); + assert_eq!(op.replica_throttle.download_bytes_per_second, Some(1)); + assert!(seed_bootstrap_throttle(true, Some(&plain), None).is_empty()); + + // BLOB mode has no source repository CR at all — its copy is capped by + // `sync-to`'s own speed flags — so the replica block stays empty rather + // than inheriting this repository's defaults. + let blob = seed_spec(serde_json::json!({ + "from": { "backend": { "s3": { "bucket": "offsite" } } }, + "sync": { "maxDownloadSpeedBytesPerSecond": 20000000 }, + })); + let op = seed_op_for(&blob, None, None, false).expect("blob op"); + assert!(op.replica_throttle.is_empty()); + // …and admission refuses `migrate` beside `from.backend`, so a blob seed + // can never carry a destination override either. + assert!(seed_bootstrap_throttle(true, Some(&blob), None).is_empty()); + } + #[test] fn a_migrate_seed_without_a_resolved_source_yields_no_payload() { // Unreachable from the caller (it resolves or parks first) and @@ -1316,7 +1505,7 @@ mod tests { 412, Some(412), ); - let fold = seed_success_fold(&performed, now); + let fold = seed_success_fold(&performed, None, now); assert_eq!(fold.reason, kopiur_api::consts::SEEDED_REASON); assert_eq!(fold.outcome, crate::metrics::SeedOutcomeLabel::Seeded); assert_eq!(fold.status["seed"]["seededAt"], serde_json::json!(now)); @@ -1334,7 +1523,7 @@ mod tests { // manifests — and the post-seed catalog listing reports the count on // storageStats instead. let blob = SeedOutcome::performed(SeedModeSpec::Blob, "S3".into(), 7, None); - let fold = seed_success_fold(&blob, now); + let fold = seed_success_fold(&blob, None, now); assert_eq!(fold.status["seed"]["snapshotCount"], serde_json::json!(7)); assert!(fold.status["seed"].get("snapshotsCopied").is_none()); @@ -1342,7 +1531,7 @@ mod tests { // reporting 0 would be a lie), its own reason and metric label, and NO // Event — nothing happened. let noop = SeedOutcome::already_initialized(SeedModeSpec::Blob, "S3".into()); - let fold = seed_success_fold(&noop, now); + let fold = seed_success_fold(&noop, None, now); assert_eq!(fold.reason, kopiur_api::consts::ALREADY_INITIALIZED_REASON); assert_eq!( fold.outcome, @@ -1353,6 +1542,94 @@ mod tests { assert!(fold.event.is_none()); } + /// HOT-LOOP REGRESSION (#396 follow-up). `finalize_bootstrap` re-enters on + /// every reconcile while the finished bootstrap Job lingers and re-folds the + /// SAME `SeedOutcome` out of the same result ConfigMap. That steady-state + /// pass must be a no-op under `io::status_patch_is_noop`, or the write bumps + /// `resourceVersion`, the primary watch re-delivers the object and the + /// repository spins until the Job's TTL. + /// + /// Shaped like the e2e's `e2e-seed-mig-dst`: a completed MIGRATE seed on a + /// `Ready` repository. Both failure modes are covered — the wall clock + /// advancing between passes, and the marker (`startedAt`) the stored + /// sub-object carries but the fold used to omit. + #[test] + fn re_folding_a_finished_seed_is_a_byte_stable_no_op() { + use kopiur_api::repository::RepositoryStatus; + + let outcome = SeedOutcome::performed( + SeedModeSpec::Migrate, + "Repository/e2e-seed-mig-src".into(), + 1, + Some(1), + ); + // The marker stamped by `seed_marker_patch` before the seeding Job ran. + let marker = SeedStatus { + started_at: Some("2026-08-26T03:33:20+00:00".into()), + ..Default::default() + }; + + // FIRST fold: nothing recorded yet, so `now` is what gets stamped. + let first = seed_success_fold(&outcome, Some(&marker), "2026-08-26T03:33:26+00:00"); + assert_eq!( + first.status["seed"]["seededAt"], + serde_json::json!("2026-08-26T03:33:26+00:00"), + "the first fold stamps the completion time" + ); + assert_eq!( + first.status["seed"]["startedAt"], + serde_json::json!("2026-08-26T03:33:20+00:00"), + "the seed-attempt marker rides the patch, or the no-op compare below \ + can never hold" + ); + + // What that first pass persisted, read back the way the reconciler reads + // it (`serde_json::to_value(&repo.status)`). + let stored = RepositoryStatus { + phase: Some(kopiur_api::RepositoryPhase::Ready), + observed_generation: Some(1), + unique_id: Some("c36584a9".into()), + backend: Some("Filesystem".into()), + seed: Some(SeedStatus { + started_at: marker.started_at.clone(), + seeded_at: Some("2026-08-26T03:33:26+00:00".into()), + mode: Some(SeedMode::Migrate), + source: Some("Repository/e2e-seed-mig-src".into()), + snapshot_count: Some(1), + snapshots_copied: Some(1), + }), + ..Default::default() + }; + let current = serde_json::to_value(Some(&stored)).expect("status serializes"); + let existing = stored.seed.as_ref(); + + // EVERY later pass, minutes later on the wall clock, is a no-op. + let mut later = serde_json::json!({ + "phase": "Ready", + "backend": "Filesystem", + "uniqueId": "c36584a9", + "observedGeneration": 1, + }); + merge_seed_status( + &mut later, + &seed_success_fold(&outcome, existing, "2026-08-26T03:51:54+00:00").status, + ); + assert!( + io::status_patch_is_noop(Some(¤t), &later), + "the steady-state re-fold must not re-write status (hot loop): {later}" + ); + + // The standing no-op arm carries the marker forward too, so an + // `AlreadyInitialized` repository is equally quiet. + let adopted = SeedOutcome::already_initialized(SeedModeSpec::Blob, "S3".into()); + let fold = seed_success_fold(&adopted, Some(&marker), "2026-08-26T03:51:54+00:00"); + assert_eq!( + fold.status["seed"]["startedAt"], + serde_json::json!("2026-08-26T03:33:20+00:00") + ); + assert!(fold.status["seed"].get("seededAt").is_none()); + } + #[test] fn merging_the_seed_fold_never_drops_a_key_already_in_the_patch() { // `status_patch` is assembled key-by-key before ONE patch, so a plain @@ -1363,6 +1640,7 @@ mod tests { }); let fold = seed_success_fold( &SeedOutcome::performed(SeedModeSpec::Blob, "S3".into(), 3, None), + None, "2026-08-17T03:00:00Z", ); merge_seed_status(&mut patch, &fold.status); diff --git a/crates/controller/src/repository.rs b/crates/controller/src/repository.rs index 3ca3b969..e9f2ec4f 100644 --- a/crates/controller/src/repository.rs +++ b/crates/controller/src/repository.rs @@ -1353,7 +1353,11 @@ async fn bootstrap_via_mover( // launch — even one that coincides with a due probe — runs in full. probe_style_launch && !catalog_create_due, repo.spec.create.as_ref(), - repo.spec.mover_defaults.as_ref(), + repo_seed::seed_bootstrap_throttle( + seed_armed, + repo.spec.seed.as_ref(), + repo.spec.mover_defaults.as_ref(), + ), cluster, foreign, read_only, @@ -1688,7 +1692,11 @@ fn bootstrap_work_spec( // reports `snapshot_count: None`, decoupling probe cost from catalog size. probe_only: bool, create: Option<&kopiur_api::common::CreateBehavior>, - mover_defaults: Option<&kopiur_api::common::MoverDefaults>, + // The cap for every connection this bootstrap opens to THIS repository, + // resolved by the caller through `repo_seed::seed_bootstrap_throttle`: the + // repository's own `moverDefaults.throttle`, plus — while a seed is armed — + // `spec.seed.migrate.throttle.destination` over it, field by field. + throttle: kopiur_mover::workspec::ThrottleSpec, cluster: Option<&str>, foreign: ForeignSnapshots, read_only: bool, @@ -1766,8 +1774,10 @@ fn bootstrap_work_spec( options: MoverOptions::default(), // Bootstrap is a connect/create probe, not a data run: kopia defaults. cache: Default::default(), - // Apply the repo throttle on the bootstrap connection too (§13(e)). - throttle: io::throttle_spec(mover_defaults), + // Apply the repo throttle on the bootstrap connection too (§13(e)) — + // and, on a seeding bootstrap, on the seed's own local connect, which + // is the only cap `kopia snapshot migrate` can honor on the write side. + throttle, } } @@ -2004,10 +2014,13 @@ async fn finalize_bootstrap( // patch. `bootstrap_outcome` has already refused a seed-armed success that // carries no outcome (the mover-skew guard), so `result.seed` being present // here is proof the running mover understood the request. - let seed_fold = result - .seed - .as_ref() - .map(|outcome| repo_seed::seed_success_fold(outcome, &now)); + let seed_fold = result.seed.as_ref().map(|outcome| { + repo_seed::seed_success_fold( + outcome, + repo.status.as_ref().and_then(|s| s.seed.as_ref()), + &now, + ) + }); let conditions = match seed_fold.as_ref() { Some(fold) => io::upsert_condition( &conditions, @@ -2853,7 +2866,7 @@ mod tests { true, probe_only, None, - None, + Default::default(), None, ForeignSnapshots::Fallback, false, @@ -2890,7 +2903,7 @@ mod tests { true, false, None, - None, + Default::default(), None, ForeignSnapshots::Fallback, read_only, @@ -2927,6 +2940,7 @@ mod tests { migrate: None, allow_empty_source: false, resume: true, + replica_throttle: Default::default(), }; let armed = build(false, Some(op.clone())); let carried = armed.seed.expect("the seed rides the op"); @@ -3036,8 +3050,22 @@ mod tests { }); let prefilter = |cluster: Option<&str>, foreign: ForeignSnapshots| { let spec = bootstrap_work_spec( - &backend, "nas", "billing", true, true, false, None, None, cluster, foreign, false, - true, false, None, None, None, + &backend, + "nas", + "billing", + true, + true, + false, + None, + Default::default(), + cluster, + foreign, + false, + true, + false, + None, + None, + None, ); match spec.operation { Operation::BootstrapRepository(op) => op.catalog_foreign_prefilter_cluster, @@ -3078,7 +3106,7 @@ mod tests { true, false, None, - None, + Default::default(), cluster, ForeignSnapshots::Fallback, read_only, diff --git a/crates/controller/src/restore/mod.rs b/crates/controller/src/restore/mod.rs index d0dc7167..a91e605e 100644 --- a/crates/controller/src/restore/mod.rs +++ b/crates/controller/src/restore/mod.rs @@ -262,11 +262,30 @@ async fn reconcile_inner(restore: &Restore, ctx: &Context) -> Result { // so a transient outage would permanently fail it. A not-Ready repository // therefore DEFERS (requeue); it never fakes a missing-snapshot outcome and // never launches a doomed Job. - if let Some(action) = - gate_on_repository_readiness(ctx, restore, &api, &namespace, &name).await? - { - return Ok(action); - } + // + // Matched exhaustively (CLAUDE.md), and both parking arms return BEFORE + // `ensure_wait_anchor` below — which is the whole of #393: a restore whose + // repository could not even be looked up must not open (and start spending) + // its `waitTimeout` window. + // + // The gate hands back THE `Restore` the rest of this pass must use, and + // `restore` is rebound to it: unchanged in the ordinary case, and a carried + // copy holding the cleared conditions when the gate cleared a stale + // `ReferentAvailable=False`. Every condition writer below rebuilds the array + // from this `restore`, and four of them (the gate parks in + // `run_restore_mover`) patch UNCONDITIONALLY — continuing from the + // reconcile-start copy would re-write the `False` just cleared and alternate + // the two writes on every requeue, forever. See `restore_with_conditions`. + let gated; + let restore = match gate_on_repository_readiness(ctx, restore, &api, &namespace, &name).await? { + RepositoryGate::Proceed(carried) => { + gated = carried; + gated.get() + } + RepositoryGate::Held(action) | RepositoryGate::Undetermined(action) => { + return Ok(action); + } + }; // The claiming PVC, looked up ONCE per pass and shared by everything below that needs // it: the wait-window anchor (a populator with no claim cannot proceed, so its window @@ -3372,21 +3391,24 @@ async fn resolve_restore_repository( /// derivation rule (explicit `spec.repository` wins; `snapshotRef`/`fromPolicy` /// derive from the referent; a raw `identity` source has nothing to derive from). /// -/// Returns `Ok(None)` whenever the ref cannot be determined: a missing -/// `snapshotRef`/`fromPolicy` referent, a Snapshot with neither pin nor repository -/// owner, or `identity` without the (required) explicit `spec.repository`. -/// Deliberately NON-FATAL — a `snapshotRef` whose Snapshot CR does not exist yet is -/// a supported shape (`resolve_snapshot` parks it on the `waitTimeout` window), so -/// erroring here would break it; the other absences produce their canonical -/// `MissingDependency`/`Validation` surfaces downstream. The gate simply does not -/// engage until the repository is known. +/// Returns a [`RepoRefLookup`], not an `Option`: "the ref cannot be determined" +/// is several different situations and they do NOT get the same treatment +/// (issue #393 — the old `Option` collapsed a `SnapshotPolicy` that was never +/// applied together with a snapshot row the restore is legitimately waiting for, +/// and let the gate fall through unverified for both). Each `get_opt` result is +/// matched rather than chained through `and_then`, so "row missing" stays +/// distinct from "row present but underivable"; the classification itself is +/// pure and unit-tested ([`classify_snapshot_lookup`], [`classify_policy_lookup`]). +/// +/// Still deliberately NON-FATAL: nothing here errors. The gate decides what each +/// shape means. async fn restore_repository_ref( ctx: &Context, restore: &Restore, namespace: &str, -) -> Result> { +) -> Result { if let Some(rref) = &restore.spec.repository { - return Ok(Some((rref.clone(), namespace.to_string()))); + return Ok(RepoRefLookup::Derived(rref.clone(), namespace.to_string())); } match &restore.spec.source { RestoreSource::SnapshotRef(sref) => { @@ -3395,36 +3417,77 @@ async fn restore_repository_ref( // — the same base `resolve_restore_repository` uses. let snap_ns = sref.namespace.as_deref().unwrap_or(namespace); let snap_api: Api = Api::namespaced(ctx.client.clone(), snap_ns); - Ok(snap_api - .get_opt(&sref.name) - .await? - .as_ref() - .and_then(repository_ref_from_snapshot) - .map(|rref| (rref, snap_ns.to_string()))) + let snap = snap_api.get_opt(&sref.name).await?; + Ok(classify_snapshot_lookup(snap.as_ref(), snap_ns)) } RestoreSource::FromPolicy(c) => { use kopiur_api::SnapshotPolicy; let cfg_ns = c.namespace.as_deref().unwrap_or(namespace); let cfg_api: Api = Api::namespaced(ctx.client.clone(), cfg_ns); - Ok(cfg_api.get_opt(&c.name).await?.and_then(|cfg| { - // Multi-repo fromPolicy with no explicit selection: the - // readiness gate cannot know which repository to wait on, and - // this contract is deliberately NON-FATAL — return None (never - // guess repository #1); `resolve_restore_repository` fails - // closed downstream with the valid choices listed. - kopiur_api::single_repository_ref(&cfg.spec) - .ok() - .map(|r| (r.clone(), cfg_ns.to_string())) - })) + let cfg = cfg_api.get_opt(&c.name).await?; + Ok(classify_policy_lookup(cfg.as_ref(), cfg_ns, &c.name)) } - RestoreSource::Identity(_) => Ok(None), + // A raw identity source has nothing to derive from, and `spec.repository` + // is REQUIRED for it (checked above and refused downstream): a spec + // problem, never a missing object. + RestoreSource::Identity(_) => Ok(RepoRefLookup::NotDerivable), } } +/// Where the repository-readiness gate leaves a restore on this pass — a +/// TRI-STATE, because "the repository is not Ready" and "kopiur cannot tell +/// whether it is Ready" are different answers with different consequences for +/// the `waitTimeout` window (issue #393). +/// +/// [`Self::Held`] and [`Self::Undetermined`] both park and both requeue; they +/// are separate variants because they are separate *facts*, written with +/// different reasons, and only one of them is a verified statement about a +/// repository. Both stop the reconcile BEFORE [`ensure_wait_anchor`], so +/// neither opens the window. +/// +/// [`WaitWindow`] deliberately gains no variant for this: a parked restore never +/// reaches the window machinery at all, so there is no third window state to +/// model — the window is simply not open yet. +/// +/// **Match this enum EXHAUSTIVELY.** Never `matches!(…)` it and never add a +/// `_ =>` arm — a fourth outcome must force its caller to decide, at compile +/// time, whether it proceeds or parks. COMPILER-ONLY guard: `cargo xtask +/// check-phases` scans only the `*Phase` enums in `kopiur-api`. +enum RepositoryGate<'a> { + /// The gate does not hold this restore: reconcile continues (and the + /// `waitTimeout` window may open). + /// + /// It carries **the `Restore` the rest of the pass must use** — borrowed + /// unchanged in the ordinary case, and an owned copy holding the cleared + /// conditions when this pass cleared a stale `ReferentAvailable=False` + /// ([`restore_with_conditions`]). Returning the object rather than a + /// "…and also remember to apply this" side value is deliberate: the caller + /// cannot obtain a `&Restore` to continue with WITHOUT taking the carried + /// one, so the clear cannot be silently dropped by a later edit. Dropping it + /// would let the unconditional downstream gate parks re-write the stale + /// `False` and alternate two writes forever — see [`restore_with_conditions`]. + /// + /// [`CarriedRestore`], not `Cow`: a `Cow`'s owned variant is stored + /// inline, and `Restore` is large enough that it would bloat every value of + /// this enum past `clippy::large_enum_variant` (denied workspace-wide) while + /// the sibling variants are one `Action` each. `CarriedRestore` boxes only + /// its owned arm, so this stays pointer-sized AND the borrowed path keeps + /// costing nothing. + Proceed(CarriedRestore<'a>), + /// VERIFIED not ready: the repository object exists and its phase is not + /// `Ready` (the backend is unreachable). Park + requeue. + Held(Action), + /// UNDETERMINED: an object the readiness check needs does not exist — the + /// `Repository`/`ClusterRepository` itself, or the `fromPolicy` + /// `SnapshotPolicy` its ref is derived from. Park + requeue; nothing is + /// claimed about the backend. + Undetermined(Action), +} + /// The Restore peer of the Snapshot reconciler's repository-readiness gate: hold a /// not-yet-launched restore in `Pending` (`Ready=False`/`RepositoryNotReady`, /// non-terminal) while its repository is not `Ready`, and requeue on the same 15s -/// cadence. Returns `Some(requeue)` while gated, `None` to proceed. +/// cadence. Returns a [`RepositoryGate`] the caller matches exhaustively. /// /// Two semantic notes, both deliberate: /// - **`spec.mode: ReadOnly` is a different axis.** The Snapshot reconciler's mode @@ -3441,65 +3504,238 @@ async fn restore_repository_ref( /// a repository outage can never fail a restore — or fake a `Continue` empty-volume /// outcome — by silently burning the window while the backend is down. /// +/// **Tri-state (issue #393).** "Ready", "not Ready" and "cannot tell" are three +/// answers, and the gate used to give two: every cannot-tell shape fell through +/// as if the repository had been verified, so `ensure_wait_anchor` stamped the +/// window against nothing. Now a MISSING referent — the `Repository` object +/// itself, or the `fromPolicy` `SnapshotPolicy` its ref derives from — parks as +/// [`RepositoryGate::Undetermined`] with its own reason, no anchor stamped. Two +/// cannot-tell shapes still deliberately proceed: a `snapshotRef` whose +/// `Snapshot` row does not exist yet (that IS what the window waits for, and +/// `onMissingSnapshot: Fail` must be able to fire for a typo'd ref) and a +/// referent that exists but derives no single repository (a spec bug for +/// downstream validation to fail closed on). See [`RepoRefLookup`]. +/// +/// **Wake-up contract.** The `Repository`/`ClusterRepository` → `Restore` watches +/// cover only an explicit `spec.repository` (`watch::repository_to_restores`); +/// there is NO `SnapshotPolicy` → `Restore` watch. An `Undetermined` park +/// therefore un-parks on its own 15s requeue, not on an event — which is why +/// every `Undetermined` arm must return a requeue. (Adding the missing watch is +/// a possible follow-up; the requeue is correct either way.) +/// /// Only a restore that has NOT launched its mover Job is gated /// ([`restore_awaiting_launch`]); a `Restoring` restore's live Job is tracked to /// terminal, never re-gated, mirroring the Snapshot reconciler's ordering. (Narrow /// crash window: a Job created moments before the controller died — before the /// `Restoring` patch landed — is re-gated as `Pending`/`Resolving`; harmless, the /// Job runs to terminal on its own and is observed once the gate opens.) -async fn gate_on_repository_readiness( +async fn gate_on_repository_readiness<'a>( ctx: &Context, - restore: &Restore, + restore: &'a Restore, api: &Api, namespace: &str, name: &str, -) -> Result> { +) -> Result> { if !restore_awaiting_launch(restore.status.as_ref().and_then(|s| s.phase.as_ref())) { - return Ok(None); + return proceed_past_gate(restore, api, name).await; } - let Some((rref, base_ns)) = restore_repository_ref(ctx, restore, namespace).await? else { - return Ok(None); + let (rref, base_ns) = match restore_repository_ref(ctx, restore, namespace).await? { + RepoRefLookup::Derived(rref, base_ns) => (rref, base_ns), + // Both cannot-determine shapes that must NOT park — see the fn doc. + RepoRefLookup::SnapshotRowMissing | RepoRefLookup::NotDerivable => { + return proceed_past_gate(restore, api, name).await; + } + RepoRefLookup::ReferentMissing { + kind, + namespace: referent_ns, + name: referent, + } => { + return park_on_missing_referent( + ctx, + restore, + api, + name, + kind, + referent_ns.as_deref(), + &referent, + ) + .await; + } }; match io::repository_ready_cached(ctx, &rref, &base_ns).await { - Ok(true) => Ok(None), + Ok(true) => proceed_past_gate(restore, api, name).await, Ok(false) => { // patch-if-changed for byte-stability: the parked status is identical // across the 15s requeues, so a re-park is a server-side no-op — no // watch event, no self-trigger (the reconcile hot-loop rule). let current = serde_json::to_value(&restore.status).ok(); + // The repository object EXISTS, so any `ReferentAvailable=False` from + // an earlier park is stale — clear it in this same patch rather than a + // second conditions write (a merge patch replaces the whole array). + let conditions = cleared_referent_conditions(restore) + .unwrap_or_else(|| existing_conditions(restore)); io::patch_status_if_changed( api, name, current.as_ref(), - restore_ready_status( + restore_ready_status_on( restore, + &conditions, RestorePhase::Pending, crate::consts::REPOSITORY_NOT_READY_REASON, &repository_not_ready_restore_message(&rref.name), ), ) .await?; - Ok(Some(Action::requeue(std::time::Duration::from_secs(15)))) + Ok(RepositoryGate::Held(Action::requeue( + std::time::Duration::from_secs(15), + ))) + } + // The repository OBJECT is missing (not merely unreachable): the readiness + // check answered nothing, so this is UNDETERMINED, not "proceed". Pre-#393 + // it fell through here and the wait window opened against an unverified + // repository (#393). + Err(Error::MissingDependency(_)) => { + park_on_missing_referent( + ctx, + restore, + api, + name, + io::repo_kind_str(rref.kind), + repository_referent_namespace(&rref, &base_ns), + &rref.name, + ) + .await } - // The repository OBJECT is missing (not merely unreachable): not this - // gate's call — fall through so the resolve/dispatch paths surface their - // canonical `MissingDependency` hold, exactly as before the gate existed. - Err(Error::MissingDependency(_)) => Ok(None), Err(e) => Err(e), } } +/// The namespace a missing repository referent was looked up in — `None` for a +/// `ClusterRepository`, which is cluster-scoped and whose message must not +/// invent one. +fn repository_referent_namespace<'a>(rref: &'a RepositoryRef, base_ns: &'a str) -> Option<&'a str> { + match rref.kind { + kopiur_api::common::RepositoryKind::Repository => { + Some(rref.namespace.as_deref().unwrap_or(base_ns)) + } + kopiur_api::common::RepositoryKind::ClusterRepository => None, + } +} + +/// Leave the gate open, clearing a stale +/// [`crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION`] +/// `= False` from an earlier park first. +/// +/// The clear is guarded on the condition actually being present-and-not-True, so +/// the overwhelmingly common path (no such condition was ever written) costs +/// nothing and the healthy wire never grows the condition. Without it the +/// registry row — which is deliberately age-independent — would keep reporting a +/// restore that proceeded hours ago as blocked. +/// +/// The cleared array is not just written to the server: the returned +/// [`RepositoryGate::Proceed`] carries a `Restore` holding it, which is the +/// object the rest of the pass must build on. The downstream gate parks rebuild +/// `conditions` from the `Restore` they are handed and patch it +/// UNCONDITIONALLY, so continuing from the reconcile-start copy would re-write +/// the `False` this just cleared — see [`restore_with_conditions`] for the write +/// loop that prevents. Nothing was cleared ⇒ the original is borrowed, no clone. +async fn proceed_past_gate<'a>( + restore: &'a Restore, + api: &Api, + name: &str, +) -> Result> { + let cleared = cleared_referent_conditions(restore); + if let Some(conditions) = &cleared { + io::patch_status(api, name, serde_json::json!({ "conditions": conditions })).await?; + } + // One construction site, pure and total: a cleared pass CANNOT continue from + // the stale object, because that combination is unwritable here. + Ok(RepositoryGate::Proceed(carried_after_clear( + restore, + cleared.as_deref(), + ))) +} + +/// Park a restore whose repository referent does not exist (issue #393): +/// `Pending` + the [`kopiur_api::gates::RESTORE_REFERENT_MISSING_GATE`] condition, +/// requeue 15s, and NO wait anchor — the caller returns before +/// [`ensure_wait_anchor`]. +/// +/// The Warning Event fires only on the park TRANSITION, gated by the same +/// status-changed check as the patch: the park re-patches a byte-identical +/// status every 15s (a server-side no-op), and Eventing that would be a +/// once-per-requeue spam. It compensates for what the pre-#393 fall-through +/// produced downstream — a `MissingDependency` error that `error_policy_for` +/// counted and Evented — which a clean park would otherwise silently remove. +async fn park_on_missing_referent( + ctx: &Context, + restore: &Restore, + api: &Api, + name: &str, + kind: &str, + referent_namespace: Option<&str>, + referent: &str, +) -> Result> { + let msg = referent_missing_restore_message(kind, referent_namespace, referent); + let conditions = io::upsert_gate( + &existing_conditions(restore), + &kopiur_api::gates::RESTORE_REFERENT_MISSING_GATE, + &msg, + restore.metadata.generation, + ); + let current = serde_json::to_value(&restore.status).ok(); + let changed = io::patch_status_if_changed( + api, + name, + current.as_ref(), + restore_ready_status_on( + restore, + &conditions, + RestorePhase::Pending, + crate::consts::RESTORE_REFERENT_MISSING_REASON, + &msg, + ), + ) + .await?; + if changed { + io::publish_warning_event( + ctx, + restore, + crate::consts::RESTORE_REFERENT_MISSING_REASON, + crate::consts::CREATE_RESTORE_REFERENT_ACTION, + &msg, + ) + .await; + } + Ok(RepositoryGate::Undetermined(Action::requeue( + std::time::Duration::from_secs(15), + ))) +} + /// Open the `waitTimeout` window if it is not open yet, and return where it stands for /// THIS pass (#380). /// /// The window is stamped ONCE, into `status.waitStartedAt`, on the first pass that gets /// PAST [`gate_on_repository_readiness`] **and** — for a `target.populator` — has a PVC /// claiming this Restore (`consumer`). "Past the gate" is what the code enforces, and it is -/// weaker than "the repository is Ready": the gate also declines to hold a restore whose -/// mover Job is already live or settled ([`restore_awaiting_launch`]), and one whose -/// `Repository` OBJECT is missing outright. Stamping in those cases is harmless — a live -/// Job keeps the deadline it was dispatched with — and the ordering still guarantees the -/// thing that matters: a restore parked waiting for its backend never opens a window. +/// still slightly weaker than "the repository is Ready", but the gap is now deliberate and +/// narrow (#393). The gate no longer lets a MISSING referent through: a `Repository` +/// object that does not exist, or a `fromPolicy` `SnapshotPolicy` that does not exist, +/// parks as [`RepositoryGate::Undetermined`] and never reaches this function. +/// +/// What still gets past the gate unverified, on purpose: +/// - a restore whose mover Job is already live or settled ([`restore_awaiting_launch`]) — +/// a live Job keeps the deadline it was dispatched with; +/// - a `snapshotRef` whose `Snapshot` ROW does not exist yet — that row is exactly what +/// the window is waiting for, and parking instead would prevent `onMissingSnapshot: +/// Fail` from ever firing for a typo'd ref; +/// - a referent that exists but derives no single repository (a spec bug, failed closed +/// downstream by `resolve_restore_repository`). +/// +/// The ordering still guarantees the thing that matters, and now guarantees it for the +/// slow-arriving-referent case too: a restore parked waiting for its backend — or for the +/// object that names it — never opens a window. /// /// Both halves of the condition are about a restore that cannot yet do anything measuring /// a window it will need later: diff --git a/crates/controller/src/restore/plan.rs b/crates/controller/src/restore/plan.rs index 5dcf7298..c89a3d2a 100644 --- a/crates/controller/src/restore/plan.rs +++ b/crates/controller/src/restore/plan.rs @@ -411,6 +411,241 @@ pub(super) fn existing_conditions(restore: &Restore) -> Vec { .map(|s| s.conditions.clone()) .unwrap_or_default() } + +/// What the repository-readiness gate could learn about the repository a restore +/// will connect to (issue #393) — the return of `super::restore_repository_ref`. +/// +/// The point of the enum is that "no repository ref" is THREE different +/// situations that the pre-#393 `Option<(RepositoryRef, String)>` collapsed into +/// one `None`, and only some of them may fall through the gate unverified: +/// +/// - [`Self::Derived`] — the ref is known; the gate goes on to check readiness. +/// - [`Self::SnapshotRowMissing`] — a `snapshotRef` whose `Snapshot` CR does not +/// exist (yet). Deliberately falls through: the `waitTimeout` window exists +/// precisely to wait for that row, and `onMissingSnapshot: Fail` must be able +/// to fire for a typo'd ref. Parking here would break both. +/// - [`Self::ReferentMissing`] — a `fromPolicy` whose `SnapshotPolicy` does not +/// exist. Nothing downstream can wait for it usefully, and letting the gate +/// fall through stamps `status.waitStartedAt` against a repository nobody +/// verified — the #393 bug. +/// - [`Self::NotDerivable`] — the referent EXISTS but names no single repository +/// (a `Snapshot` with neither pin nor repository owner, a multi-repository +/// `fromPolicy` with no explicit selection, a raw `identity` source). That is a +/// spec problem, not a missing object: it falls through to the downstream +/// validation that fails closed and lists the valid choices. Parking would +/// hide a permanent misconfiguration behind a "waiting…" message. +/// +/// **Match this enum EXHAUSTIVELY.** Never `matches!(…)` it and never add a +/// `_ =>` arm: the compiler's exhaustiveness check is the only thing that makes +/// a fifth shape decide, at compile time, whether it may spend a restore's wait +/// window. This is a COMPILER-ONLY guard: `cargo xtask check-phases` scans only +/// the `*Phase` enums in `kopiur-api`, so nothing but exhaustiveness protects it +/// — which is exactly why the rule is written down here (same convention as the +/// `unreadable_phase` named predicate in `repository_replication`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum RepoRefLookup { + /// The repository ref, plus the namespace it resolves relative to. + Derived(kopiur_api::common::RepositoryRef, String), + /// `snapshotRef` naming a `Snapshot` CR that does not exist — the shape the + /// `waitTimeout` window is FOR. The gate must not engage. + SnapshotRowMissing, + /// A referent object the repository ref is derived FROM does not exist. + /// `namespace` is `None` for a cluster-scoped referent. + ReferentMissing { + /// The referent's kind, for the message (`SnapshotPolicy`, …). + kind: &'static str, + /// The namespace it was looked up in; `None` when cluster-scoped. + namespace: Option, + /// The referent's name. + name: String, + }, + /// The referent exists but yields no single repository ref (or there is + /// nothing to derive from at all): a spec problem for downstream validation. + NotDerivable, +} + +/// Classify a `snapshotRef` lookup: the fetched `Snapshot` (`None` ⇒ the row +/// does not exist) into a [`RepoRefLookup`]. Pure. +/// +/// A missing ROW is [`RepoRefLookup::SnapshotRowMissing`] — a supported, +/// waited-for shape. A row that exists but carries no derivable repository +/// (neither `status.resolved.repository`, nor `spec.repository`, nor a +/// repository `ownerReference`) is [`RepoRefLookup::NotDerivable`]: the object +/// is there, so nothing will appear later to fix it. +pub(super) fn classify_snapshot_lookup( + snapshot: Option<&kopiur_api::Snapshot>, + snapshot_namespace: &str, +) -> RepoRefLookup { + match snapshot { + None => RepoRefLookup::SnapshotRowMissing, + Some(snap) => match kopiur_api::snapshot::repository_ref_for(snap) { + Some(rref) => RepoRefLookup::Derived(rref, snapshot_namespace.to_string()), + None => RepoRefLookup::NotDerivable, + }, + } +} + +/// Classify a `fromPolicy` lookup: the fetched `SnapshotPolicy` (`None` ⇒ the +/// object does not exist) into a [`RepoRefLookup`]. Pure. +/// +/// A missing POLICY is [`RepoRefLookup::ReferentMissing`] — the gate parks. A +/// policy that exists but fans out over several repositories with no explicit +/// selection is [`RepoRefLookup::NotDerivable`]: the gate cannot know which +/// repository to wait on and must never guess repository #1, so it falls through +/// to `resolve_restore_repository`, which fails closed listing the valid choices. +pub(super) fn classify_policy_lookup( + policy: Option<&kopiur_api::SnapshotPolicy>, + policy_namespace: &str, + policy_name: &str, +) -> RepoRefLookup { + match policy { + None => RepoRefLookup::ReferentMissing { + kind: "SnapshotPolicy", + namespace: Some(policy_namespace.to_string()), + name: policy_name.to_string(), + }, + Some(cfg) => match kopiur_api::single_repository_ref(&cfg.spec) { + Ok(rref) => RepoRefLookup::Derived(rref.clone(), policy_namespace.to_string()), + Err(_) => RepoRefLookup::NotDerivable, + }, + } +} + +/// Message for a restore parked because a referent it derives its repository +/// from does not exist (issue #393). Pure so the text is unit-asserted. +/// +/// Says what is missing (kind + namespaced name), why that blocks the restore, +/// that the `waitTimeout` window is deliberately NOT running meanwhile, and how +/// to clear it. +pub(super) fn referent_missing_restore_message( + kind: &str, + namespace: Option<&str>, + name: &str, +) -> String { + let target = match namespace { + Some(ns) => format!("{ns}/{name}"), + None => name.to_string(), + }; + format!( + "waiting for {kind} `{target}` to exist: the repository this restore connects to is \ + derived from it, so kopiur cannot verify the backend is reachable and will not launch \ + the restore. The `policy.waitTimeout` window is NOT running while this is missing \ + (status.waitStartedAt stays unstamped) — it opens once the {kind} exists and its \ + repository becomes `Ready`. Create the {kind} (or repoint the Restore at one that \ + exists) to proceed." + ) +} + +/// A stale [`crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION`] = `False` +/// left by an earlier park, flipped back to `True` — or `None` when there is +/// nothing to clear (the healthy wire never grows the condition). Pure. +/// +/// Without this the park's gate condition outlives the park: the registry row is +/// age-independent, so `kubectl kopiur doctor` would keep reporting a restore +/// that has long since proceeded as blocked on a referent that has existed for +/// hours. Mirrors how the `MoverPermitted`/`CredentialsAvailable` gates clear. +pub(super) fn cleared_referent_conditions(restore: &Restore) -> Option> { + use crate::consts::{RESTORE_REFERENT_AVAILABLE_CONDITION, RESTORE_REFERENT_FOUND_REASON}; + let existing = existing_conditions(restore); + if !existing + .iter() + .any(|c| c.type_ == RESTORE_REFERENT_AVAILABLE_CONDITION && c.status != "True") + { + return None; + } + Some(io::upsert_condition( + &existing, + RESTORE_REFERENT_AVAILABLE_CONDITION, + true, + RESTORE_REFERENT_FOUND_REASON, + "the referent the restore derives its repository from now exists", + restore.metadata.generation, + )) +} + +/// `restore` with `conditions` substituted — the in-memory mirror of a status +/// patch this pass has ALREADY made, so the rest of the pass builds on what the +/// server now holds instead of the reconcile-start copy. Pure. +/// +/// This is what keeps [`cleared_referent_conditions`] from starting a write +/// loop. Nearly every condition writer downstream of the readiness gate rebuilds +/// the array from the `restore` it was handed (a merge patch replaces the array +/// wholesale, so it has to), and four of them — the `MissingCaBundle`, +/// `MissingServiceAccount`, `PrivilegedMover` and `MissingCredentials` gate parks +/// in `run_restore_mover` — patch UNCONDITIONALLY. Against a reconcile-start copy +/// that still carries `ReferentAvailable=False`, a clear written earlier in the +/// same pass would be re-written back to `False` by that park, then cleared again +/// next pass: two resourceVersion-bumping writes per iteration, each waking the +/// watch, forever — for exactly the GitOps bring-up (repository up, credentials +/// Secret not yet) this feature exists to serve. Those parks are byte-identical +/// no-ops today ONLY because nothing writes conditions before them; carrying the +/// cleared copy forward preserves that. +pub(super) fn restore_with_conditions(restore: &Restore, conditions: Vec) -> Restore { + let mut carried = restore.clone(); + let mut status = carried.status.take().unwrap_or_default(); + status.conditions = conditions; + carried.status = Some(status); + carried +} + +/// The `Restore` the rest of a reconcile pass must build on: the one the +/// reconcile was handed, or — when the readiness gate cleared a stale +/// `ReferentAvailable=False` — an owned copy carrying the cleared conditions. +/// +/// A hand-written two-variant carrier rather than `Cow<'a, Restore>`, for one +/// concrete reason: `Restore` is a large struct, so a `Cow` stores the owned +/// variant INLINE and every value of the enum carrying it (here +/// [`super::RepositoryGate`], whose other variants are one `Action` each) grows +/// to that size — `clippy::large_enum_variant`, denied workspace-wide. Boxing +/// the whole `Cow` (clippy's suggestion) would allocate on the borrowed path +/// too, which is the overwhelmingly common one and must stay free. Boxing only +/// [`Self::Cleared`] keeps the carrier pointer-sized in both arms: the unchanged +/// path is still a bare borrow with no clone and no allocation, and the cleared +/// path pays one `Box` on top of a `Restore` clone it was already making. +/// +/// [`Self::get`] is an exhaustive `match`, so a third carrier shape would have +/// to say which object the pass continues with. +#[derive(Debug)] +pub(super) enum CarriedRestore<'a> { + /// Nothing was cleared: the reconcile's own `Restore`, borrowed. + Unchanged(&'a Restore), + /// The gate cleared a stale gate condition: this copy mirrors the status + /// patch it just made, and is what every later writer must rebuild from. + Cleared(Box), +} + +impl CarriedRestore<'_> { + /// The `Restore` to continue the pass with. Exhaustive. + pub(super) fn get(&self) -> &Restore { + match self { + Self::Unchanged(restore) => restore, + Self::Cleared(restore) => restore, + } + } +} + +/// The [`CarriedRestore`] for this pass, given what +/// [`cleared_referent_conditions`] produced: the original borrowed when nothing +/// was cleared (the overwhelmingly common path — no clone), an owned copy +/// carrying the cleared conditions when something was. Pure and TOTAL. +/// +/// The single construction site for the readiness gate's proceed payload, so the +/// "cleared but continued from the stale copy" combination — the one that starts +/// the alternating-write loop described on [`restore_with_conditions`] — has no +/// place to be written. Both arms are unit-asserted. +pub(super) fn carried_after_clear<'a>( + restore: &'a Restore, + cleared: Option<&[Condition]>, +) -> CarriedRestore<'a> { + match cleared { + None => CarriedRestore::Unchanged(restore), + Some(conditions) => CarriedRestore::Cleared(Box::new(restore_with_conditions( + restore, + conditions.to_vec(), + ))), + } +} + /// Where the `waitTimeout` window is anchored: `status.waitStartedAt` once /// [`super::ensure_wait_anchor`] has stamped it, else `created_epoch`. /// diff --git a/crates/controller/src/restore/tests.rs b/crates/controller/src/restore/tests.rs index b765e582..540db08a 100644 --- a/crates/controller/src/restore/tests.rs +++ b/crates/controller/src/restore/tests.rs @@ -142,6 +142,246 @@ fn repository_not_ready_restore_message_says_what_why_how() { ); } +// --- #393: the readiness gate's tri-state repository lookup --------------- + +/// A `Snapshot` fixture, parsed the cluster's way. `pin` is the +/// `spec.repository` mint-time pin (`None` ⇒ nothing derivable at all). +fn snapshot_fixture(pin: Option<&str>) -> kopiur_api::Snapshot { + let mut spec = serde_json::json!({ "sources": [ { "pvc": { "name": "data" } } ] }); + if let Some(name) = pin { + spec["repository"] = serde_json::json!({ "kind": "Repository", "name": name }); + } + serde_json::from_value(serde_json::json!({ + "apiVersion": "kopiur.home-operations.com/v1alpha1", + "kind": "Snapshot", + "metadata": { "name": "b", "namespace": "apps" }, + "spec": spec, + })) + .expect("Snapshot fixture") +} + +/// A `SnapshotPolicy` fixture: one repository (`repository`) or a +/// multi-repository fan-out (`repositories`), which has no single ref. +fn policy_fixture(repositories: &[&str]) -> kopiur_api::SnapshotPolicy { + let mut spec = serde_json::json!({ "sources": [ { "pvc": { "name": "data" } } ] }); + match repositories { + [one] => spec["repository"] = serde_json::json!({ "kind": "Repository", "name": one }), + many => { + spec["repositories"] = serde_json::json!( + many.iter() + .map(|n| serde_json::json!({ "kind": "Repository", "name": n })) + .collect::>() + ) + } + } + serde_json::from_value(serde_json::json!({ + "apiVersion": "kopiur.home-operations.com/v1alpha1", + "kind": "SnapshotPolicy", + "metadata": { "name": "cfg", "namespace": "apps" }, + "spec": spec, + })) + .expect("SnapshotPolicy fixture") +} + +/// The distinction the pre-#393 chained `get_opt(..).and_then(..)` erased: a +/// `Snapshot` ROW that does not exist is a supported, waited-for shape; a row +/// that exists but names no repository is a spec problem. +#[test] +fn snapshot_lookup_separates_a_missing_row_from_an_underivable_one() { + // Missing row: the shape the `waitTimeout` window is FOR. The gate must not + // engage, or `onMissingSnapshot: Fail` could never fire for a typo'd ref. + assert_eq!( + classify_snapshot_lookup(None, "apps"), + RepoRefLookup::SnapshotRowMissing + ); + // Row present WITH a pin: derived, relative to the snapshot's namespace. + let pinned = snapshot_fixture(Some("nas")); + match classify_snapshot_lookup(Some(&pinned), "apps") { + RepoRefLookup::Derived(rref, ns) => { + assert_eq!(rref.name, "nas"); + assert_eq!(ns, "apps"); + } + other => panic!("a pinned Snapshot must derive its repository: {other:?}"), + } + // Row present WITHOUT a pin (no status, no spec.repository, no repository + // owner): the object is there, so nothing will appear later to fix it — + // falls through to downstream validation rather than parking forever. + assert_eq!( + classify_snapshot_lookup(Some(&snapshot_fixture(None)), "apps"), + RepoRefLookup::NotDerivable + ); +} + +/// The other half: a `SnapshotPolicy` that does not exist parks the gate; one +/// that exists but fans out over several repositories does not (the gate must +/// never guess repository #1). +#[test] +fn policy_lookup_separates_a_missing_policy_from_a_multi_repo_one() { + assert_eq!( + classify_policy_lookup(None, "apps", "cfg"), + RepoRefLookup::ReferentMissing { + kind: "SnapshotPolicy", + namespace: Some("apps".into()), + name: "cfg".into(), + } + ); + let single = policy_fixture(&["nas"]); + match classify_policy_lookup(Some(&single), "apps", "cfg") { + RepoRefLookup::Derived(rref, ns) => { + assert_eq!(rref.name, "nas"); + assert_eq!(ns, "apps"); + } + other => panic!("a single-repository policy must derive its repository: {other:?}"), + } + // Multi-repo with no explicit selection: `resolve_restore_repository` fails + // closed downstream listing the valid choices — parking would hide that. + assert_eq!( + classify_policy_lookup(Some(&policy_fixture(&["a", "b"])), "apps", "cfg"), + RepoRefLookup::NotDerivable + ); +} + +#[test] +fn referent_missing_message_says_what_why_and_how() { + let msg = referent_missing_restore_message("SnapshotPolicy", Some("apps"), "cfg"); + // WHAT is missing, namespaced. + assert!(msg.contains("SnapshotPolicy `apps/cfg`"), "{msg}"); + // WHY it blocks: the repository is derived from it and cannot be verified. + assert!(msg.contains("derived from it"), "{msg}"); + // The #393 promise itself: the window is NOT running meanwhile. + assert!(msg.contains("waitTimeout"), "{msg}"); + assert!(msg.contains("status.waitStartedAt"), "{msg}"); + // HOW to clear it. + assert!(msg.contains("Create the SnapshotPolicy"), "{msg}"); + // A cluster-scoped referent must not be given an invented namespace. + let cluster = referent_missing_restore_message("ClusterRepository", None, "offsite"); + assert!(cluster.contains("ClusterRepository `offsite`"), "{cluster}"); + assert!(!cluster.contains('/'), "{cluster}"); + // The reason is distinct from the not-Ready one: a SnapshotPolicy that was + // never applied is not an unreachable backend. + assert_ne!( + crate::consts::RESTORE_REFERENT_MISSING_REASON, + crate::consts::REPOSITORY_NOT_READY_REASON + ); + assert_eq!( + crate::consts::RESTORE_REFERENT_MISSING_REASON, + "RestoreReferentMissing" + ); +} + +/// The park's gate condition must not outlive the park: the registry row is +/// age-independent, so a stale `ReferentAvailable=False` would keep `kubectl +/// kopiur doctor` reporting a restore that proceeded hours ago as blocked. +#[test] +fn the_referent_gate_condition_clears_only_when_it_is_stale() { + use crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION; + // Never parked: nothing to clear, and the healthy wire must not GROW the + // condition (a write per pass would be pure churn). + assert!(cleared_referent_conditions(&restore_with_condition("Resolved", "True")).is_none()); + // Parked: flipped back to True in place, keeping the array a single row. + let parked = restore_with_condition(RESTORE_REFERENT_AVAILABLE_CONDITION, "False"); + let cleared = cleared_referent_conditions(&parked).expect("a stale park must clear"); + let row = cleared + .iter() + .find(|c| c.type_ == RESTORE_REFERENT_AVAILABLE_CONDITION) + .expect("the condition survives, flipped"); + assert_eq!(row.status, "True"); + assert_eq!(row.reason, crate::consts::RESTORE_REFERENT_FOUND_REASON); + // Already cleared: idempotent, so the clear cannot flip-flop with the + // condition writers that rebuild from the reconcile-start copy. + let healthy = restore_with_condition(RESTORE_REFERENT_AVAILABLE_CONDITION, "True"); + assert!(cleared_referent_conditions(&healthy).is_none()); +} + +/// The write-loop guard: a pass that clears the referent gate must carry the +/// cleared conditions forward, or the UNCONDITIONAL gate parks downstream +/// (`run_restore_mover`'s `MissingCaBundle`/`MissingServiceAccount`/ +/// `PrivilegedMover`/`MissingCredentials` writes) rebuild the array from the +/// reconcile-start copy and put `ReferentAvailable=False` straight back. +/// +/// That alternation is not cosmetic: both writes bump `resourceVersion`, each +/// wakes the watch and re-enqueues immediately, so the pair repeats forever (two +/// writes + an Event per iteration) for as long as the Secret/SA/ConfigMap is +/// missing — precisely the GitOps bring-up this feature serves. Those parks are +/// byte-identical no-ops today only because nothing writes conditions ahead of +/// them; this test pins the property that keeps that true. +/// +/// The CALLER half of the contract is enforced by the compiler rather than here: +/// `RepositoryGate::Proceed` carries the `CarriedRestore` to continue with, so +/// `reconcile_inner` cannot get a `&Restore` for the rest of the pass without +/// taking the carried one. This test pins the mechanism that carrying provides. +#[test] +fn a_cleared_referent_condition_survives_a_downstream_gate_park() { + use crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; + let status_of = |conds: &[Condition]| { + conds + .iter() + .find(|c| c.type_ == RESTORE_REFERENT_AVAILABLE_CONDITION) + .map(|c| c.status.clone()) + }; + + let parked = restore_with_condition(RESTORE_REFERENT_AVAILABLE_CONDITION, "False"); + let cleared = cleared_referent_conditions(&parked).expect("a stale park clears"); + // Built through the one construction site the gate uses, so this exercises + // the real seam rather than a re-implementation of it. + let carrier = carried_after_clear(&parked, Some(&cleared)); + assert!( + matches!(carrier, CarriedRestore::Cleared(_)), + "a cleared pass must carry an OWNED copy, never the stale borrow" + ); + let carried = carrier.get().clone(); + + // What a downstream gate park computes from the CARRIED copy: the clear holds. + let after_park = io::upsert_gate( + &existing_conditions(&carried), + &kopiur_api::gates::MISSING_CREDENTIALS_GATE, + "the credentials Secret is not in the mover namespace", + carried.metadata.generation, + ); + assert_eq!( + status_of(&after_park).as_deref(), + Some("True"), + "carrying the cleared conditions forward must survive an unconditional \ + downstream gate park: {after_park:?}" + ); + // ...and the same park computed from the reconcile-start copy is the write + // that used to alternate with the clear. Asserted so this test fails loudly + // if the clobber ever stops being reproducible (i.e. the guard goes vacuous). + let clobbered = io::upsert_gate( + &existing_conditions(&parked), + &kopiur_api::gates::MISSING_CREDENTIALS_GATE, + "the credentials Secret is not in the mover namespace", + parked.metadata.generation, + ); + assert_eq!( + status_of(&clobbered).as_deref(), + Some("False"), + "the reconcile-start copy still carries the stale park — that is the write \ + the carried copy exists to prevent" + ); + + // The carried copy is also a fixed point: a second pass over it clears + // nothing, so the loop cannot restart from the other side. + assert!(cleared_referent_conditions(&carried).is_none()); + // Carrying preserves everything else about the object (only conditions move). + assert_eq!(carried.metadata.generation, parked.metadata.generation); + assert_eq!(carried.spec, parked.spec); + + // Both arms of the construction site: nothing cleared ⇒ the original is + // BORROWED (the common path pays no clone and changes nothing). + let untouched = restore_with_condition("Resolved", "True"); + let same = carried_after_clear( + &untouched, + cleared_referent_conditions(&untouched).as_deref(), + ); + assert!( + matches!(same, CarriedRestore::Unchanged(_)), + "the common path must borrow: no clone, no allocation, nothing changed" + ); + assert_eq!(same.get().status, untouched.status); +} + #[test] fn populator_state_depends_on_target_variant() { use kopiur_api::PopulatorTarget; diff --git a/crates/controller/src/snapshot_replication.rs b/crates/controller/src/snapshot_replication.rs index 14337178..291ee67a 100644 --- a/crates/controller/src/snapshot_replication.rs +++ b/crates/controller/src/snapshot_replication.rs @@ -1275,7 +1275,20 @@ pub fn build_snapshot_replication_work_spec( ) -> MoverWorkSpec { let selection = repl.spec.selection.as_ref(); let (include, exclude) = selection_matchers(repl); - let migrate = repl.spec.migrate.unwrap_or_default(); + let migrate = repl.spec.migrate.clone().unwrap_or_default(); + // `snapshot migrate` has no speed flags: each side's cap must ride its own + // kopia connection, so resolve TWO specs — the source's from the source + // repository's `moverDefaults` and the destination's from the destination's, + // each overlaid field-wise by that side of `spec.migrate.throttle`. + let migrate_throttle = migrate.throttle.as_ref(); + let source_throttle = io::merged_throttle( + source.mover_defaults.as_ref(), + migrate_throttle.and_then(|t| t.source.as_ref()), + ); + let destination_throttle = io::merged_throttle( + dest.mover_defaults.as_ref(), + migrate_throttle.and_then(|t| t.destination.as_ref()), + ); MoverWorkSpec { version: 1, operation: Operation::SnapshotReplicate(SnapshotReplicateOp { @@ -1297,6 +1310,7 @@ pub fn build_snapshot_replication_work_spec( parallel: migrate.parallel, policies: policy_copy_mode_spec(migrate.policies), pruning: pruning_spec(repl.spec.pruning.as_ref()), + destination_throttle, }), // Replication does not snapshot; a stable sentinel identity (like // maintenance / blob replication). @@ -1315,7 +1329,7 @@ pub fn build_snapshot_replication_work_spec( hook_plan: Default::default(), options: MoverOptions::default(), cache: Default::default(), - throttle: io::throttle_spec(source.mover_defaults.as_ref()), + throttle: source_throttle, } } @@ -1831,17 +1845,47 @@ mod tests { r.spec.migrate = Some(MigrateOptions { parallel: Some(4), policies: PolicyCopyMode::CopyOverwrite, + // Per-side caps, each PARTIAL, over repositories that both carry + // their own `moverDefaults.throttle` below — so this asserts the + // full two-layer, two-side resolution, not just "the field arrived". + throttle: Some(kopiur_api::common::MigrateThrottle { + source: Some(kopiur_api::common::Throttle { + download_bytes_per_second: Some(11), + ..Default::default() + }), + destination: Some(kopiur_api::common::Throttle { + upload_bytes_per_second: Some(22), + ..Default::default() + }), + }), }); r.spec.pruning = Some(Pruning::Retention( serde_json::from_value(serde_json::json!({ "keepDaily": 7, "keepWeekly": 4 })).unwrap(), )); - let ws = build_snapshot_replication_work_spec( - &r, - &sample_source(), - &sample_dest(), - "ns", - "offsite", - ); + // Each repository brings its OWN moverDefaults.throttle; the CR's + // per-side override must win field-by-field WITHOUT dropping the rest, + // and the two sides must never cross (a source default landing on the + // destination block would be invisible in production until the wrong + // link got saturated). + let mut source = sample_source(); + source.mover_defaults = Some(kopiur_api::common::MoverDefaults { + throttle: Some(kopiur_api::common::Throttle { + download_bytes_per_second: Some(1), + read_ops_per_second: Some(2), + ..Default::default() + }), + ..Default::default() + }); + let mut dest = sample_dest(); + dest.mover_defaults = Some(kopiur_api::common::MoverDefaults { + throttle: Some(kopiur_api::common::Throttle { + upload_bytes_per_second: Some(3), + write_ops_per_second: Some(4), + ..Default::default() + }), + ..Default::default() + }); + let ws = build_snapshot_replication_work_spec(&r, &source, &dest, "ns", "offsite"); match &ws.operation { Operation::SnapshotReplicate(op) => { assert_eq!(op.include.len(), 1); @@ -1859,9 +1903,32 @@ mod tests { } other => panic!("expected retention pruning, got {other:?}"), } + // DESTINATION side: the CR's upload override wins, the dest + // repo's writeOps default survives, and NOTHING from the source + // repo's defaults leaks in. + assert_eq!(op.destination_throttle.upload_bytes_per_second, Some(22)); + assert_eq!(op.destination_throttle.write_ops_per_second, Some(4)); + assert_eq!( + op.destination_throttle.download_bytes_per_second, None, + "the SOURCE repo's download default must not reach the destination" + ); + assert_eq!( + op.destination_throttle.read_ops_per_second, None, + "the SOURCE repo's readOps default must not reach the destination" + ); } other => panic!("expected snapshot-replicate op, got {}", other.kind_str()), } + // SOURCE side rides the work spec's own throttle: CR override wins on + // download, the source repo's readOps default survives, and the DEST + // repo's upload/writeOps defaults stay out. + assert_eq!(ws.throttle.download_bytes_per_second, Some(11)); + assert_eq!(ws.throttle.read_ops_per_second, Some(2)); + assert_eq!( + ws.throttle.upload_bytes_per_second, None, + "the DESTINATION repo's upload default must not reach the source" + ); + assert_eq!(ws.throttle.write_ops_per_second, None); } #[test] diff --git a/crates/e2e/mise.toml b/crates/e2e/mise.toml index ca80f5c1..191f62fc 100644 --- a/crates/e2e/mise.toml +++ b/crates/e2e/mise.toml @@ -108,7 +108,7 @@ docker exec "$NODE" sh -c ' # PVC ROOT is the kopia repo — a subdir under one shared PVC gives no isolation. # Each scenario binds its own PVC over one of these dirs. KEEP IN LOCKSTEP with # crates/e2e/src/consts.rs::REPO_SUBPATHS. - for s in moverdefaults scc-shadow recmeta nsdel-orphan nsdel-delete pin pinrestore populator populator2 popempty popempty2 readonly kstatus verify vfydeep vfyinh vfygate repl-src repl-dst repl-s3-src repl-run-src repl-run-dst projgate hooks gfs errh colocation colocation-off colocation-missing copymethod copymethod-csi copymethod-explicit idxhealth epochparams epochparams-default staging-recover staging-timeout staging-override staging-mismatch massdel-cascade massdel-breaker massdel-prune massdel-single massdel-nooverlap massdel-throttle massdel-outage massdel-heldprune adopt-prune adopt-ignore polcasc-retain polcasc-delete polcasc-simul recrestore waitanchor multipvc-fanout multipvc-group unchanged-dedup unchanged-default srepl-src srepl-dst srepl-coex-src srepl-coex-dst srepl-sel-src srepl-sel-dst srepl-susp-src srepl-susp-dst srepl-run-src srepl-run-dst mrepo-a mrepo-b mrepo-ret-a mrepo-ret-b mrepo-down-a mrepo-down-b mrepo-rest-a mrepo-rest-b seed-primary seed-mirror seed-blob seed-mig-src seed-mig-dst seed-empty-src seed-empty-dst; do + for s in moverdefaults scc-shadow recmeta nsdel-orphan nsdel-delete pin pinrestore populator populator2 popempty popempty2 readonly kstatus verify vfydeep vfyinh vfygate repl-src repl-dst repl-s3-src repl-run-src repl-run-dst projgate hooks gfs errh colocation colocation-off colocation-missing copymethod copymethod-csi copymethod-explicit idxhealth epochparams epochparams-default staging-recover staging-timeout staging-override staging-mismatch massdel-cascade massdel-breaker massdel-prune massdel-single massdel-nooverlap massdel-throttle massdel-outage massdel-heldprune adopt-prune adopt-ignore polcasc-retain polcasc-delete polcasc-simul recrestore waitanchor waitref multipvc-fanout multipvc-group unchanged-dedup unchanged-default srepl-src srepl-dst srepl-coex-src srepl-coex-dst srepl-sel-src srepl-sel-dst srepl-susp-src srepl-susp-dst srepl-run-src srepl-run-dst mrepo-a mrepo-b mrepo-ret-a mrepo-ret-b mrepo-down-a mrepo-down-b mrepo-rest-a mrepo-rest-b seed-primary seed-mirror seed-blob seed-mig-src seed-mig-dst seed-empty-src seed-empty-dst; do mkdir -p "/kopiur-e2e/repos/$s" done # Source dir with one UNREADABLE file for the errorHandling.ignoreFileErrors diff --git a/crates/e2e/src/consts.rs b/crates/e2e/src/consts.rs index e66cc2fe..d5381901 100644 --- a/crates/e2e/src/consts.rs +++ b/crates/e2e/src/consts.rs @@ -144,6 +144,11 @@ pub const REPO_SUBPATHS: &[&str] = &[ // (repository Ready), not at the Restore's creation — needs its own repo, created // SUSPENDED so it is cold until the scenario releases it. "waitanchor", + // #393: the readiness gate must PARK (no `waitStartedAt`) while the repository + // referent does not exist. Distinct from `waitanchor`, whose repository + // pre-exists (suspended): here the `Repository` and the `SnapshotPolicy` are + // created MID-TEST, so the shard must start with no repository at all. + "waitref", // SnapshotReplication (issue #368, crates/e2e/tests/snapshot_replication.rs): // logical fs→fs replication. Each scenario needs its OWN source AND destination // kopia repository so copy/prune/idempotency counts never leak between scenarios: @@ -241,6 +246,20 @@ pub const PV_SRC_EH: &str = "kopiur-e2e-src-eh"; /// PVC (operator namespace) binding [`PV_SRC_EH`]. pub const PVC_SRC_EH: &str = "e2e-src-eh"; +// --- Repository throttle (#374) ------------------------------------------------ +/// The mover's log line proving `moverDefaults.throttle` reached kopia on a +/// connection (`crates/mover/src/main.rs::apply_repository_throttle`). Scenarios +/// assert on THIS, not on timing: kopia's limits only bite cold-cache backend +/// traffic, so a generous cap changes no runtime and a timing-based assertion +/// would be a silent green. Keep in lockstep with the mover's `info!` message. +pub const THROTTLE_APPLIED_LOG: &str = "applied repository throttle"; + +/// A deliberately NON-BINDING throttle (100 MiB/s) for the scenarios that assert +/// the throttle is applied at all. Generous on purpose: it must not move any +/// scenario's runtime, and a low byte cap is punishing out of all proportion on +/// the small-object workloads these fixtures produce. +pub const THROTTLE_BYTES_PER_SECOND: i64 = 104_857_600; + // --- Secrets ------------------------------------------------------------------- /// Filesystem-backend credentials (just `KOPIA_PASSWORD`). pub const SECRET_FS_CREDS: &str = "kopia-creds"; diff --git a/crates/e2e/src/wait.rs b/crates/e2e/src/wait.rs index fd6c57c6..585dc343 100644 --- a/crates/e2e/src/wait.rs +++ b/crates/e2e/src/wait.rs @@ -71,6 +71,111 @@ pub async fn pod_succeeded(client: &Client, ns: &str, name: &str) -> Result<()> Err(anyhow!("pod {ns}/{name} ended {phase}; logs:\n{logs}")) } +/// Per-pod stdout for every non-terminating Pod matching `selector` in `ns`, +/// plus the LAST per-pod log error, if any. +/// +/// Mover Jobs carry their component/instance labels on the POD template as well +/// as the Job, so the same selector a scenario uses to find a mover Job finds +/// the pod that ran it. +/// +/// Kept per-pod rather than concatenated because "how many times did ONE mover +/// log this" is a real question: a seeding bootstrap applies the repository +/// throttle to two different connections, and only a per-pod count can tell that +/// apart from two pods that each applied it once. +/// +/// A pod whose logs cannot be read yet (still `Pending`/`ContainerCreating`) is +/// skipped rather than failing the whole call — one such pod left over from an +/// earlier attempt must not hide a good pod's logs — but its error is RETURNED +/// alongside, so a caller polling to a timeout can report why it never saw what +/// it wanted instead of just "not found". +pub async fn pod_logs_by_pod( + client: &Client, + ns: &str, + selector: &str, +) -> std::result::Result<(Vec<(String, String)>, Option), kube::Error> { + let api: Api = Api::namespaced(client.clone(), ns); + let list = api + .list(&kube::api::ListParams::default().labels(selector)) + .await?; + let mut out = Vec::new(); + let mut last_err = None; + for pod in list + .items + .into_iter() + .filter(|p| p.metadata.deletion_timestamp.is_none()) + { + let Some(name) = pod.metadata.name else { + continue; + }; + match api.logs(&name, &LogParams::default()).await { + Ok(logs) => out.push((name, logs)), + Err(e) => last_err = Some(e), + } + } + Ok((out, last_err)) +} + +/// Concatenated stdout of every non-terminating Pod matching `selector` in `ns`. +/// The whole-population view over [`pod_logs_by_pod`], for assertions about what +/// SOME mover did rather than which attempt did it. +pub async fn pod_logs_for_selector( + client: &Client, + ns: &str, + selector: &str, +) -> std::result::Result { + let (logs, _) = pod_logs_by_pod(client, ns, selector).await?; + Ok(logs.into_iter().map(|(_, l)| l).collect()) +} + +/// Poll until some single pod matching `selector` has logged `needle` at least +/// `times`. +/// +/// The assertion form for "the mover actually did X" when X leaves no trace in +/// any CR status — e.g. the repository throttle (#374), whose only observable is +/// [`crate::consts::THROTTLE_APPLIED_LOG`]. Polls rather than reading once, +/// because the pod may not exist (or may still be starting) when the caller +/// first looks. +/// +/// `times > 1` is what makes a multi-connection flow's assertion non-vacuous: +/// counting PER POD means a second pod that logged the line once cannot stand in +/// for a second connection the one pod under test never capped. +pub async fn wait_for_pod_log_times( + client: &Client, + ns: &str, + selector: &str, + needle: &str, + times: usize, +) -> Result<()> { + crate::wait_until( + &format!("a pod matching `{selector}` logs `{needle}` at least {times}x"), + default_timeout(), + crate::poll_interval(), + || async { + let (logs, last_err) = pod_logs_by_pod(client, ns, selector).await?; + if logs.iter().any(|(_, l)| l.matches(needle).count() >= times) { + return Ok(Some(())); + } + // Never let a log-fetch blip mask a match, but do carry it to the + // timeout message when there was no match to report. + match last_err { + Some(e) => Err(e), + None => Ok(None), + } + }, + ) + .await +} + +/// [`wait_for_pod_log_times`] for the ordinary "logged it at all" case. +pub async fn wait_for_pod_log( + client: &Client, + ns: &str, + selector: &str, + needle: &str, +) -> Result<()> { + wait_for_pod_log_times(client, ns, selector, needle, 1).await +} + /// Fetch a Pod's logs (best-effort context for failure messages). pub async fn pod_logs(client: &Client, ns: &str, name: &str) -> Result { let api: Api = Api::namespaced(client.clone(), ns); diff --git a/crates/e2e/tests/object_store.rs b/crates/e2e/tests/object_store.rs index 4678f984..52980110 100644 --- a/crates/e2e/tests/object_store.rs +++ b/crates/e2e/tests/object_store.rs @@ -496,6 +496,17 @@ async fn s3_maintenance_runs_in_a_mover_job() { let repo_json = { let mut v = s3_repository_json("e2e-s3-maint", "kopiur-maint", "kopia-s3-creds", true); v["spec"]["maintenance"] = serde_json::json!({ "enabled": false }); + // #374: a repository-wide throttle every mover flow must honor. This repo + // drives TWO of the flows that used to ignore it — the BOOTSTRAP Job below + // and the MAINTENANCE Job after it — and both are asserted at the mover's + // own log line further down. Deliberately non-binding (100 MiB/s against + // an in-cluster MinIO), so it proves the wiring without moving runtimes. + v["spec"]["moverDefaults"] = serde_json::json!({ + "throttle": { + "uploadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND, + "downloadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND + } + }); v }; repos @@ -506,6 +517,19 @@ async fn s3_maintenance_runs_in_a_mover_job() { .await .expect("S3 Repository should reach Ready via the bootstrap Job"); + // #374 regression guard, flow 1 of 2: the BOOTSTRAP mover applied the + // repository throttle. Before the fix, bootstrap connected on its own and + // silently dropped `spec.throttle` — the run still succeeded, which is + // exactly why only the mover's own log line can prove it. + kopiur_e2e::wait::wait_for_pod_log( + &client, + E2E_NAMESPACE, + "kopiur.home-operations.com/repository=e2e-s3-maint", + consts::THROTTLE_APPLIED_LOG, + ) + .await + .expect("the bootstrap mover must apply moverDefaults.throttle on its own connect (#374)"); + // Create the explicit Maintenance; the controller spawns a per-slot mover Job. // (The operator honors an externally-authored Maintenance even when the repo // opted out of managed maintenance — ADR §3.7.) @@ -537,6 +561,19 @@ async fn s3_maintenance_runs_in_a_mover_job() { .await .expect("a maintenance mover Job should run to completion"); + // #374 regression guard, flow 2 of 2: the MAINTENANCE mover applied the same + // repository throttle on its own connect. A full maintenance rewrites and + // drops blobs, so this is real backend traffic that used to run uncapped. + kopiur_e2e::wait::wait_for_pod_log( + &client, + E2E_NAMESPACE, + "app.kubernetes.io/component=maintenance,\ + kopiur.home-operations.com/maintenance=e2e-s3-maint", + consts::THROTTLE_APPLIED_LOG, + ) + .await + .expect("the maintenance mover must apply moverDefaults.throttle on its own connect (#374)"); + // The mover ran `kopia maintenance` against S3 and PATCHed the status: the // first run is full (which also stamps quick), and the lease is owned. This // can only be true if maintenance actually executed. diff --git a/crates/e2e/tests/replication.rs b/crates/e2e/tests/replication.rs index 38a082fd..19e2f0e9 100644 --- a/crates/e2e/tests/replication.rs +++ b/crates/e2e/tests/replication.rs @@ -31,6 +31,49 @@ async fn repository_replication_mirrors_to_second_filesystem_repo() { }; world.ensure(&[Need::Filesystem]).await.expect("fixtures"); let client = world.client().clone(); + + // #374: the SOURCE repository carries a repository-wide throttle, so the + // replication mover — which opens its own connect to the source and reads the + // whole repository through it — must apply it. Created HERE rather than + // through `ensure_seed` (which hardcodes an empty overlay and is + // create-if-absent); no other scenario uses this repo, so pre-creating it is + // race-free. Non-binding at 100 MiB/s over a local hostPath. + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + ensure_repo(&client, "repl-src").await; + create_idempotent( + &repos, + &cr(repository_json( + "e2e-repl-src", + "repl-src", + serde_json::json!({ + "moverDefaults": { + "throttle": { + "uploadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND, + "downloadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND + } + } + }), + )), + "create the throttled replication source Repository", + ) + .await; + // Bare spec fields, not a `{"spec": ...}` wrapper — the wrapper is dropped + // silently and would leave the throttle assertion below vacuous. + let landed = repos + .get("e2e-repl-src") + .await + .expect("read back the throttled replication source Repository"); + assert_eq!( + landed + .spec + .mover_defaults + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.upload_bytes_per_second), + Some(consts::THROTTLE_BYTES_PER_SECOND), + "moverDefaults.throttle must survive onto the source Repository" + ); + // A source repo with a real snapshot to mirror. ensure_seed( &client, @@ -139,6 +182,18 @@ async fn repository_replication_mirrors_to_second_filesystem_repo() { .await .expect("the replication should run and stamp status.lastReplicated"); + // #374 regression guard: the replication mover applied the SOURCE + // repository's throttle on its own connect. `sync-to` succeeds either way, + // so the mover's log line is the only proof the cap was in force. + kopiur_e2e::wait::wait_for_pod_log( + &client, + E2E_NAMESPACE, + &selector, + consts::THROTTLE_APPLIED_LOG, + ) + .await + .expect("the replication mover must apply moverDefaults.throttle on its own connect (#374)"); + // kstatus consistency guard (regression: the same two-pass heal bug as // restore/snapshot). The mover stamps `phase: Succeeded` + `lastReplicated`; // the controller heals `Ready=True` in a FOLLOW-UP reconcile. Gate on the @@ -381,6 +436,11 @@ async fn repository_replication_s3_to_s3_uses_destination_scoped_credentials() { /// It pins the halves a unit test cannot reach: the requested run rides the /// ordinary mover path (a real `sync-to`, tagged `run-trigger: manual` on its /// Job), and `status.manualRun` answers the exact timestamp requested. +/// +/// It then pins issue #394 against a REAL apiserver: a follow-up request parked +/// as `Pending` must carry no `completedAt` VALUE from the finished run, which +/// only the explicit-null serialization achieves (the apiserver may delete the +/// key or store the null; either way the stale stamp is gone). #[tokio::test] #[ignore = "requires the e2e harness (mise run //crates/e2e:test): kind + built images + helm install"] async fn repository_replication_runs_on_demand_from_the_run_requested_annotation() { @@ -533,5 +593,61 @@ async fn repository_replication_runs_on_demand_from_the_run_requested_annotation "the requested run's Job must be tagged run-trigger: manual; got {triggers:?}" ); + // Issue #394: a FOLLOW-UP request must not inherit the finished run's + // `completedAt`. The non-terminal patch serializes an explicit null; the + // apiserver then either DELETES the key (plain RFC-7386) or stores the null + // verbatim — a nullable CRD field on k8s 1.33 was observed keeping it. Both + // converge on the property that matters and the assertion below tests + // exactly that: no stale timestamp survives under the fresh `Pending`. + // + // Suspending is what makes the park deterministic (#380 records an + // unanswered request as `Pending` and stops). Ordering hazard: the + // reconciler checks `spec.suspend` BEFORE the run request, but annotating an + // un-suspended CR can spawn the Job and record `Running` before the suspend + // lands — so both fields ride ONE patch. + let second_request = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + repls + .patch( + name, + &PatchParams::default(), + &Patch::Merge(serde_json::json!({ + "metadata": { "annotations": { + kopiur_api::consts::RUN_REQUESTED_ANNOTATION: second_request + } }, + "spec": { "suspend": true } + })), + ) + .await + .expect("suspend and re-request in one patch"); + + let parked = wait_until( + "the follow-up request parks as Pending on the suspended replication", + default_timeout(), + poll_interval(), + || { + let repls = repls.clone(); + let want = second_request.clone(); + async move { + let s = status_json(&repls, name).await; + Ok(s.get("manualRun") + .filter(|m| { + m.get("phase").and_then(|p| p.as_str()) == Some("Pending") + && m.get("requestedAt").and_then(|r| r.as_str()) == Some(want.as_str()) + }) + .cloned()) + } + }, + ) + .await + .expect("the suspended replication should record the new request as Pending"); + assert!( + parked + .get("completedAt") + .is_none_or(serde_json::Value::is_null), + "a non-terminal manualRun must carry NO completedAt VALUE — absent or an \ + explicit null are both fine (the apiserver picks), but the previous run's \ + stamp must not still be standing; got {parked}" + ); + let _ = repls.delete(name, &DeleteParams::default()).await; } diff --git a/crates/e2e/tests/restore.rs b/crates/e2e/tests/restore.rs index bbbba75b..b4ade596 100644 --- a/crates/e2e/tests/restore.rs +++ b/crates/e2e/tests/restore.rs @@ -1808,3 +1808,339 @@ async fn restore_wait_window_opens_when_the_repository_becomes_ready() { cleanup_restore(&restores, WA_RESTORE).await; } + +// --- #393: the readiness gate is TRI-STATE — a MISSING referent parks ------- + +/// Isolated repo for the two referent-park scenarios: unlike [`WA_SUBPATH`], whose +/// `Repository` pre-exists (created suspended), here the repository is created +/// **mid-test** — the whole point is a `Restore` that arrives before the objects it +/// derives its repository from. Subpath in `consts::REPO_SUBPATHS` + the mise +/// node-seed list (LOCKSTEP). +const WR_SUBPATH: &str = "waitref"; +/// Scenario 1: the `Repository` the Restore names explicitly. +const WR_REPO: &str = "e2e-wr-repo"; +const WR_RESTORE: &str = "e2e-r-waitref"; +/// Scenario 2: a `fromPolicy` Restore with NO `spec.repository`, so the repository +/// ref is derived from this `SnapshotPolicy` — which is what goes missing. +const WR_CFG: &str = "e2e-wr-cfg"; +const WR_CFG_REPO: &str = "e2e-wr-cfg-repo"; +const WR_CFG_RESTORE: &str = "e2e-r-waitref-policy"; +/// The `waitTimeout` both scenarios configure. Long enough that the window cannot +/// close mid-test (which would turn "the anchor stamped" into a race), and it only +/// ever starts running after the assertions are done. +const WR_WINDOW: &str = "10m"; +/// How long the park is observed to HOLD with no anchor — comfortably more than the +/// gate's 15s requeue, so a gate that stamps `waitStartedAt` on a later pass (rather +/// than only on the first) still fails this. +const WR_PARK_PROBE_SECS: u64 = 45; + +/// The `Restore` under test for the referent scenarios: a direct `target.pvc` with a +/// `waitTimeout`, plus the caller's `source`/`repository` fields. +fn waitref_restore_json(name: &str, extra_spec: serde_json::Value) -> serde_json::Value { + let mut spec = serde_json::json!({ + "policy": { "waitTimeout": WR_WINDOW }, + "target": { "pvc": { "name": format!("{name}-dst"), "capacity": "1Gi", "accessModes": ["ReadWriteOnce"] } } + }); + if let Some(extra) = extra_spec.as_object() { + for (k, v) in extra { + spec[k] = v.clone(); + } + } + serde_json::json!({ + "apiVersion": "kopiur.home-operations.com/v1alpha1", + "kind": "Restore", + "metadata": { "name": name, "namespace": E2E_NAMESPACE }, + "spec": spec + }) +} + +/// Poll until `status.waitStartedAt` is stamped, returning it. +async fn wait_anchor_stamped(restores: &Api, name: &str) -> anyhow::Result { + wait_until( + &format!("{name} stamps status.waitStartedAt"), + default_timeout(), + poll_interval(), + || { + let restores = restores.clone(); + let name = name.to_string(); + async move { + let s = status_json(&restores, &name).await; + Ok(s.get("waitStartedAt") + .and_then(|v| v.as_str()) + .map(str::to_string)) + } + }, + ) + .await +} + +/// #393 — the wait window must not open against a `Repository` that does not exist. +/// +/// Pre-fix the readiness gate conflated "verified Ready" with three cannot-determine +/// shapes: a missing `Repository` OBJECT made `repository_ready_cached` return +/// `MissingDependency`, the gate fell through as if the repository had been checked, +/// and `ensure_wait_anchor` stamped `status.waitStartedAt` ≈ the Restore's creation. +/// A `Repository` that then took longer than `waitTimeout` to arrive (a GitOps apply +/// ordering, a DR bring-up) found the window already spent. +/// +/// Post-fix the gate parks: `Pending`, `ReferentAvailable=False` / +/// `RestoreReferentMissing`, and NO anchor — asserted over a stable poll so a gate +/// that merely stamps one pass later still fails. Creating the `Repository` releases +/// it (this path also has a watch), and only THEN does the anchor appear. +#[tokio::test] +#[ignore = "requires the e2e harness (mise run //crates/e2e:test)"] +async fn restore_parks_without_wait_anchor_until_its_repository_exists() { + let Some(world) = World::connect().await else { + return; + }; + world + .ensure(&[Need::Filesystem]) + .await + .expect("fixtures ready"); + let client = world.client().clone(); + common::ensure_repo(&client, WR_SUBPATH).await; + + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + let restores: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + + // Clean slate: the scenario is only meaningful while the Repository is ABSENT. + cleanup_restore(&restores, WR_RESTORE).await; + let _ = repos.delete(WR_REPO, &DeleteParams::default()).await; + wait_until( + "the waitref Repository is gone", + default_timeout(), + poll_interval(), + || { + let repos = repos.clone(); + async move { Ok(repos.get_opt(WR_REPO).await?.is_none().then_some(())) } + }, + ) + .await + .expect("the previous run's waitref Repository should be deleted"); + + // A Restore naming a Repository that does not exist. The source snapshotRef is + // deliberately a name that never appears: nothing about it must matter until the + // repository shows up. + restores + .create( + &PostParams::default(), + &cr(waitref_restore_json( + WR_RESTORE, + serde_json::json!({ + "repository": { "kind": "Repository", "name": WR_REPO }, + "source": { "snapshotRef": { "name": "e2e-wr-never" } } + }), + )), + ) + .await + .expect("create the Restore whose Repository does not exist yet"); + + wait_condition(&restores, WR_RESTORE, "ReferentAvailable", "False") + .await + .expect("a Restore whose Repository object is missing must park on the referent gate"); + let parked = status_json(&restores, WR_RESTORE).await; + assert_eq!( + parked.get("phase").and_then(|v| v.as_str()), + Some("Pending"), + "the park is non-terminal: {parked}" + ); + assert_eq!( + condition_reason(&parked, "ReferentAvailable"), + "RestoreReferentMissing", + "the park must use its OWN reason, not RepositoryNotReady (which would claim \ + the backend was checked): {parked}" + ); + assert_eq!( + condition_reason(&parked, "Ready"), + "RestoreReferentMissing", + "the Ready condition carries the same reason: {parked}" + ); + let message = condition_field(&parked, "ReferentAvailable", "message"); + assert!( + message.contains(WR_REPO), + "the message must name the missing object: {message}" + ); + assert!( + parked.get("waitStartedAt").is_none(), + "the wait window must NOT open while the Repository object does not exist: {parked}" + ); + + // ...and it STAYS closed across several requeues (the pre-fix bug stamped on the + // very first pass, but a partial fix could stamp on a later one). + tokio::time::sleep(std::time::Duration::from_secs(WR_PARK_PROBE_SECS)).await; + let still_parked = status_json(&restores, WR_RESTORE).await; + assert!( + still_parked.get("waitStartedAt").is_none(), + "the park must hold the window closed for as long as the Repository is absent: \ + {still_parked}" + ); + assert_eq!( + still_parked.get("phase").and_then(|v| v.as_str()), + Some("Pending"), + "still parked, still non-terminal: {still_parked}" + ); + + // Create the Repository: it bootstraps, reaches Ready, and the gate opens. + repos + .create( + &PostParams::default(), + &cr(common::repository_json( + WR_REPO, + WR_SUBPATH, + serde_json::json!({ "maintenance": { "enabled": false } }), + )), + ) + .await + .expect("create the waitref Repository"); + wait_phase(&repos, WR_REPO, "Ready") + .await + .expect("the waitref Repository should reach Ready"); + + // The anchor stamps only NOW — the same assertion shape as the #380 scenario. + let opened = wait_anchor_stamped(&restores, WR_RESTORE).await.expect( + "the released Restore must stamp status.waitStartedAt once its repository is Ready", + ); + assert!( + chrono::DateTime::parse_from_rfc3339(&opened).is_ok(), + "waitStartedAt {opened} must be RFC3339" + ); + // The gate condition clears rather than lingering as a phantom block for + // `kubectl kopiur doctor`. + wait_condition(&restores, WR_RESTORE, "ReferentAvailable", "True") + .await + .expect("the referent gate must clear once the Repository exists"); + + cleanup_restore(&restores, WR_RESTORE).await; + let _ = repos.delete(WR_REPO, &DeleteParams::default()).await; +} + +/// #393 — the same park for a `fromPolicy` source whose `SnapshotPolicy` is missing, +/// and the un-park path that has NO watch behind it. +/// +/// This Restore sets no `spec.repository`, so the repository ref is DERIVED from the +/// `SnapshotPolicy` — and there is no `SnapshotPolicy` → `Restore` watch +/// (`watch::repository_to_restores` covers only an explicit `spec.repository`). The +/// park therefore has to un-park on its own 15s requeue, which is exactly what this +/// scenario proves: an `Undetermined` arm that forgot to requeue would park here +/// FOREVER, and every phase-based check would report the Restore as merely "Pending". +#[tokio::test] +#[ignore = "requires the e2e harness (mise run //crates/e2e:test)"] +async fn restore_parks_when_frompolicy_names_a_missing_policy() { + let Some(world) = World::connect().await else { + return; + }; + world + .ensure(&[Need::Filesystem]) + .await + .expect("fixtures ready"); + let client = world.client().clone(); + common::ensure_repo(&client, WR_SUBPATH).await; + + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + let configs: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + let restores: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + + // Clean slate: the SnapshotPolicy must be ABSENT when the Restore is created. + cleanup_restore(&restores, WR_CFG_RESTORE).await; + let _ = configs.delete(WR_CFG, &DeleteParams::default()).await; + wait_until( + "the waitref SnapshotPolicy is gone", + default_timeout(), + poll_interval(), + || { + let configs = configs.clone(); + async move { Ok(configs.get_opt(WR_CFG).await?.is_none().then_some(())) } + }, + ) + .await + .expect("the previous run's waitref SnapshotPolicy should be deleted"); + + restores + .create( + &PostParams::default(), + &cr(waitref_restore_json( + WR_CFG_RESTORE, + // No `spec.repository`: the repository is derived from the policy, + // which is the referent under test. + serde_json::json!({ "source": { "fromPolicy": { "name": WR_CFG } } }), + )), + ) + .await + .expect("create the Restore whose SnapshotPolicy does not exist yet"); + + wait_condition(&restores, WR_CFG_RESTORE, "ReferentAvailable", "False") + .await + .expect("a fromPolicy Restore whose SnapshotPolicy is missing must park"); + let parked = status_json(&restores, WR_CFG_RESTORE).await; + assert_eq!( + parked.get("phase").and_then(|v| v.as_str()), + Some("Pending"), + "the park is non-terminal: {parked}" + ); + assert_eq!( + condition_reason(&parked, "ReferentAvailable"), + "RestoreReferentMissing", + "{parked}" + ); + let message = condition_field(&parked, "ReferentAvailable", "message"); + assert!( + message.contains("SnapshotPolicy") && message.contains(WR_CFG), + "the message must name the missing SnapshotPolicy, not a repository: {message}" + ); + assert!( + parked.get("waitStartedAt").is_none(), + "the wait window must NOT open while the SnapshotPolicy does not exist: {parked}" + ); + tokio::time::sleep(std::time::Duration::from_secs(WR_PARK_PROBE_SECS)).await; + let still_parked = status_json(&restores, WR_CFG_RESTORE).await; + assert!( + still_parked.get("waitStartedAt").is_none(), + "the park must hold the window closed for as long as the policy is absent: \ + {still_parked}" + ); + + // Provide the referent (and the repository it names). NOTHING watches + // SnapshotPolicy on behalf of a Restore, so the un-park below can only come from + // the gate's own 15s requeue. + if repos.get_opt(WR_CFG_REPO).await.ok().flatten().is_none() { + let _ = repos + .create( + &PostParams::default(), + &cr(common::repository_json( + WR_CFG_REPO, + WR_SUBPATH, + serde_json::json!({ "maintenance": { "enabled": false } }), + )), + ) + .await; + } + wait_phase(&repos, WR_CFG_REPO, "Ready") + .await + .expect("the waitref policy Repository should reach Ready"); + configs + .create( + &PostParams::default(), + &cr(common::snapshot_policy_json( + E2E_NAMESPACE, + WR_CFG, + "Repository", + WR_CFG_REPO, + serde_json::json!({}), + )), + ) + .await + .expect("create the waitref SnapshotPolicy"); + + // Un-parked by the requeue: the anchor finally opens. + wait_anchor_stamped(&restores, WR_CFG_RESTORE).await.expect( + "the parked Restore must un-park on its own requeue once the SnapshotPolicy exists \ + — there is no SnapshotPolicy watch to wake it", + ); + wait_condition(&restores, WR_CFG_RESTORE, "ReferentAvailable", "True") + .await + .expect("the referent gate must clear once the SnapshotPolicy exists"); + + cleanup_restore(&restores, WR_CFG_RESTORE).await; + let _ = configs.delete(WR_CFG, &DeleteParams::default()).await; + let _ = repos.delete(WR_CFG_REPO, &DeleteParams::default()).await; +} diff --git a/crates/e2e/tests/seed.rs b/crates/e2e/tests/seed.rs index d49ace5a..e6822987 100644 --- a/crates/e2e/tests/seed.rs +++ b/crates/e2e/tests/seed.rs @@ -516,7 +516,33 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident // DIFFERENT from the shared `kopia-creds` password — the whole point. const SEEDED_PASSWORD: &str = "e2e-seed-migrate-password-789"; - // The source repository, with real history under a real identity. + // The source repository, with real history under a real identity. Created + // HERE rather than by `ensure_seed` (which is idempotent and will find it + // present) so the REPLICA can carry its own `moverDefaults.throttle` — the + // base layer the per-CR `seed.migrate.throttle.source` override below has to + // merge over. Caps are deliberately GENEROUS: the point is to prove the + // limits reach kopia on the replica's connection, and a low byte cap is far + // more punitive than nominal on a small-object cold-cache migrate, which + // would only make this scenario slow and flaky. + ensure_repo(&client, "seed-mig-src").await; + { + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + create_idempotent( + &repos, + &cr(repository_json( + SOURCE, + "seed-mig-src", + serde_json::json!({ + "moverDefaults": { "throttle": { + "downloadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND, + "readOpsPerSecond": 10000 + } } + }), + )), + "create the seed SOURCE Repository with moverDefaults.throttle", + ) + .await; + } ensure_seed(&client, SOURCE, POLICY, BACKUP, "seed-mig-src").await; let backups: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); let source_identity = snapshot_identity(&backups, BACKUP).await; @@ -553,9 +579,33 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident // itself, and a seed-armed bootstrap never takes the create // fallback — so reaching Ready here proves both at once. "create": { "enabled": false }, + // #374: a migrate seed opens THREE kopia connections in one pod + // — the replica (read-only), this repository's own seed-local + // client (which `snapshot migrate` reopens, and which has no + // speed flags of its own, making it the ONLY cap on the heaviest + // transfer kopiur performs) and the post-seed bootstrap + // reconnect. All must be capped; the assertion below counts the + // mover's log line per pod. Non-binding at 100 MiB/s over a + // local hostPath. + "moverDefaults": { + "throttle": { + "uploadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND, + "writeOpsPerSecond": 10000 + } + }, "seed": { "from": { "repository": { "kind": "Repository", "name": SOURCE } }, - "migrate": { "parallel": 2 } + // Per-CR, per-side overrides (#374). Each side sets ONE knob, + // so the merged block must also carry the knob that side's + // repository defaulted — the field-wise merge, end to end. + // `source` is the REPLICA, `destination` this repository. + "migrate": { + "parallel": 2, + "throttle": { + "source": { "downloadBytesPerSecond": 209715200 }, + "destination": { "uploadBytesPerSecond": 209715200 } + } + } } }), )), @@ -571,6 +621,51 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident Some(SOURCE), "spec.seed.from.repository must have survived admission" ); + // Bare spec fields, not a `{"spec": ...}` wrapper — read the throttle back + // too, or the log-count assertion below would pass vacuously against a repo + // that never carried a cap. + assert_eq!( + live.spec + .mover_defaults + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.upload_bytes_per_second), + Some(consts::THROTTLE_BYTES_PER_SECOND), + "moverDefaults.throttle must survive onto the seeded Repository" + ); + // The per-side seed overrides too — an apiserver whose CRD schema predates + // them PRUNES them silently, which is the exact #374 symptom this change is + // about, and the log-count assertion below would then pass for the wrong + // reason (the repositories' own defaults alone). + { + let seed = live + .spec + .seed + .as_ref() + .and_then(|s| s.migrate.as_ref()) + .and_then(|m| m.throttle.as_ref()) + .unwrap_or_else(|| { + panic!( + "spec.seed.migrate.throttle survived admission (a pruned field means a stale \ + CRD): {:?}", + live.spec.seed + ) + }); + assert_eq!( + seed.source + .as_ref() + .and_then(|t| t.download_bytes_per_second), + Some(209_715_200), + "the REPLICA-side override must survive: {seed:?}" + ); + assert_eq!( + seed.destination + .as_ref() + .and_then(|t| t.upload_bytes_per_second), + Some(209_715_200), + "the DESTINATION-side override must survive: {seed:?}" + ); + } wait_seeded(&repos, SEEDED, "True", "Seeded").await; wait_phase(&repos, SEEDED, "Ready") @@ -593,6 +688,37 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident "a migrate seed must report the manifests it copied; status.seed: {seed_status}" ); + // #374 regression guard for the highest-value call sites of the fix: every + // connection a migrate seed opens. The arithmetic for THIS pod, which is why + // the number is 3: + // + // probe connect 0 — the backend is uninitialized, so the connect FAILS + // (that failure is what arms the seed) and never + // reaches its throttle; + // replica connect +1 — the read-only source open, capped from + // `replicaThrottle` (source repo defaults ⊕ + // seed.migrate.throttle.source); + // seed-local +1 — this repository under its own kopia config, the + // only cap `snapshot migrate` can honor; + // post-seed +1 — the bootstrap reconnect after the copy. + // + // Counting per pod is load-bearing: the same selector also matches later + // catalog-rescan/probe pods that log it once each, and a bare `contains` (or + // a count across pods) would stay green with any one application deleted. + kopiur_e2e::wait::wait_for_pod_log_times( + &client, + E2E_NAMESPACE, + &format!("kopiur.home-operations.com/repository={SEEDED}"), + consts::THROTTLE_APPLIED_LOG, + 3, + ) + .await + .expect( + "the seeding mover must cap ALL THREE of its connections — the replica read, the \ + seed's local connect and the post-seed reconnect (#374); two occurrences means one \ + of them is uncapped", + ); + // Identity survives the migrate, so a restore keyed on the SOURCE's // identity finds the history in the SEEDED repository. let restores: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); diff --git a/crates/e2e/tests/snapshot_replication.rs b/crates/e2e/tests/snapshot_replication.rs index 22181b17..c8f05a0f 100644 --- a/crates/e2e/tests/snapshot_replication.rs +++ b/crates/e2e/tests/snapshot_replication.rs @@ -48,7 +48,7 @@ use kopiur_api::consts::{ CONFIG_LABEL, ORIGIN_LABEL, REPOSITORY_UID_LABEL, SNAPSHOT_ID_LABEL, SNAPSHOT_REPLICATION_LABEL, }; use kopiur_api::{Repository, Snapshot, SnapshotPolicy, SnapshotReplication}; -use kopiur_e2e::{E2E_NAMESPACE, Need, World, default_timeout, poll_interval, wait_until}; +use kopiur_e2e::{E2E_NAMESPACE, Need, World, consts, default_timeout, poll_interval, wait_until}; /// In-pod mount path every DESTINATION repository pins (see module docs). const DEST_REPO_PATH: &str = "/repo-dst"; @@ -217,6 +217,34 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { // 1. Source history: seed (repo + policy + snapshot 1), then a second // distinct backup under the same policy. + // + // The source repository is created HERE rather than by `ensure_seed` (which + // is idempotent and will find it present) so it can carry its own + // `moverDefaults.throttle` — the base layer the per-CR override below has to + // merge over. Caps are deliberately GENEROUS (100 MiB/s, 10k ops/s): the + // point is to prove the limits reach kopia on both connections, and a low + // byte cap is far more punitive than nominal on a small-object cold-cache + // migrate (a 2 MB/s cap measured ~14s for 28 KiB), which would only make + // this scenario slow and flaky. + ensure_repo(&client, "srepl-src").await; + { + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + create_idempotent( + &repos, + &cr(repository_json( + REPO_SRC, + "srepl-src", + serde_json::json!({ + "moverDefaults": { "throttle": { + "downloadBytesPerSecond": 104857600, + "readOpsPerSecond": 10000 + } } + }), + )), + "create source Repository with moverDefaults.throttle", + ) + .await; + } ensure_seed(&client, REPO_SRC, POLICY, B1, "srepl-src").await; let backups: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); create_idempotent( @@ -274,7 +302,13 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { "srepl-dst", DST_SECRET, serde_json::json!({ - "maintenance": { "enabled": false } + "maintenance": { "enabled": false }, + // The DESTINATION's own defaults — a separate repository, so a + // separate throttle layer (kopia's limits are per connection). + "moverDefaults": { "throttle": { + "uploadBytesPerSecond": 104857600, + "writeOpsPerSecond": 10000 + } } }), ) .await; @@ -291,12 +325,38 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { "spec": { "sourceRef": { "kind": "Repository", "name": REPO_SRC }, "destinationRef": { "kind": "Repository", "name": REPO_DST }, - "schedule": { "cron": "* * * * *" } + "schedule": { "cron": "* * * * *" }, + // Per-CR, per-side overrides (#374). Each side sets ONE knob, so + // the merged block must also carry the knob that side's + // repository defaulted — the field-wise merge, end to end. + "migrate": { "throttle": { + "source": { "downloadBytesPerSecond": 209715200 }, + "destination": { "uploadBytesPerSecond": 209715200 } + } } } })), "create SnapshotReplication", ) .await; + // The CRD must actually carry the new fields — an apiserver whose CRD schema + // predates them PRUNES them silently, which is the exact #374 symptom this + // whole change is about. Read it back before asserting on downstream effects. + { + let live = serde_json::to_value(repls.get(REPL).await.expect("get SnapshotReplication")) + .unwrap_or_default(); + assert_eq!( + live.pointer("/spec/migrate/throttle/source/downloadBytesPerSecond") + .and_then(|v| v.as_u64()), + Some(209715200), + "spec.migrate.throttle survived admission (a pruned field means a stale CRD): {live}" + ); + assert_eq!( + live.pointer("/spec/migrate/throttle/destination/uploadBytesPerSecond") + .and_then(|v| v.as_u64()), + Some(209715200), + "{live}" + ); + } // The per-slot mover Job's inline work-spec env is the controller↔mover // contract seam (mirrors the replication.rs #216 guard): the operation must @@ -363,6 +423,53 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { "unset knob {absent} must be elided from the wire: {op}" ); } + // Both merged throttle blocks ride the wire, one per connection: the SOURCE + // side on the work spec's own `throttle`, the DESTINATION side on the op's + // `destinationThrottle`. Each must show the CR's override AND the knob its + // own repository defaulted — and NOT the other repository's knobs, which is + // how a crossed-sides merge would show up. + let src_throttle = spec + .get("throttle") + .unwrap_or_else(|| panic!("work spec must carry the merged SOURCE throttle: {spec}")); + assert_eq!( + src_throttle + .get("downloadBytesPerSecond") + .and_then(|v| v.as_u64()), + Some(209715200), + "the CR's source override wins: {src_throttle}" + ); + assert_eq!( + src_throttle + .get("readOpsPerSecond") + .and_then(|v| v.as_u64()), + Some(10000), + "the SOURCE repo's moverDefaults knob survives the partial override: {src_throttle}" + ); + assert!( + src_throttle.get("uploadBytesPerSecond").is_none(), + "the DESTINATION repo's upload default must not cross into the source: {src_throttle}" + ); + let dst_throttle = op + .get("destinationThrottle") + .unwrap_or_else(|| panic!("op must carry the merged DESTINATION throttle: {op}")); + assert_eq!( + dst_throttle + .get("uploadBytesPerSecond") + .and_then(|v| v.as_u64()), + Some(209715200), + "the CR's destination override wins: {dst_throttle}" + ); + assert_eq!( + dst_throttle + .get("writeOpsPerSecond") + .and_then(|v| v.as_u64()), + Some(10000), + "the DESTINATION repo's moverDefaults knob survives: {dst_throttle}" + ); + assert!( + dst_throttle.get("readOpsPerSecond").is_none(), + "the SOURCE repo's readOps default must not cross into the destination: {dst_throttle}" + ); // 4. A run succeeds and records lastReplicated + the run counters. The // counter assert tolerates an in-place nextest retry (copies may already @@ -398,6 +505,29 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { "the run must account for both source snapshots (copied {copied} + alreadyPresent {already}): {s1}" ); + // 4b. #374 regression guard: a run that succeeded must ALSO have capped both + // of its connections. The srepl mover opens exactly TWO repositories — + // the source (read-only, under its own kopia config, the only cap + // `snapshot migrate` can honor on the read side) and the destination — + // and neither goes through the generic connect funnel, so each is its own + // chance to ship #374 again. The line is emitted once per APPLICATION and + // only after kopia accepted the limits, so two occurrences in ONE pod can + // only mean both connections were capped; counting per pod (not across + // pods) is what keeps deleting ONE of the two applications from staying + // green on a suite that runs several replication pods. + kopiur_e2e::wait::wait_for_pod_log_times( + &client, + E2E_NAMESPACE, + &selector, + consts::THROTTLE_APPLIED_LOG, + 2, + ) + .await + .expect( + "the snapshot-replication mover must cap BOTH the source and the destination \ + connection (#374); one occurrence means one side's throttle is gone", + ); + // kstatus two-pass heal guard: the mover stamps the terminal status, the // controller heals Ready=True on a FOLLOW-UP reconcile — gate on the healed // condition like every sibling. diff --git a/crates/e2e/tests/verification.rs b/crates/e2e/tests/verification.rs index a0519480..af7f3aa6 100644 --- a/crates/e2e/tests/verification.rs +++ b/crates/e2e/tests/verification.rs @@ -74,6 +74,50 @@ async fn verification_quick_with_success_expr_stamps_last_verified() { }; world.ensure(&[Need::Filesystem]).await.expect("fixtures"); let client = world.client().clone(); + + // #374: this repository carries a repository-wide throttle, so the verify + // mover — which owns its OWN connect and used to drop `spec.throttle` on the + // floor — must apply it. Created HERE rather than through `ensure_seed`, + // which hardcodes an empty overlay and is create-if-absent; this repo is + // used by no other scenario, so pre-creating it is race-free. The cap is + // deliberately non-binding (100 MiB/s over a local hostPath). + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + ensure_repo(&client, "verify").await; + create_idempotent( + &repos, + &cr(repository_json( + "e2e-verify-repo", + "verify", + serde_json::json!({ + "moverDefaults": { + "throttle": { + "uploadBytesPerSecond": kopiur_e2e::consts::THROTTLE_BYTES_PER_SECOND, + "downloadBytesPerSecond": kopiur_e2e::consts::THROTTLE_BYTES_PER_SECOND + } + } + }), + )), + "create the throttled verify Repository", + ) + .await; + // The overlay is BARE spec fields; a `{"spec": ...}` wrapper would be dropped + // by serde AND pruned by the apiserver, leaving a repo with no throttle and a + // test that asserts nothing. Read it back and prove the field landed. + let landed = repos + .get("e2e-verify-repo") + .await + .expect("read back the throttled verify Repository"); + assert_eq!( + landed + .spec + .mover_defaults + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.upload_bytes_per_second), + Some(kopiur_e2e::consts::THROTTLE_BYTES_PER_SECOND), + "moverDefaults.throttle must survive onto the Repository, or the assertion below is vacuous" + ); + // Seed a real snapshot so quick-verify has something to verify. ensure_seed( &client, @@ -135,6 +179,18 @@ async fn verification_quick_with_success_expr_stamps_last_verified() { // mover contract (`operation.verify.tier.quick.*`). let selector = "app.kubernetes.io/component=verify,\ kopiur.home-operations.com/verify=e2e-verify-policy"; + + // #374 regression guard: the verify mover applied the repository throttle on + // its own connect. A capped repo whose verify Job ignores the cap still + // stamps `lastVerified`, so the mover's log line is the only proof. + kopiur_e2e::wait::wait_for_pod_log( + &client, + E2E_NAMESPACE, + selector, + kopiur_e2e::consts::THROTTLE_APPLIED_LOG, + ) + .await + .expect("the verify mover must apply moverDefaults.throttle on its own connect (#374)"); let spec = verify_work_spec_json(&client, selector).await; let quick = spec .pointer("/operation/verify/tier/quick") diff --git a/crates/kopia/src/client/mod.rs b/crates/kopia/src/client/mod.rs index 79d7406e..c157735f 100644 --- a/crates/kopia/src/client/mod.rs +++ b/crates/kopia/src/client/mod.rs @@ -1415,6 +1415,42 @@ impl KopiaClient { self.run_ok(&args).await.map(|_| ()) } + /// Flip the CONNECTED repository's client-side read-only bit + /// (`kopia repository set-client --read-only` / `--read-write`), issue #374. + /// + /// Three properties, all pinned against kopia 0.23.1 by + /// `crates/kopia/tests/integration_set_client_throttle.rs`: + /// + /// - **Works on a read-only connection.** `set-client` registers kopia's + /// `repositoryReaderAction`, so it opens fine against a config connected with + /// `--readonly` — unlike [`Self::repository_set_parameters`], which needs a real + /// blob write and hard-errors there. + /// - **The bit lives in the local config, not the repository.** `read_only: true` + /// writes `"readonly": true` into the `KOPIA_CONFIG_PATH` JSON; `false` removes the + /// key. The test fingerprints every blob in the backend across the flip and requires + /// it unchanged, so this never mutates shared state and needs no maintenance + /// ownership/lease. + /// - **It is the "flip window" primitive** for a read-only-connected config that must + /// run a write-requiring verb: flip read-write, run the verb, flip back. Note that + /// [`Self::repository_throttle_set`] does *not* need this window — M3a measured it + /// succeeding directly on a `--readonly` connection, and leaving the read-only bit + /// intact, because it only rewrites the local config's `throttlingLimits`. The flip + /// is defensive there, not required. + pub async fn repository_set_client_read_only(&self, read_only: bool) -> Result<(), KopiaError> { + let mode = if read_only { + "--read-only" + } else { + "--read-write" + }; + self.run_ok(&[ + "repository".to_string(), + "set-client".to_string(), + mode.to_string(), + ]) + .await + .map(|_| ()) + } + /// Rewrite mutable repository parameters on the CONNECTED repository /// (`kopia repository set-parameters [flags]`), issue #258. No-op when nothing is set. /// diff --git a/crates/kopia/src/client/tests.rs b/crates/kopia/src/client/tests.rs index 2b59a700..bd2a1d96 100644 --- a/crates/kopia/src/client/tests.rs +++ b/crates/kopia/src/client/tests.rs @@ -1476,6 +1476,61 @@ fn throttle_args_builds_per_second_flags_and_empties() { assert!(ThrottleArgs::default().args().is_empty()); } +#[cfg(unix)] +#[tokio::test] +async fn set_client_read_only_emits_the_matching_polarity_flag() { + use std::io::Write; + use std::os::unix::fs::PermissionsExt; + + // `repository_set_client_read_only` builds argv only — no parsing — so the + // assertion has to be on what the process is actually SPAWNED with. A shim + // standing in for `kopia` appends its argv to a file (issue #374). + let dir = std::env::temp_dir().join(format!("kopiur-setclient-{}", std::process::id())); + // The shim APPENDS, so a leftover argv.out from an earlier run in the same + // process-id directory would prepend phantom lines. Start from nothing. + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let shim = dir.join("kopia"); + let out = dir.join("argv.out"); + { + let mut f = std::fs::File::create(&shim).unwrap(); + write!( + f, + "#!/bin/sh\necho \"$@\" >> \"$KOPIUR_SETCLIENT_OUT\"\nexit 0\n" + ) + .unwrap(); + std::fs::set_permissions(&shim, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + + let client = KopiaClient::builder() + .binary(&shim) + .env("KOPIUR_SETCLIENT_OUT", out.to_str().unwrap()) + .build(); + + // Both polarities: `true` parks the config read-only, `false` opens the + // flip window. kopia spells these as two distinct flags, not a negation of + // one — passing the wrong one silently leaves the bit where it was. + client.repository_set_client_read_only(false).await.unwrap(); + client.repository_set_client_read_only(true).await.unwrap(); + + let lines: Vec = std::fs::read_to_string(&out) + .unwrap() + .lines() + .map(|l| l.trim().to_string()) + .collect(); + // `--no-auto-maintenance` is the unconditional common arg every invocation + // carries; asserting the whole argv keeps it honest rather than substring-matching. + assert_eq!( + lines, + vec![ + "repository set-client --read-write --no-auto-maintenance", + "repository set-client --read-only --no-auto-maintenance", + ] + ); + + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn server_start_args_password_mode() { let spec = ServerStartSpec { diff --git a/crates/kopia/tests/integration_set_client_throttle.rs b/crates/kopia/tests/integration_set_client_throttle.rs new file mode 100644 index 00000000..e4d24281 --- /dev/null +++ b/crates/kopia/tests/integration_set_client_throttle.rs @@ -0,0 +1,473 @@ +//! Real-kopia integration test for the read-only/read-write client flip and the +//! throttle limits it brackets (issue #374, milestone M3a). +//! +//! Gated behind the `integration` feature and `#[ignore]` by default so the +//! hermetic `cargo test` never invokes the real binary (the `integration` +//! feature lifts the `#[ignore]`, so it is not needed on the command line): +//! +//! ```text +//! cargo test -p kopiur-kopia --features integration --test integration_set_client_throttle +//! ``` +//! +//! The kopia binary comes from mise's PATH — the SAME pin (`v0.23.1`) the mover +//! image ships — so a Renovate bump moves both and this test fails loudly on +//! semantic drift. That is the point: everything asserted here is a premise the +//! replication/seed movers' throttle handling rests on, and none of it is +//! documented by kopia. +//! +//! What it pins against kopia 0.23.1: +//! +//! * **`repository throttle set` does NOT need a read-write window.** It succeeds +//! directly on a `--readonly` connection — contradicting the design premise +//! this milestone was written to confirm (kopia registers it as a write action, +//! but it never puts a blob, so read-only storage never rejects it). The +//! sequence that shipped is therefore `connect --readonly` → `throttle set` → +//! migrate, with **no flip** — that is what steps (1b)→(6) assert, end to end, +//! and it is the sequence the timing probe measures. `repository set-client +//! --read-only/--read-write` consequently stays UNWIRED in the mover flows; +//! the flip coverage below is a contingency guard, kept so that a future kopia +//! turning `throttle set` into a real write action finds the alternative +//! already proven rather than having to discover it under an outage. +//! * **…and that sequence leaves the source READ-ONLY.** `throttle set` rewrites +//! the config file, so "the bit survives the rewrite" is a data-safety +//! invariant for a replication source, not a formality. Step (1c) asserts it +//! both in the config JSON and behaviorally, immediately after the cap lands +//! and before anything flips. +//! * `repository set-client --read-only` / `--read-write` works on a config +//! connected `--readonly`, and only rewrites the LOCAL config's `readonly` key +//! (`true` present / key absent when read-write) — the backend is fingerprinted +//! across the flip to pin that nothing reaches it. +//! * The flip is real, not cosmetic: `repository set-parameters` — a genuinely +//! write-requiring verb, and the one that DOES need the window — fails while +//! read-only, succeeds inside the flip window, and fails again after the flip +//! back. +//! * Limits persist as `throttlingLimits` in the config JSON and survive the flip +//! sequence untouched. +//! * The load-bearing claim for the source side: `snapshot migrate +//! --source-config ` genuinely honors those config-persisted +//! limits. `snapshot migrate` has no speed flags at all, so the persisted +//! config is the only lever there is — if this were false, the whole +//! throttle-the-source design would be inert. The bound is self-calibrating: +//! the test times its own UNTHROTTLED baseline migrate first and requires the +//! capped one to be at least an order of magnitude slower, so the verdict does +//! not depend on how fast the machine is. +//! * The caveat that comes with it: throttling only bites on traffic that +//! actually reaches the backend. A warm source cache makes even a 1 KB/s cap +//! invisible (the migrate serves from local disk and finishes in ~200 ms), so +//! both timed migrates drop the cache first — which is also what a fresh mover +//! pod looks like. + +#![cfg(unix)] + +use std::collections::BTreeMap; +use std::time::Instant; + +use kopiur_kopia::client::SetParametersArgs; +use kopiur_kopia::{ + ConnectOptions, ConnectSpec, KopiaClient, MigratePolicies, MigrateSources, + SnapshotMigrateOptions, ThrottleArgs, +}; + +const SRC_PASSWORD: &str = "setclient-pass-a"; +const DEST_PASSWORD: &str = "setclient-pass-b"; +const IDENTITY: &str = "throttleuser@throttlehost:/data"; + +/// The cap used for the migrate timing probe, in bytes/sec. +/// +/// Chosen from a measured curve, not guessed. Against this fixture (an 8 KiB +/// incompressible payload → a ~28 KiB source repository) on the pinned binary, +/// with a COLD source cache, `snapshot migrate` takes: +/// +/// | cap B/s | elapsed | +/// |-----------|---------| +/// | unlimited | 0.21 s | +/// | 20 M | 0.21 s | +/// | 10 M | 0.21 s | +/// | 5 M | 2.2 s | +/// | 2 M | 14.2 s | +/// | 1 M | 34.2 s | +/// +/// Note how brutally non-linear that is: kopia's throttler costs far more than +/// the nominal rate implies (28 KiB "should" take 14 ms at 2 MB/s, and takes +/// 14 s), and caps at or above ~10 MB/s do not bind at all here. 2 MB/s is the +/// sweet spot — a ~14 s run, ~5x above the assertion floor and ~70x above the +/// unthrottled time, while a 1 MB/s cap would drag the test to 35 s and lower +/// caps to hours. +const MIGRATE_CAP_BYTES_PER_SEC: i64 = 2_000_000; + +/// How many times slower than the UNTHROTTLED baseline the capped migrate must +/// be. The bound is relative on purpose: the test measures its own baseline on +/// the machine it is running on (an identical migrate of the identical fixture, +/// cold cache, no cap) instead of hard-coding a wall-clock figure that a slow CI +/// box could breach for the wrong reason. The measured ratio here is ~68x +/// (14.2 s vs 0.21 s), so 10x leaves ~7x of headroom, while a completely +/// ignored throttle lands at ~1x and cannot squeak past. +const MIGRATE_MIN_SLOWDOWN_FACTOR: u32 = 10; + +/// A wall-clock floor kept ALONGSIDE the relative bound, as a backstop for the +/// degenerate case where the baseline itself is pathologically slow (a stalled +/// CI runner) and would make the ratio trivially satisfiable. Well under the +/// ~14 s measured at [`MIGRATE_CAP_BYTES_PER_SEC`]; throttling can only ever +/// make a run slower, so a faster machine cannot push a genuinely throttled run +/// below it. +const MIGRATE_MIN_ELAPSED_SECS: u64 = 3; + +fn config_path(config_dir: &std::path::Path) -> String { + config_dir.join("repository.config").display().to_string() +} + +/// A client whose kopia state is isolated inside `config_dir`, same pattern as +/// `integration_migrate.rs`. +fn isolated_client(config_dir: &std::path::Path, password: &str) -> KopiaClient { + KopiaClient::builder() + .binary("kopia") + .env("KOPIA_PASSWORD", password) + .env("KOPIA_CONFIG_PATH", config_path(config_dir)) + .env( + "KOPIA_CACHE_DIRECTORY", + config_dir.join("cache").display().to_string(), + ) + .env( + "KOPIA_LOG_DIR", + config_dir.join("logs").display().to_string(), + ) + .env("KOPIA_CHECK_FOR_UPDATES", "false") + .build() +} + +/// A DESTINATION client able to run `snapshot migrate`: it must NOT pin +/// `KOPIA_CACHE_DIRECTORY`, or migrate's source open reads the destination's +/// cached format blob and fails with "invalid repository password" (proven by +/// `integration_migrate.rs`). Cache isolation comes from `XDG_CACHE_HOME`. +fn dest_client(config_dir: &std::path::Path, password: &str) -> KopiaClient { + KopiaClient::builder() + .binary("kopia") + .env("KOPIA_PASSWORD", password) + .env("KOPIA_CONFIG_PATH", config_path(config_dir)) + .env( + "XDG_CACHE_HOME", + config_dir.join("xdg-cache").display().to_string(), + ) + .env_remove("KOPIA_CACHE_DIRECTORY") + .env( + "KOPIA_LOG_DIR", + config_dir.join("logs").display().to_string(), + ) + .env("KOPIA_CHECK_FOR_UPDATES", "false") + .build() +} + +/// The parsed kopia config JSON for a client's config directory. The read-only +/// bit and the throttling limits both live here — this test asserts on the file +/// because that persistence IS the mechanism the movers depend on. +fn read_config(config_dir: &std::path::Path) -> serde_json::Value { + let raw = std::fs::read_to_string(config_path(config_dir)).expect("read kopia config JSON"); + serde_json::from_str(&raw).expect("kopia config is JSON") +} + +/// A genuinely write-requiring verb, used to prove the read-only bit is really +/// in force (not just recorded). `repository set-parameters` must reach the +/// backend to rewrite `kopia.blobcfg`, so a read-only connection hard-errors. +/// The value varies per call so a rejected attempt can never be confused with +/// kopia skipping a no-change write. +async fn try_set_epoch_advance_on_count( + client: &KopiaClient, + count: i64, +) -> Result<(), kopiur_kopia::KopiaError> { + client + .repository_set_parameters(&SetParametersArgs { + epoch_advance_on_count: Some(count), + ..Default::default() + }) + .await +} + +/// Assert the connection is read-only, BOTH ways: the recorded bit in the config +/// JSON, and behaviorally — a write-requiring verb still gets rejected. Asserting +/// only the first would let a kopia that records the bit but stops honoring it +/// pass; asserting only the second would miss the config regressing under a later +/// rewrite. `probe_count` must differ from the last value successfully written, +/// so a rejection can never be kopia short-circuiting a no-change write. +async fn assert_read_only(client: &KopiaClient, config_dir: &std::path::Path, probe_count: i64) { + assert_eq!( + read_config(config_dir).get("readonly"), + Some(&serde_json::Value::Bool(true)), + "the config must record `readonly: true`" + ); + let err = try_set_epoch_advance_on_count(client, probe_count) + .await + .expect_err("a write-requiring verb must be rejected while read-only"); + assert!( + err.to_string().contains("read-only") || err.to_string().contains("readonly"), + "expected a read-only storage rejection, got: {err}" + ); +} + +/// Every file in the repository backend, by path and size. Used to pin the claim +/// that `set-client` touches nothing on the backend — a claim that would +/// otherwise be inference from "it works on read-only storage". +fn backend_fingerprint(repo_dir: &std::path::Path) -> BTreeMap { + fn walk(dir: &std::path::Path, out: &mut BTreeMap) { + for entry in std::fs::read_dir(dir).expect("read repository directory") { + let entry = entry.expect("repository directory entry"); + let meta = entry.metadata().expect("repository entry metadata"); + if meta.is_dir() { + walk(&entry.path(), out); + } else { + out.insert(entry.path(), meta.len()); + } + } + } + let mut out = BTreeMap::new(); + walk(repo_dir, &mut out); + out +} + +/// Run one `snapshot migrate` of the whole source into `dest` and return how long +/// it took, with the source cache dropped first. +/// +/// Dropping the cache is load-bearing, not hygiene: kopia only throttles traffic +/// that actually reaches the backend, and the migrating process reads the source +/// through the cache directory recorded in the SOURCE config +/// (`caching.cacheDirectory`, relative to the config file → `/cache`). Left +/// warm, the migrate serves everything from local disk and finishes unthrottled +/// in ~200 ms whatever the cap says. A fresh mover pod's cache is cold by +/// construction, which is what this models — and it is also the caveat the +/// dependent milestones have to design around. +async fn timed_cold_migrate( + dest: &KopiaClient, + src_ro_cfg: &std::path::Path, +) -> std::time::Duration { + std::fs::remove_dir_all(src_ro_cfg.join("cache")).expect("drop the source cache"); + let started = Instant::now(); + dest.snapshot_migrate(&SnapshotMigrateOptions { + source_config_path: config_path(src_ro_cfg), + sources: MigrateSources::All, + latest_only: false, + parallel: None, + policies: MigratePolicies::None, + }) + .await + .expect("snapshot migrate"); + started.elapsed() +} + +#[tokio::test] +#[cfg_attr(not(feature = "integration"), ignore)] +async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() { + let src_repo = tempfile::tempdir().unwrap(); + let src_admin_cfg = tempfile::tempdir().unwrap(); + let src_ro_cfg = tempfile::tempdir().unwrap(); + let data_dir = tempfile::tempdir().unwrap(); + let dest_repo = tempfile::tempdir().unwrap(); + let dest_cfg = tempfile::tempdir().unwrap(); + let base_repo = tempfile::tempdir().unwrap(); + let base_cfg = tempfile::tempdir().unwrap(); + + // ---- fixture: a source repository holding one snapshot of 8 KiB of + // deliberately incompressible-but-DETERMINISTIC bytes (a Knuth-multiplicative + // scramble). Deterministic keeps the calibrated timings in + // `MIGRATE_CAP_BYTES_PER_SEC` reproducible run to run; incompressible stops a + // future compression default from shrinking the payload and quietly robbing + // the timing probe of the bytes it measures. + let payload: Vec = (0..8192u32) + .map(|i| (i.wrapping_mul(2654435761) >> 13) as u8) + .collect(); + std::fs::write(data_dir.path().join("blob.bin"), &payload).unwrap(); + let src_admin = isolated_client(src_admin_cfg.path(), SRC_PASSWORD); + src_admin + .repository_create( + &ConnectSpec::Filesystem { + path: src_repo.path().to_path_buf(), + }, + Default::default(), + &Default::default(), + ) + .await + .expect("source repository create"); + src_admin + .snapshot_create( + data_dir.path().to_str().unwrap(), + &BTreeMap::new(), + Some(IDENTITY), + ) + .await + .expect("source snapshot create"); + + // Two destinations: one for the unthrottled BASELINE migrate that calibrates + // the timing bound, one for the throttled measurement. They must be separate + // repositories — migrate is idempotent by (source, startTime), so a second + // run into the same destination would copy nothing and time nothing. + let dest = dest_client(dest_cfg.path(), DEST_PASSWORD); + dest.repository_create( + &ConnectSpec::Filesystem { + path: dest_repo.path().to_path_buf(), + }, + Default::default(), + &Default::default(), + ) + .await + .expect("destination repository create"); + let baseline_dest = dest_client(base_cfg.path(), DEST_PASSWORD); + baseline_dest + .repository_create( + &ConnectSpec::Filesystem { + path: base_repo.path().to_path_buf(), + }, + Default::default(), + &Default::default(), + ) + .await + .expect("baseline destination repository create"); + + // ---- the connect the replication/seed movers perform for their SOURCE: + // a separate config, read-only, credentials persisted beside it. + let src_ro = isolated_client(src_ro_cfg.path(), SRC_PASSWORD); + src_ro + .repository_connect_with( + &ConnectSpec::Filesystem { + path: src_repo.path().to_path_buf(), + }, + Default::default(), + ConnectOptions { + readonly: true, + persist_credentials: true, + }, + ) + .await + .expect("source reconnect --readonly --persist-credentials"); + assert_eq!( + read_config(src_ro_cfg.path()).get("readonly"), + Some(&serde_json::Value::Bool(true)), + "a --readonly connect must record `readonly: true` in the config" + ); + + // (1) The read-only bit is REALLY in force: a verb that must write a blob + // fails. This is the control for everything below — without it, "throttle + // set works on read-only" would prove nothing. + assert_read_only(&src_ro, src_ro_cfg.path(), 10).await; + + // (1a) BASELINE: an identical migrate with NO cap, on this machine, to + // calibrate the timing bound in step (6). Taken before any throttle exists so + // it measures the genuinely unthrottled cost. + let baseline = timed_cold_migrate(&baseline_dest, src_ro_cfg.path()).await; + + // (1b) THE PRODUCTION SEQUENCE, and the premise this milestone was written to + // check — which does NOT hold as the plan stated it. `repository throttle set` + // SUCCEEDS on the read-only connection: it only rewrites the local config's + // `throttlingLimits`, it never puts a blob. So what shipped is `connect + // --readonly` → `throttle set` → migrate, with NO flip, and that is the + // sequence measured in step (6). + // + // If a future kopia turns this into a real write action, this fails loudly and + // the flip below stops being defensive and becomes mandatory. + src_ro + .repository_throttle_set(&ThrottleArgs { + upload_bytes_per_second: Some(MIGRATE_CAP_BYTES_PER_SEC), + download_bytes_per_second: Some(MIGRATE_CAP_BYTES_PER_SEC), + ..Default::default() + }) + .await + .expect("throttle set succeeds on a READ-ONLY connection (kopia 0.23.1)"); + let limits = read_config(src_ro_cfg.path())["throttlingLimits"].clone(); + assert_eq!( + limits["maxUploadSpeedBytesPerSecond"], + serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), + "throttle set must persist into the local config even when read-only: {limits}" + ); + assert_eq!( + limits["maxDownloadSpeedBytesPerSecond"], + serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), + "throttle set must persist into the local config even when read-only: {limits}" + ); + + // (1c) …AND THE SOURCE IS STILL READ-ONLY. This is the data-safety invariant + // of the whole no-flip sequence: `throttle set` rewrites the config file, and + // a rewrite that dropped the `readonly` key would silently leave a replication + // SOURCE writable. Asserted both ways — the recorded bit and the behavioral + // probe — because nothing downstream would notice otherwise: step (2) flips + // read-write on purpose, and step (3) re-establishes the bit explicitly. + assert_read_only(&src_ro, src_ro_cfg.path(), 11).await; + + // (2) The flip window itself — kept because the client method has to be + // proven for the verbs that genuinely DO need it. Opening it makes the same + // write-requiring verb that failed twice above SUCCEED. + let backend_before = backend_fingerprint(src_repo.path()); + src_ro + .repository_set_client_read_only(false) + .await + .expect("set-client --read-write"); + assert!( + read_config(src_ro_cfg.path()).get("readonly").is_none(), + "read-write must clear the config's `readonly` key" + ); + assert_eq!( + backend_fingerprint(src_repo.path()), + backend_before, + "set-client must not touch the backend — the flip is a LOCAL config edit" + ); + try_set_epoch_advance_on_count(&src_ro, 10) + .await + .expect("set-parameters must succeed inside the read-write flip window"); + + // (3) Flip back, and (4) assert read-only is genuinely restored — with a + // DIFFERENT epoch count (12, not the 10 just committed) so the rejection + // cannot be a no-change short-circuit. + src_ro + .repository_set_client_read_only(true) + .await + .expect("set-client --read-only"); + assert_read_only(&src_ro, src_ro_cfg.path(), 12).await; + + // (5) The cap set back at (1b), on the read-only connection, survived the + // whole flip sequence untouched — so the config step (6) measures is the one + // the no-flip production sequence wrote. + let limits = read_config(src_ro_cfg.path())["throttlingLimits"].clone(); + assert_eq!( + limits["maxUploadSpeedBytesPerSecond"], + serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), + "upload cap must survive the flip sequence: {limits}" + ); + assert_eq!( + limits["maxDownloadSpeedBytesPerSecond"], + serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), + "download cap must survive the flip sequence: {limits}" + ); + + // (6) THE LOAD-BEARING CLAIM: `snapshot migrate` reopens the source config + // and honors its persisted limits. `snapshot migrate` exposes no speed + // flags, so if this were false there would be no way to throttle a + // seed-migrate at all. + // + // The config this reads is the one the NO-FLIP sequence wrote at (1b) and + // (5) proved untouched since — i.e. this measures exactly what production + // will do, not a flip-bracketed variant of it. + let elapsed = timed_cold_migrate(&dest, src_ro_cfg.path()).await; + + let migrated = dest.snapshot_list_all().await.expect("dest list --all"); + assert_eq!( + migrated.len(), + 1, + "the snapshot must actually arrive: {migrated:?}" + ); + assert_eq!(migrated[0].source.identity(), IDENTITY); + + // Both bounds, and the failure message carries the baseline so a breach is + // diagnosable rather than just "too fast". The mode this guards is "the + // throttle was ignored entirely", which lands at ~1x the baseline. + assert!( + elapsed >= baseline * MIGRATE_MIN_SLOWDOWN_FACTOR, + "migrate finished in {elapsed:?} with the source config capped at \ + {MIGRATE_CAP_BYTES_PER_SEC} B/s, against an UNTHROTTLED baseline of \ + {baseline:?} for the identical migrate — under {MIGRATE_MIN_SLOWDOWN_FACTOR}x \ + it must have IGNORED the persisted throttle. The entire source-side \ + throttle design rests on migrate honoring it." + ); + assert!( + elapsed.as_secs() >= MIGRATE_MIN_ELAPSED_SECS, + "migrate finished in {elapsed:?} with the source config capped at \ + {MIGRATE_CAP_BYTES_PER_SEC} B/s (baseline {baseline:?}) — below the \ + {MIGRATE_MIN_ELAPSED_SECS}s absolute floor, so the relative bound was \ + satisfied only because the baseline itself was pathologically slow." + ); +} diff --git a/crates/mover/src/bootstrap.rs b/crates/mover/src/bootstrap.rs index 101ba413..9f5dd841 100644 --- a/crates/mover/src/bootstrap.rs +++ b/crates/mover/src/bootstrap.rs @@ -1019,6 +1019,7 @@ mod tests { migrate: None, allow_empty_source: false, resume, + replica_throttle: Default::default(), } } diff --git a/crates/mover/src/main.rs b/crates/mover/src/main.rs index 99908fae..7fbf82af 100644 --- a/crates/mover/src/main.rs +++ b/crates/mover/src/main.rs @@ -41,15 +41,15 @@ use kopiur_mover::resolve::{match_current_manifest, matches_source}; use kopiur_mover::serve::ServerWorkSpec; use kopiur_mover::status::{ SnapshotReplicationRunStats, StatusReporter, StatusUpdate, lease_blocked_body, - maintenance_failed_body, maintenance_ran_body, replicate_failed_body, replicate_ok_body, - snapshot_replicate_failed_body, snapshot_replicate_ok_body, split_api_version, - verify_failed_body, verify_ok_body, + maintenance_failed_body, maintenance_failed_body_from_mover, maintenance_ran_body, + replicate_failed_body, replicate_ok_body, snapshot_replicate_failed_body, + snapshot_replicate_ok_body, split_api_version, verify_failed_body, verify_ok_body, }; use kopiur_mover::workspec::{ self, BootstrapRepositoryOp, BrowseSessionOp, KOPIA_KEEP_MAX, KOPIUR_PIN_NAME, MaintenanceOp, MoverWorkSpec, Operation, ReplicateOp, RestoreOp, RestoreSelection, RestoreSelector, - SnapshotAnchor, SnapshotDeleteBatchOp, SnapshotPinOp, SnapshotReplicateOp, VerifyOp, - VerifyTier, maintenance_restamp_target, + SnapshotAnchor, SnapshotDeleteBatchOp, SnapshotPinOp, SnapshotReplicateOp, ThrottleSpec, + VerifyOp, VerifyTier, maintenance_restamp_target, }; #[cfg(test)] use kopiur_mover::workspec::{SnapshotDeleteItem, SnapshotDeleteOp}; @@ -264,7 +264,25 @@ async fn run(cli: &MoverCli) -> Result<()> { Operation::BrowseSession(op) => { run_browse_session_flow(&client, &spec, op, &connect).await } - _ => { + // Everything else shares ONE lifecycle: connect, apply the + // repository throttle, then run with periodic progress PATCHes. + // + // Listed variant by variant, NOT `_ =>`. The catch-all that used to + // stand here is exactly how #374 shipped: every flow above owns its + // own connect, and because the throttle lived only in this arm, a + // `moverDefaults.throttle` the controller faithfully put on the wire + // was silently inert for six of them. A new operation must now + // decide its throttle story to compile. + // + // `SnapshotDeleteBatch` and `SnapshotPin` reach here with a + // `Default` (empty) throttle from their controllers, so + // `apply_repository_throttle` is a no-op for them today — that is a + // controller-side gap, deliberately out of scope for this change. + Operation::Snapshot(_) + | Operation::Restore(_) + | Operation::SnapshotDelete(_) + | Operation::SnapshotDeleteBatch(_) + | Operation::SnapshotPin(_) => { // A best-effort status reporter. If we cannot build a kube client // (e.g. running outside a cluster), we log instead of failing. // SnapshotDeleteBatch's targetRef names the Job itself — nothing @@ -278,48 +296,23 @@ async fn run(cli: &MoverCli) -> Result<()> { } else { StatusReporter::try_new(&spec).await }; - match client.repository_connect(&connect, spec.cache).await { + // Connect AND throttle as one operation — see `connect_and_throttle`. + match connect_and_throttle(&client, &connect, spec.cache, &spec.throttle).await { Err(e) => { - terminal_failure( - &reporter, - MoverError::Kopia { - op: KopiaOp::RepositoryConnect, - source: e, - }, - ) - .await + terminal_failure(&reporter, e.into_mover_error(KopiaOp::RepositoryConnect)) + .await } - Ok(()) => { - // Apply repository throttle (moverDefaults.throttle, ADR-0005 - // §13(e)) after connecting, before the data op. A throttle - // failure is terminal: an un-throttled run could saturate the - // link the user explicitly capped. - if !spec.throttle.is_empty() - && let Err(e) = client - .repository_throttle_set(&spec.throttle.to_kopia()) - .await - { - return terminal_failure( - &reporter, - MoverError::Kopia { - op: KopiaOp::ThrottleSet, - source: e, - }, - ) - .await; - } - match execute(&client, &spec, &reporter).await { - Ok(update) => { - reporter.report(&update).await; - info!( - phase = update.phase.as_deref().unwrap_or("done"), - "operation succeeded" - ); - Ok(()) - } - Err(e) => terminal_failure(&reporter, e).await, + Ok(()) => match execute(&client, &spec, &reporter).await { + Ok(update) => { + reporter.report(&update).await; + info!( + phase = update.phase.as_deref().unwrap_or("done"), + "operation succeeded" + ); + Ok(()) } - } + Err(e) => terminal_failure(&reporter, e).await, + }, } } }, @@ -337,6 +330,143 @@ async fn run(cli: &MoverCli) -> Result<()> { result } +/// Why a repository connection is not usable for work: it never opened, or it +/// opened UNCAPPED. Mirrors [`BootstrapConnectFailure`] and for the same reason +/// — the two are different facts and some callers must tell them apart — but +/// unlike bootstrap's, this one is generic over the calling flow's connect op +/// label. +enum ConnectFailure { + /// `repository connect` itself failed. + Connect(KopiaError), + /// The connection opened, but `repository throttle set` failed. Already + /// carries [`KopiaOp::ThrottleSet`]. + Throttle(MoverError), +} + +impl ConnectFailure { + /// The typed mover failure, labeled with the CALLING flow's connect op + /// (`MaintenanceConnect`, `VerifyConnect`, `ReplicateConnect`, …). A + /// throttle failure passes through with its own label intact, so a status + /// block always names the invocation that actually failed. + fn into_mover_error(self, connect_op: KopiaOp) -> MoverError { + match self { + ConnectFailure::Connect(source) => MoverError::Kopia { + op: connect_op, + source, + }, + ConnectFailure::Throttle(e) => e, + } + } +} + +/// Open a repository connection **and** apply its throttle, as ONE operation. +/// +/// This is the funnel every flow that owns its own read-write connect goes +/// through (the generic `run()` arm, maintenance, verify, replicate, and a +/// seed's local repository); bootstrap has its own funnel, +/// [`bootstrap_connect`], because its connect failure additionally feeds a +/// create-vs-seed verdict. +/// +/// **Why a funnel rather than two calls.** #374 was not a missing feature, it +/// was six remembered call sites: `repository_connect` and `throttle set` were +/// separate steps, so a flow could — and did — do the first and forget the +/// second, and every test still passed because the run succeeded uncapped. +/// Fusing them means a future flow cannot obtain a connection without also +/// obtaining its cap. `connect_funnel_is_the_only_way_flows_connect` (below) +/// makes an attempt to bypass it loud. +async fn connect_and_throttle( + client: &KopiaClient, + connect: &ConnectSpec, + cache: kopiur_kopia::CacheTuning, + throttle: &ThrottleSpec, +) -> std::result::Result<(), ConnectFailure> { + client + .repository_connect(connect, cache) + .await + .map_err(ConnectFailure::Connect)?; + apply_repository_throttle(client, throttle) + .await + .map_err(ConnectFailure::Throttle) +} + +/// Apply the work spec's [`ThrottleSpec`] (`moverDefaults.throttle`, ADR-0005 +/// §13(e)) to the repository connection the caller just opened. +/// +/// **The invariant: every mover flow that connects a repository applies the +/// throttle to that connection.** Reached through one of the two funnels — +/// [`connect_and_throttle`] or [`bootstrap_connect`] — never by a flow +/// remembering to call this itself. #374 shipped because the `throttle set` +/// lived in ONE connect arm of `run()`, while bootstrap, seed, maintenance, +/// verify and replicate each open their own connection — so a cap the user +/// configured and the controller faithfully put on the wire was inert for all +/// of them. kopia's limits are per-CONNECTION (persisted into that connection's +/// client config), so a second connect under a second config needs its own +/// call; there is no repository-wide setting to inherit. +/// +/// The connects that do NOT go through a funnel, each for a stated reason — +/// keep this list in step with `connect_funnel_is_the_only_way_flows_connect`: +/// +/// * **`kopiur-mover serve`** (the browse/UI kopia server) runs off a +/// [`ServerWorkSpec`], which carries no throttle field at all. Nothing to +/// apply until that wire type grows one. +/// * **browse session** — its work spec is built by the CLI with +/// `throttle: Default::default()`, so `spec.throttle` is structurally always +/// empty; capping it means teaching the CLI to resolve `moverDefaults` first. +/// * **snapshot-replication source and destination** — a dual connect under two +/// kopia configs, so ONE funnel call cannot serve both: each side is connected +/// directly and then throttled in place from its own resolved block (the work +/// spec's `throttle` for the source, the op's `destinationThrottle` for the +/// destination). Both applications are terminal, via the flow's `srepl_terminal`. +/// * **a seed's SOURCE (replica) connect** — same shape: the seed holds its own +/// repository and the replica open at once under two kopia configs, so the +/// replica is connected directly and then throttled in place from the op's +/// `replicaThrottle`. Terminal, and reported as the bootstrap's failure. +/// +/// No limits set → no kopia process is spawned (the common case stays free). +/// +/// A failure is TERMINAL for the run, never best-effort: continuing would +/// saturate exactly the link the user capped, which is worse than not running. +/// +/// The `info!` line is the observable proof the limits reached kopia — the e2e +/// scenarios assert on it, and its ABSENCE is the version-skew signal for a new +/// controller driving an old mover image. Emitted only after kopia accepted +/// them, so it can never claim a throttle that is not in force. One line per +/// APPLICATION, not per run: a flow that opens several connections logs it once +/// per connection, which is what lets an e2e assert that every one of them was +/// capped. A first migrate seed opens three (the replica, the seed's local +/// client, the post-seed reconnect — its probe connect FAILS on the +/// uninitialized backend, which is what arms the seed, so it never reaches its +/// own application); a resuming one opens four, since that probe connect +/// succeeds. +/// +/// Applied on read-only connections too. `repository throttle set` registers a +/// kopia *write* action, but is empirically accepted on a `--readonly` connect +/// against kopia 0.23.1 — pinned by +/// `crates/kopia/tests/integration_set_client_throttle.rs`, with `set-parameters` +/// as the control probe that does hard-error there. So no read-write flip is +/// needed anywhere, and `mode: ReadOnly` repositories are throttled like any +/// other. +async fn apply_repository_throttle(client: &KopiaClient, throttle: &ThrottleSpec) -> Result<()> { + if throttle.is_empty() { + return Ok(()); + } + client + .repository_throttle_set(&throttle.to_kopia()) + .await + .map_err(|source| MoverError::Kopia { + op: KopiaOp::ThrottleSet, + source, + })?; + info!( + upload = ?throttle.upload_bytes_per_second, + download = ?throttle.download_bytes_per_second, + read_ops = ?throttle.read_ops_per_second, + write_ops = ?throttle.write_ops_per_second, + "applied repository throttle" + ); + Ok(()) +} + /// Whether `run()`'s generic connect+execute path must hand-select the /// log-only status reporter instead of `StatusReporter::try_new`'s /// best-effort kube-client attempt. There is no compiler-enforced link @@ -1077,6 +1207,32 @@ fn identity_retention_policy( } } +/// Why a bootstrap connect did not leave a usable, throttled connection. +/// +/// The two are kept apart because only ONE of them is a verdict about the +/// backend: [`bootstrap_connect_probe`] reads a `Connect` failure to decide +/// whether this backend holds a repository, is uninitialized (create/seed it), +/// or is simply unreachable. A throttle failure says nothing about any of that +/// — folding it into the connect error would let a `NotFound`-classified +/// throttle failure be read as "empty backend" and initialize a repository. +enum BootstrapConnectFailure { + /// `repository connect` itself failed — the shape the probe classifies. + Connect(KopiaError), + /// The connection opened, but the repository throttle could not be applied. + /// Terminal for the bootstrap, and never a statement about the backend. + Throttle(MoverError), +} + +impl BootstrapConnectFailure { + /// The terminal [`BootstrapResult`] this failure reports to the controller. + fn into_result(self) -> BootstrapResult { + match self { + BootstrapConnectFailure::Connect(e) => BootstrapResult::failed(&e), + BootstrapConnectFailure::Throttle(e) => BootstrapResult::from_mover_error(&e), + } + } +} + /// Connect for bootstrap, honoring [`BootstrapRepositoryOp::read_only`] /// (M6): a `mode: ReadOnly` repository's bootstrap connects with `--readonly` /// (`repository_connect_readonly`) instead of the normal read-write connect — @@ -1084,17 +1240,30 @@ fn identity_retention_policy( /// ReadOnly consumer repo was exactly what let it clobber the primary's /// maintenance owner. Every other mover flow (restore, delete, snapshot) /// stays on the plain read-write connect regardless. +/// +/// The ONE funnel for every bootstrap connect (probe, post-create reconnect, +/// post-seed reconnect), so [`apply_repository_throttle`]'s no-exceptions +/// invariant holds for all of them by construction rather than by three +/// remembered call sites. The `--readonly` probe is throttled too: `throttle +/// set` is accepted on a read-only connect (see `apply_repository_throttle`), +/// and a bootstrap's catalog scan reads real backend traffic that a capped link +/// must not carry unthrottled. async fn bootstrap_connect( client: &KopiaClient, spec: &ConnectSpec, cache: kopiur_kopia::CacheTuning, read_only: bool, -) -> std::result::Result<(), KopiaError> { - if read_only { + throttle: &ThrottleSpec, +) -> std::result::Result<(), BootstrapConnectFailure> { + let connected = if read_only { client.repository_connect_readonly(spec, cache).await } else { client.repository_connect(spec, cache).await - } + }; + connected.map_err(BootstrapConnectFailure::Connect)?; + apply_repository_throttle(client, throttle) + .await + .map_err(BootstrapConnectFailure::Throttle) } /// How a seed step reports failure: `Ok` on success, or the terminal @@ -1193,6 +1362,13 @@ fn seed_source_builder( /// `--persist-credentials` writes the password beside the config, which is what /// `snapshot migrate --source-config` reads in migrate mode; it is harmless in /// blob mode and kept symmetrical so both arms share this function. +/// +/// The connection is capped in place from the op's `replicaThrottle` right after +/// it opens — this is one of the connects that cannot go through +/// [`connect_and_throttle`], because a seed holds two repositories open under two +/// kopia configs and the work spec's own `throttle` belongs to the other one. +/// Empty in blob mode (no source repository CR to take `moverDefaults` from; its +/// caps ride `sync-to`'s own speed flags), so that arm spawns no kopia process. async fn seed_connect_source( spec: &MoverWorkSpec, seed: &workspec::SeedOpSpec, @@ -1230,12 +1406,35 @@ async fn seed_connect_source( ) .await { - Ok(()) => Ok(SeedSource { - client, - connect: source, - }), - Err(e) => Err(Box::new(seed_source_connect_failure(seed, &e))), + Ok(()) => {} + Err(e) => return Err(Box::new(seed_source_connect_failure(seed, &e))), + } + // Cap the REPLICA read side on this connection, from the op's own + // `replicaThrottle` (the SOURCE repository's `moverDefaults.throttle` + // overlaid by `spec.seed.migrate.throttle.source`). kopia's limits are + // per-connection, so the cap the bootstrap put on THIS repository says + // nothing here — and in migrate mode `snapshot migrate` reopens this very + // config, which is the only lever it has (it carries no speed flags). + // Applied straight onto the read-only connect: `repository throttle set` is + // accepted there, so no read-write flip is involved. + // + // Deliberately BEFORE migrate mode's persisted-password probe: that probe's + // whole job is to prove the config `snapshot migrate` will reopen is usable, + // so it should see that config in its FINAL state — throttle limits written + // and all. + // + // Terminal, like every other throttle application: proceeding would pull a + // whole repository across exactly the link the user asked to protect, and a + // seed is the largest read kopiur ever makes of a replica that is usually + // still another cluster's live off-site copy. + if let Err(e) = apply_repository_throttle(&client, &seed.replica_throttle).await { + error!(class = %e.kopia_class(), "could not cap the seed source connection"); + return Err(Box::new(BootstrapResult::from_mover_error(&e))); } + Ok(SeedSource { + client, + connect: source, + }) } /// Classify a failed seed-source connect. Split out so the "the backend @@ -1473,16 +1672,22 @@ async fn seed_create_and_connect_local( .env_remove(kopiur_kopia::env::CACHE_DIRECTORY_ENV) .env("XDG_CACHE_HOME", &paths.local_xdg) .build(); - match local_client - .repository_connect(local_connect, spec.cache) - .await + // Connect AND cap, as one operation. THIS repository's own + // `moverDefaults.throttle` goes on its own connection: a seed's local client + // runs under a separate kopia config from the bootstrap client the probe + // capped, and kopia's limits are per-config. `snapshot migrate` has no speed + // flags of its own, so the limits persisted into this config are the ONLY cap + // on the heaviest transfer kopiur performs. (The seed SOURCE's own cap is a + // separate knob on a separate connection — `replicaThrottle`, applied in + // `seed_connect_source`; kopia's limits never cross between the two.) + if let Err(e) = + connect_and_throttle(&local_client, local_connect, spec.cache, &spec.throttle).await { - Ok(()) => Ok(local_client), - Err(e) => { - error!(class = %e.class(), "could not connect the repository a migrate seed writes into"); - Err(Box::new(BootstrapResult::failed(&e))) - } + let e = e.into_mover_error(KopiaOp::SeedLocalConnect); + error!(class = %e.kopia_class(), "could not open a capped connection to the repository a migrate seed writes into"); + return Err(Box::new(BootstrapResult::from_mover_error(&e))); } + Ok(local_client) } /// The MANDATORY post-verify for a migrate-mode seed: how many snapshots @@ -1763,15 +1968,19 @@ async fn bootstrap_create_arm( op: &BootstrapRepositoryOp, connect_spec: &ConnectSpec, cache: kopiur_kopia::CacheTuning, + throttle: &ThrottleSpec, ) -> SeedStep<()> { + // `repository create` runs before any connection exists to throttle; it + // writes one format blob, so there is nothing here for a bandwidth cap to + // bound. The reconnect below is throttled like every other connect. if let Err(ce) = client .repository_create(connect_spec, cache, &op.create_options()) .await { return Err(Box::new(BootstrapResult::failed(&ce))); } - if let Err(ce) = bootstrap_connect(client, connect_spec, cache, op.read_only).await { - return Err(Box::new(BootstrapResult::failed(&ce))); + if let Err(ce) = bootstrap_connect(client, connect_spec, cache, op.read_only, throttle).await { + return Err(Box::new(ce.into_result())); } stamp_owner_on_new_repository(client, op.maintenance_owner.as_ref()).await; Ok(()) @@ -1828,8 +2037,10 @@ async fn bootstrap_seed_arm( workspec::SeedModeSpec::Blob => false, workspec::SeedModeSpec::Migrate => !local_initialized, }; - if let Err(ce) = bootstrap_connect(client, connect_spec, cache, op.read_only).await { - return Err(Box::new(BootstrapResult::failed(&ce))); + if let Err(ce) = + bootstrap_connect(client, connect_spec, cache, op.read_only, &spec.throttle).await + { + return Err(Box::new(ce.into_result())); } if created { stamp_owner_on_new_repository(client, op.maintenance_owner.as_ref()).await; @@ -1866,10 +2077,18 @@ async fn bootstrap_connect_probe( connect_spec: &ConnectSpec, cache: kopiur_kopia::CacheTuning, op: &BootstrapRepositoryOp, -) -> BootstrapProbe { - let err = bootstrap_connect(client, connect_spec, cache, op.read_only) - .await - .err(); + throttle: &ThrottleSpec, +) -> SeedStep { + // Exhaustive over the failure shapes: a CONNECT failure is the probe's raw + // material (it decides initialized/uninitialized/create/seed), while a + // THROTTLE failure ends the bootstrap here — the connection is open but + // uncapped, and nothing about the backend has been learned that would + // justify creating or seeding a repository over it. + let err = match bootstrap_connect(client, connect_spec, cache, op.read_only, throttle).await { + Ok(()) => None, + Err(BootstrapConnectFailure::Connect(e)) => Some(e), + Err(e @ BootstrapConnectFailure::Throttle(_)) => return Err(Box::new(e.into_result())), + }; let local_initialized = err.is_none(); let uninitialized = err .as_ref() @@ -1895,13 +2114,13 @@ async fn bootstrap_connect_probe( err.as_ref().map(KopiaError::class), uninitialized, ); - BootstrapProbe { + Ok(BootstrapProbe { err, local_initialized, uninitialized, already_initialized, action, - } + }) } /// The bootstrap routine: connect-first (adopt an existing repo), create only @@ -1930,7 +2149,14 @@ async fn run_bootstrap( // maintenance forever. let mut seeded = false; - let probe = bootstrap_connect_probe(client, &connect_spec, cache, op).await; + // A throttle the connection could not accept ends the bootstrap here — the + // probe learned nothing about the backend, so there is no verdict to carry on + // with. + let probe = + match bootstrap_connect_probe(client, &connect_spec, cache, op, &spec.throttle).await { + Ok(p) => p, + Err(result) => return *result, + }; let (connect_err, local_initialized, uninitialized) = (probe.err, probe.local_initialized, probe.uninitialized); // A successful connect over a NON-resuming armed seed is the documented @@ -1960,7 +2186,9 @@ async fn run_bootstrap( class = ?connect_err.as_ref().map(KopiaError::class), "connect failed; attempting repository create" ); - if let Err(result) = bootstrap_create_arm(client, op, &connect_spec, cache).await { + if let Err(result) = + bootstrap_create_arm(client, op, &connect_spec, cache, &spec.throttle).await + { return *result; } created = true; @@ -2348,13 +2576,14 @@ async fn run_maintenance_flow( ); // Connect first: for object stores this pod is the only place with repo // access, which is exactly why the lease decision is made here. - if let Err(e) = client.repository_connect(connect, spec.cache).await { - patch_maintenance_status(&spec.target_ref, &maintenance_failed_body(&e)).await; - error!(class = %e.class(), "maintenance connect failed"); - return Err(MoverError::Kopia { - op: KopiaOp::MaintenanceConnect, - source: e, - }); + // Connect AND cap, as one operation (`connect_and_throttle`): a full + // maintenance rewrites and drops blobs, so it is exactly the kind of backend + // traffic `moverDefaults.throttle` exists to bound. + if let Err(e) = connect_and_throttle(client, connect, spec.cache, &spec.throttle).await { + let e = e.into_mover_error(KopiaOp::MaintenanceConnect); + patch_maintenance_status(&spec.target_ref, &maintenance_failed_body_from_mover(&e)).await; + error!(class = %e.kopia_class(), "could not open a capped connection to the repository undergoing maintenance"); + return Err(e); } // Assume the STABLE lease-derived client identity before anything else: @@ -2503,13 +2732,14 @@ async fn run_verify_flow( policy = %spec.target_ref.name, "running verification" ); - if let Err(e) = client.repository_connect(connect, spec.cache).await { + // Connect AND cap, as one operation: verification READS the repository — a + // deep verify scratch-restores a whole snapshot — so it is throttled like + // any other data flow. + if let Err(e) = connect_and_throttle(client, connect, spec.cache, &spec.throttle).await { + let e = e.into_mover_error(KopiaOp::VerifyConnect); patch_verify_status(&spec.target_ref, &verify_failed_body(&e.to_string())).await; - error!(class = %e.class(), "verify connect failed"); - return Err(MoverError::Kopia { - op: KopiaOp::VerifyConnect, - source: e, - }); + error!(class = %e.kopia_class(), "could not open a capped connection to the repository being verified"); + return Err(e); } // Run the tier and collect the result environment for successExpr. A kopia @@ -2747,13 +2977,15 @@ async fn run_replicate_flow( ); // Connect to the source first (this pod is the only place with repo access for // object stores — the same rationale as maintenance/verify). - if let Err(e) = client.repository_connect(connect, spec.cache).await { + // Connect AND cap, as one operation: the source repository's own cap goes on + // the connection `repository sync-to` reads through — a whole-repository blob + // mirror is the single heaviest read this flow performs, and the source is + // often the link the user capped. + if let Err(e) = connect_and_throttle(client, connect, spec.cache, &spec.throttle).await { + let e = e.into_mover_error(KopiaOp::ReplicateConnect); patch_replicate_status(&spec.target_ref, &replicate_failed_body(&e.to_string())).await; - error!(class = %e.class(), "replication source connect failed"); - return Err(MoverError::Kopia { - op: KopiaOp::ReplicateConnect, - source: e, - }); + error!(class = %e.kopia_class(), "could not open a capped connection to the source repository a replication reads from"); + return Err(e); } // The destination's credentials arrive under the KOPIUR_DEST_ env prefix so they @@ -2854,6 +3086,12 @@ async fn srepl_terminal_kopia( /// PATCH a terminal failure body for a typed mover error in the snapshot /// replication flow (with whatever run counters exist) and return it. +/// +/// A [`MoverError::Kopia`] arriving here (a throttle application, say) carries a +/// `class` exactly like one going through [`srepl_terminal_kopia`], so it is +/// logged with the same structured field — otherwise whether the operator can +/// filter on `class` would depend on which helper the call site happened to +/// pick. Non-kopia errors have no class to report and log plainly. async fn srepl_terminal( target: &workspec::TargetRef, err: MoverError, @@ -2864,7 +3102,10 @@ async fn srepl_terminal( &snapshot_replicate_failed_body(&err.to_string(), stats), ) .await; - error!("{err}"); + match &err { + MoverError::Kopia { .. } => error!(class = %err.kopia_class(), "{err}"), + _ => error!("{err}"), + } Err(err) } @@ -2944,6 +3185,22 @@ async fn srepl_connect_source( unreachable!("srepl_terminal_kopia always errors"); } + // Cap the SOURCE read side on this connection. `snapshot migrate` reopens + // this very config, and kopia persists the limits into it — that is the only + // lever migrate has, since it carries no speed flags of its own. Applied + // straight onto the read-only connect: `repository throttle set` is accepted + // there (pinned by the kopia crate's integration test), so no read-write + // flip is involved. + // + // Deliberately BEFORE the no-env-password probe below, not after: the probe's + // whole job is to prove the config migrate will reopen is usable, so it + // should see that config in its FINAL state — throttle limits written and + // all — rather than a state no later step ever runs against. + if let Err(e) = apply_repository_throttle(&source_client, &spec.throttle).await { + srepl_terminal(&spec.target_ref, e, None).await?; + unreachable!("srepl_terminal always errors"); + } + let probe_client = srepl_client_builder(spec, kopia_binary) .env(kopiur_kopia::env::CONFIG_PATH_ENV, &paths.source_config) .env(kopiur_kopia::env::CACHE_DIRECTORY_ENV, &paths.source_cache) @@ -3026,6 +3283,15 @@ async fn srepl_connect_dest( srepl_terminal_kopia(&spec.target_ref, Op::SnapshotReplicateDestConnect, e).await?; unreachable!("srepl_terminal_kopia always errors"); } + // Cap the DESTINATION write side on ITS connection, from the op's own + // `destinationThrottle` (the destination repository's `moverDefaults` + // overlaid by `spec.migrate.throttle.destination`). kopia's limits are + // per-connection, so the source's cap above says nothing here — this is the + // block that keeps a migrate from saturating the off-site link it writes to. + if let Err(e) = apply_repository_throttle(&dest_client, &op.destination_throttle).await { + srepl_terminal(&spec.target_ref, e, None).await?; + unreachable!("srepl_terminal always errors"); + } Ok(dest_client) } @@ -3360,6 +3626,12 @@ async fn run_browse_session_flow( source: e, }); } + // NO throttle here, and not by oversight: a browse session's work spec is + // built by the CLI (`crates/cli/src/cmd/browse/session.rs`) with + // `throttle: Default::default()`, so `spec.throttle` is structurally always + // empty and the call would be a guaranteed no-op. The session's reads run + // through `kopia` invocations the CLI execs into this pod; giving them a cap + // means teaching the CLI to resolve the repository's `moverDefaults` first. // Signal readiness: the marker flips the pod Ready so the CLI knows the // session is exec-able. A marker that cannot be written would leave the @@ -3649,36 +3921,241 @@ mod tests { assert!(!wants_log_only_reporter(&delete)); } - // --- delete_one / delete_batch against a fake kopia binary --- + // --- a fake kopia binary, shared by the shim-driven test modules below --- // // Mirrors `crates/kopia/tests/fake_shim.rs`: a tiny shell script stands in - // for kopia so the self-heal gate and the attempt-all-then-fail loop are - // exercised through the real `KopiaClient` subprocess path with no real - // kopia binary. + // for kopia so a code path is exercised through the real `KopiaClient` + // subprocess path with no real kopia binary. #[cfg(unix)] - mod delete_shim_tests { + struct Shim { + _dir: tempfile::TempDir, + path: PathBuf, + } + + #[cfg(unix)] + fn shim(script: &str) -> Shim { use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("kopia-shim.sh"); + std::fs::write(&path, script).unwrap(); + let mut perms = std::fs::metadata(&path).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&path, perms).unwrap(); + Shim { _dir: dir, path } + } + + #[cfg(unix)] + fn client_for(shim: &Shim) -> KopiaClient { + KopiaClient::builder().binary(shim.path.clone()).build() + } + + // --- #374: the repository throttle every own-connect flow now applies --- + #[cfg(unix)] + mod throttle_tests { use super::*; - struct Shim { - _dir: tempfile::TempDir, - path: PathBuf, + fn full_throttle() -> ThrottleSpec { + ThrottleSpec { + upload_bytes_per_second: Some(1_000), + download_bytes_per_second: Some(2_000), + read_ops_per_second: Some(3), + write_ops_per_second: Some(4), + } + } + + /// An empty throttle must not spawn kopia at all: the common case (no + /// `moverDefaults.throttle`) now runs on EVERY connect, so a wasted + /// subprocess per connect would be the cost of the fix. + #[tokio::test] + async fn empty_throttle_never_invokes_kopia() { + let marker_dir = tempfile::tempdir().unwrap(); + let marker = marker_dir.path().join("invoked"); + let s = shim(&format!( + "#!/bin/sh\ntouch \"{}\"\nexit 0\n", + marker.display() + )); + apply_repository_throttle(&client_for(&s), &ThrottleSpec::default()) + .await + .expect("an empty throttle is a no-op"); + assert!( + !marker.exists(), + "an empty throttle must not spawn a kopia process" + ); + } + + /// Every configured limit reaches kopia's argv. The controller resolves + /// all four knobs; dropping one on the floor here would be #374 again, + /// one field at a time. + #[tokio::test] + async fn every_limit_reaches_kopia_argv() { + let argv_dir = tempfile::tempdir().unwrap(); + let argv = argv_dir.path().join("argv"); + let s = shim(&format!( + "#!/bin/sh\necho \"$*\" > \"{}\"\nexit 0\n", + argv.display() + )); + apply_repository_throttle(&client_for(&s), &full_throttle()) + .await + .expect("throttle set succeeds"); + let recorded = std::fs::read_to_string(&argv).expect("the shim recorded its argv"); + for expected in [ + "repository throttle set", + "--upload-bytes-per-second 1000", + "--download-bytes-per-second 2000", + "--read-requests-per-second 3", + "--write-requests-per-second 4", + ] { + assert!( + recorded.contains(expected), + "kopia argv must carry `{expected}`; got: {recorded}" + ); + } } - fn shim(script: &str) -> Shim { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("kopia-shim.sh"); - std::fs::write(&path, script).unwrap(); - let mut perms = std::fs::metadata(&path).unwrap().permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&path, perms).unwrap(); - Shim { _dir: dir, path } + /// A throttle kopia refused is TERMINAL and labeled `ThrottleSet` — the + /// run must not continue uncapped over the link the user capped, and the + /// persisted failure must name the invocation that failed rather than + /// the connect that preceded it. + #[tokio::test] + async fn a_refused_throttle_is_terminal_and_labeled() { + let s = shim("#!/bin/sh\necho 'nope' >&2\nexit 1\n"); + let err = apply_repository_throttle(&client_for(&s), &full_throttle()) + .await + .expect_err("a failing throttle set must surface"); + assert!( + matches!( + err, + MoverError::Kopia { + op: KopiaOp::ThrottleSet, + .. + } + ), + "expected a ThrottleSet kopia failure, got {err:?}" + ); } - fn client_for(shim: &Shim) -> KopiaClient { - KopiaClient::builder().binary(shim.path.clone()).build() + /// This file, for the source-level ratchet below. `include_str!` is + /// relative to this file, so it is exactly the module being compiled. + const MAIN_RS: &str = include_str!("main.rs"); + + /// Every DIRECT `repository_connect*` call this file is allowed to + /// contain. Two are the funnels themselves; the rest are the connects + /// that legitimately do not carry a `MoverWorkSpec.throttle` — keep in + /// step with the exception list on `apply_repository_throttle`. + const SANCTIONED_DIRECT_CONNECTS: &[(&str, &str)] = &[ + ("connect_and_throttle", "the funnel (1 call)"), + ( + "bootstrap_connect", + "the bootstrap funnel (2 calls: rw + readonly)", + ), + ( + "main_serve", + "kopia server start; ServerWorkSpec has no throttle field (2 calls)", + ), + ( + "seed_connect_source", + "seed SOURCE/replica; dual-config connect, throttled in place from \ + op.replicaThrottle immediately after (1 call)", + ), + ( + "srepl_connect_source", + "snapshot-replication source; dual-config connect, throttled in place \ + from spec.throttle immediately after (1 call)", + ), + ( + "srepl_connect_dest", + "snapshot-replication destination; dual-config connect, throttled in \ + place from op.destinationThrottle immediately after (1 call)", + ), + ( + "run_browse_session_flow", + "CLI-built work spec, throttle always empty (1 call)", + ), + ]; + + /// Total direct calls the sanctioned list above accounts for. + const SANCTIONED_DIRECT_CONNECT_CALLS: usize = 9; + + /// **The #374 ratchet.** A mover flow must obtain its connection from + /// [`connect_and_throttle`] or [`bootstrap_connect`], so that "connects + /// a repository" and "applies the repository throttle" are one + /// operation. Nothing in the type system stops a new flow from calling + /// `client.repository_connect` directly and quietly shipping #374 again + /// — every hermetic gate would stay green, because an uncapped run + /// still succeeds. So the ratchet is source-level and deliberately + /// blunt: add a direct connect and this test fails, forcing the author + /// to either use a funnel or state why their connect carries no cap. + #[test] + fn connect_funnel_is_the_only_way_flows_connect() { + // Assembled at runtime, never written whole: a literal here would + // match THIS test's own source lines and inflate the count. + let base = ".repository_connect"; + let patterns = [ + format!("{base}("), + format!("{base}_readonly("), + format!("{base}_with("), + ]; + let direct = MAIN_RS + .lines() + .filter(|l| !l.trim_start().starts_with("//")) + .filter(|l| patterns.iter().any(|p| l.contains(p.as_str()))) + .count(); + assert_eq!( + direct, SANCTIONED_DIRECT_CONNECT_CALLS, + "direct `repository_connect*` calls in this file changed ({direct} found, \ + {SANCTIONED_DIRECT_CONNECT_CALLS} sanctioned). A flow that connects a \ + repository must go through `connect_and_throttle` (or `bootstrap_connect`) \ + so it cannot get a connection without its throttle — that separation is \ + exactly how #374 shipped. If a new connect genuinely carries no \ + `MoverWorkSpec.throttle`, add it here AND to the exception list on \ + `apply_repository_throttle`, with the reason. Sanctioned today: \ + {SANCTIONED_DIRECT_CONNECTS:?}" + ); + } + + /// A throttle failure during bootstrap must never be reported as a + /// CONNECT verdict. `bootstrap_connect_probe` reads a connect failure to + /// decide whether to CREATE or SEED a repository over the backend; a + /// throttle failure says nothing about the backend, and conflating them + /// could initialize a repository on the strength of an unrelated error. + #[test] + fn connect_and_throttle_failures_report_differently() { + let kopia_err = KopiaError::NonZeroExit { + args: "repository connect filesystem".into(), + code: Some(1), + class: KopiaErrorClass::NotFound, + stderr_tail: "repository not initialized".into(), + }; + let connect = BootstrapConnectFailure::Connect(kopia_err).into_result(); + let cf = connect.failure.expect("a connect failure is reported"); + assert_eq!(cf.kopia_error_class, "NotFound"); + assert_eq!(cf.op, None, "a bare connect error carries no op label"); + + let throttled = BootstrapConnectFailure::Throttle(MoverError::Kopia { + op: KopiaOp::ThrottleSet, + source: KopiaError::NonZeroExit { + args: "repository throttle set".into(), + code: Some(1), + class: KopiaErrorClass::RepositoryUnavailable, + stderr_tail: "backend went away".into(), + }, + }) + .into_result(); + let tf = throttled.failure.expect("a throttle failure is reported"); + assert_eq!(tf.kopia_error_class, "RepositoryUnavailable"); + assert_eq!( + tf.op.as_deref(), + Some(KopiaOp::ThrottleSet.as_str()), + "a throttle failure must name the throttle invocation, not the connect" + ); } + } + + // --- delete_one / delete_batch against a fake kopia binary --- + #[cfg(unix)] + mod delete_shim_tests { + use super::*; #[tokio::test] async fn delete_one_skips_self_heal_when_anchor_has_no_start_time() { diff --git a/crates/mover/src/status.rs b/crates/mover/src/status.rs index a766b7c1..c74e17ed 100644 --- a/crates/mover/src/status.rs +++ b/crates/mover/src/status.rs @@ -767,12 +767,31 @@ pub fn lease_blocked_body(owner: &str, reason: &str, message: &str) -> serde_jso /// `{ "status": ... }` body for a failed kopia maintenance call. pub fn maintenance_failed_body(e: &KopiaError) -> serde_json::Value { + maintenance_failure_body(e.class(), &e.to_string()) +} + +/// [`maintenance_failed_body`] for a maintenance step whose typed cause is a +/// [`MoverError`] rather than a bare [`KopiaError`] — the repository throttle +/// applied right after the maintenance connect. Same condition shape; the class +/// comes from the error's own classification +/// ([`MoverError::kopia_class`](crate::error::MoverError::kopia_class)) so the +/// controller's retry hint cannot drift from the message. +pub fn maintenance_failed_body_from_mover(e: &MoverError) -> serde_json::Value { + maintenance_failure_body(e.kopia_class(), &e.to_string()) +} + +/// The one `MaintenanceFailed` condition body, so the two constructors above +/// cannot drift in reason, wording or timestamp shape. +fn maintenance_failure_body( + class: kopiur_kopia::KopiaErrorClass, + message: &str, +) -> serde_json::Value { serde_json::json!({ "status": { "conditions": [lease_condition_body( "False", "MaintenanceFailed", - &format!("maintenance failed (class {}): {e}", e.class()), + &format!("maintenance failed (class {class}): {message}"), &chrono::Utc::now(), )], } @@ -1043,6 +1062,48 @@ mod tests { "2026-06-01T12:00:00Z".parse().unwrap() } + /// #374: a maintenance run can now fail on the repository THROTTLE, whose + /// typed cause is a `MoverError`. Both constructors must produce the same + /// `MaintenanceFailed` condition shape and carry the real class, so the + /// controller's retry hint is identical whichever step failed. + #[test] + fn both_maintenance_failure_bodies_share_one_condition_shape() { + let kopia = KopiaError::NonZeroExit { + args: "maintenance run".into(), + code: Some(1), + class: KopiaErrorClass::Locked, + stderr_tail: "repository is locked".into(), + }; + let from_kopia = maintenance_failed_body(&kopia); + let from_mover = maintenance_failed_body_from_mover(&crate::error::MoverError::Kopia { + op: crate::error::KopiaOp::ThrottleSet, + source: KopiaError::NonZeroExit { + args: "repository throttle set".into(), + code: Some(1), + class: KopiaErrorClass::Locked, + stderr_tail: "repository is locked".into(), + }, + }); + for body in [&from_kopia, &from_mover] { + let cond = &body["status"]["conditions"][0]; + assert_eq!(cond["type"], kopiur_api::maintenance::LEASE_OWNED_CONDITION); + assert_eq!(cond["status"], "False"); + assert_eq!(cond["reason"], "MaintenanceFailed"); + assert!( + cond["message"] + .as_str() + .is_some_and(|m| m.contains("class Locked")), + "the condition message must carry the real kopia class: {cond}" + ); + } + assert!( + from_mover["status"]["conditions"][0]["message"] + .as_str() + .is_some_and(|m| m.contains("repository throttle set")), + "a throttle failure must name the invocation that failed" + ); + } + #[test] fn stats_from_create_result_uses_crd_field_names() { let json = r#"{ diff --git a/crates/mover/src/workspec/mod.rs b/crates/mover/src/workspec/mod.rs index 263cf573..2a6dbd11 100644 --- a/crates/mover/src/workspec/mod.rs +++ b/crates/mover/src/workspec/mod.rs @@ -791,6 +791,21 @@ pub struct SeedOpSpec { /// `(SourceInfo, StartTime)`. Absent on old work specs (serde default). #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub resume: bool, + /// Throttle limits for the seed SOURCE (replica) connection — applied with + /// `kopia repository throttle set` right after that read-only connect. + /// + /// THIS repository's own side rides the work spec's + /// [`MoverWorkSpec::throttle`]: a migrate seed opens two repositories under + /// two kopia configs, and kopia's limits are per connection, so each side + /// needs its own resolved block. The controller merges the SOURCE + /// repository's `moverDefaults.throttle` with + /// `spec.seed.migrate.throttle.source` field by field; blob mode leaves it + /// empty (its source has no repository CR, and its caps ride + /// [`SeedSyncSpec`]'s `sync-to` speed flags). All-`None` ⇒ the mover skips + /// the call. Absent on old work specs (serde default ⇒ an uncapped replica + /// read, the pre-#374 behavior). + #[serde(default, skip_serializing_if = "ThrottleSpec::is_empty")] + pub replica_throttle: ThrottleSpec, } impl SeedOpSpec { @@ -1689,6 +1704,16 @@ pub struct SnapshotReplicateOp { /// [`PruningSpec::None`] (never prune) — absent on the wire means none. #[serde(default, skip_serializing_if = "PruningSpec::is_none")] pub pruning: PruningSpec, + /// Throttle limits for the DESTINATION connection (`kopia repository + /// throttle set` after the dest connect). The SOURCE side rides the work + /// spec's own [`MoverWorkSpec::throttle`] — a snapshot replication opens two + /// repositories under two kopia configs, and kopia's limits are per + /// connection, so each side needs its own resolved block. The controller + /// merges the destination repository's `moverDefaults.throttle` with + /// `spec.migrate.throttle.destination` field by field. All-`None` ⇒ the + /// mover skips the dest throttle call. + #[serde(default, skip_serializing_if = "ThrottleSpec::is_empty")] + pub destination_throttle: ThrottleSpec, } /// A fully-resolved repository CR reference on the wire (kind + name + diff --git a/crates/mover/src/workspec/tests.rs b/crates/mover/src/workspec/tests.rs index ade894b2..5a5b86ba 100644 --- a/crates/mover/src/workspec/tests.rs +++ b/crates/mover/src/workspec/tests.rs @@ -1712,6 +1712,11 @@ fn sample_snapshot_replicate_op() -> SnapshotReplicateOp { keep_weekly: Some(4), ..Default::default() }), + destination_throttle: ThrottleSpec { + upload_bytes_per_second: Some(5 * 1024 * 1024), + write_ops_per_second: Some(50), + ..Default::default() + }, } } @@ -1758,6 +1763,31 @@ fn snapshot_replicate_roundtrip_and_wire_shape() { assert_eq!(op["pruning"]["retention"]["keepDaily"], 7); assert_eq!(op["pruning"]["retention"]["keepWeekly"], 4); assert!(op["pruning"]["retention"].get("keepLatest").is_none()); + // The DESTINATION-side cap rides the op (the source's rides the work spec's + // own `throttle`): two connections, two blocks, camelCase, unset knobs elided. + assert_eq!(op["destinationThrottle"]["uploadBytesPerSecond"], 5242880); + assert_eq!(op["destinationThrottle"]["writeOpsPerSecond"], 50); + assert!( + op["destinationThrottle"] + .get("downloadBytesPerSecond") + .is_none(), + "unset dest knobs are elided: {op}" + ); +} + +#[test] +fn snapshot_replicate_empty_destination_throttle_is_elided_from_the_wire() { + // The common case must not grow a key: an all-None dest throttle is the + // mover's "skip `throttle set`" signal, so it serializes to nothing at all. + let op = SnapshotReplicateOp { + destination_throttle: ThrottleSpec::default(), + ..sample_snapshot_replicate_op() + }; + let v = serde_json::to_value(&op).unwrap(); + assert!( + v.get("destinationThrottle").is_none(), + "an empty dest throttle is elided: {v}" + ); } #[test] @@ -1799,6 +1829,10 @@ fn snapshot_replicate_old_wire_decodes_with_selection_and_pruning_defaulted() { assert!(op.pruning.is_none()); assert_eq!(op.destination_repository.namespace, None); assert_eq!(op.source_repository.namespace, None); + // A spec stamped before `destinationThrottle` existed decodes to an empty + // block — i.e. an old controller driving a new mover leaves the destination + // uncapped rather than failing to decode. + assert!(op.destination_throttle.is_empty()); } #[test] @@ -2395,6 +2429,9 @@ fn blob_seed() -> SeedOpSpec { migrate: None, allow_empty_source: false, resume: false, + // Blob mode caps its copy through `sync-to`'s own speed flags above; + // there is no source repository CR to carry `moverDefaults`. + replica_throttle: ThrottleSpec::default(), } } @@ -2417,6 +2454,11 @@ fn migrate_seed() -> SeedOpSpec { }), allow_empty_source: true, resume: false, + replica_throttle: ThrottleSpec { + download_bytes_per_second: Some(20 * 1024 * 1024), + read_ops_per_second: Some(200), + ..Default::default() + }, } } @@ -2541,6 +2583,55 @@ fn resume_rides_the_wire_and_defaults_off_on_old_specs() { assert_eq!(serde_json::from_value::(v).unwrap(), resuming); } +#[test] +fn the_replica_throttle_rides_the_seed_op_and_defaults_empty_on_old_specs() { + // The SOURCE (replica) side's cap rides the op; THIS repository's side rides + // the work spec's own `throttle`. Two repositories, two kopia connections, + // two independent blocks — a migrate seed is the only flow that opens both. + let v = serde_json::to_value(migrate_seed()).unwrap(); + assert_eq!( + v["replicaThrottle"]["downloadBytesPerSecond"], + 20 * 1024 * 1024 + ); + assert_eq!(v["replicaThrottle"]["readOpsPerSecond"], 200); + assert!( + v["replicaThrottle"].get("uploadBytesPerSecond").is_none(), + "unset replica knobs are elided: {v}" + ); + assert_eq!( + serde_json::from_value::(v).unwrap(), + migrate_seed() + ); + + // The common case must not grow a key: an all-None block is the mover's + // "skip `throttle set`" signal, so it serializes to nothing at all. + let uncapped = SeedOpSpec { + replica_throttle: ThrottleSpec::default(), + ..migrate_seed() + }; + assert!( + serde_json::to_value(&uncapped) + .unwrap() + .get("replicaThrottle") + .is_none(), + "an empty replica throttle is elided from the wire" + ); + + // Old→new: a spec stamped before `replicaThrottle` existed decodes to an + // empty block, i.e. an old controller driving a new mover leaves the replica + // read uncapped rather than failing to decode. + let old: SeedOpSpec = serde_json::from_value(serde_json::json!({ + "from": { "repository": { + "kind": "Repository", + "name": "offsite", + "connect": { "filesystem": { "path": "/mnt/offsite" } } + } }, + "sourceDescription": "Repository/offsite" + })) + .unwrap(); + assert!(old.replica_throttle.is_empty()); +} + #[test] fn seed_migrate_policies_default_to_an_explicit_no_policies() { // kopia's OWN default copies the source's policies, retention included, diff --git a/crates/xtask/tests/snapshots/snapshots_test__repository_crd.snap b/crates/xtask/tests/snapshots/snapshots_test__repository_crd.snap index 8d74a7b2..6c18fb4b 100644 --- a/crates/xtask/tests/snapshots/snapshots_test__repository_crd.snap +++ b/crates/xtask/tests/snapshots/snapshots_test__repository_crd.snap @@ -2958,6 +2958,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/crates/xtask/tests/snapshots/snapshots_test__snapshot_replication_crd.snap b/crates/xtask/tests/snapshots/snapshots_test__snapshot_replication_crd.snap index ff013f22..5e5542ae 100644 --- a/crates/xtask/tests/snapshots/snapshots_test__snapshot_replication_crd.snap +++ b/crates/xtask/tests/snapshots/snapshots_test__snapshot_replication_crd.snap @@ -110,6 +110,75 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + migrate` has no speed flags, so each side is applied as `kopia repository + throttle set` on that side's connection; `source` overrides the source + repository's `moverDefaults.throttle` and `destination` the destination + repository's, field by field (a field left unset keeps that repository's + default). Absent: both sides use their repository's defaults. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object mover: description: Mover (Job pod) overrides for the replication run. diff --git a/deploy/crds/all-crds.yaml b/deploy/crds/all-crds.yaml index 3754f9b9..eedb5104 100644 --- a/deploy/crds/all-crds.yaml +++ b/deploy/crds/all-crds.yaml @@ -2954,6 +2954,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- @@ -6739,6 +6818,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- @@ -11091,13 +11249,17 @@ spec: description: |- When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a - `target.populator` a PVC already claims it), NOT when the Restore was created. If the - referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate - falls through unverified and the anchor can be stamped as early as creation. Stamped - once and then honored verbatim, so the window survives controller restarts and Job pod - retries; cleared when a populator re-opens resolution for a re-created claim, so that - claim gets the full window again. Absent means the window has not opened yet (or no - `policy.waitTimeout` is configured, in which case there is no window to anchor). + `target.populator` a PVC already claims it), NOT when the Restore was created. The + window does NOT open while the referenced `Repository` object or `fromPolicy` + `SnapshotPolicy` doesn't exist: the restore parks in `Pending` + (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It + DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so + `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose + mover Job already launched. Stamped once and then honored verbatim, so the window + survives controller restarts and Job pod retries; cleared when a populator re-opens + resolution for a re-created claim, so that claim gets the full window again. Absent + means the window has not opened yet (or no `policy.waitTimeout` is configured, in + which case there is no window to anchor). nullable: true type: string type: object @@ -13077,6 +13239,75 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + migrate` has no speed flags, so each side is applied as `kopia repository + throttle set` on that side's connection; `source` overrides the source + repository's `moverDefaults.throttle` and `destination` the destination + repository's, field by field (a field left unset keeps that repository's + default). Absent: both sides use their repository's defaults. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object mover: description: Mover (Job pod) overrides for the replication run. diff --git a/deploy/crds/clusterrepositories.yaml b/deploy/crds/clusterrepositories.yaml index 6e064dd1..0b85edf4 100644 --- a/deploy/crds/clusterrepositories.yaml +++ b/deploy/crds/clusterrepositories.yaml @@ -3007,6 +3007,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/deploy/crds/repositories.yaml b/deploy/crds/repositories.yaml index 790b7667..b217e606 100644 --- a/deploy/crds/repositories.yaml +++ b/deploy/crds/repositories.yaml @@ -2954,6 +2954,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/deploy/crds/restores.yaml b/deploy/crds/restores.yaml index 73d541d2..172997d0 100644 --- a/deploy/crds/restores.yaml +++ b/deploy/crds/restores.yaml @@ -944,13 +944,17 @@ spec: description: |- When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a - `target.populator` a PVC already claims it), NOT when the Restore was created. If the - referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate - falls through unverified and the anchor can be stamped as early as creation. Stamped - once and then honored verbatim, so the window survives controller restarts and Job pod - retries; cleared when a populator re-opens resolution for a re-created claim, so that - claim gets the full window again. Absent means the window has not opened yet (or no - `policy.waitTimeout` is configured, in which case there is no window to anchor). + `target.populator` a PVC already claims it), NOT when the Restore was created. The + window does NOT open while the referenced `Repository` object or `fromPolicy` + `SnapshotPolicy` doesn't exist: the restore parks in `Pending` + (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It + DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so + `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose + mover Job already launched. Stamped once and then honored verbatim, so the window + survives controller restarts and Job pod retries; cleared when a populator re-opens + resolution for a re-created claim, so that claim gets the full window again. Absent + means the window has not opened yet (or no `policy.waitTimeout` is configured, in + which case there is no window to anchor). nullable: true type: string type: object diff --git a/deploy/crds/snapshotreplications.yaml b/deploy/crds/snapshotreplications.yaml index 197e684b..3bfb4893 100644 --- a/deploy/crds/snapshotreplications.yaml +++ b/deploy/crds/snapshotreplications.yaml @@ -107,6 +107,75 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + migrate` has no speed flags, so each side is applied as `kopia repository + throttle set` on that side's connection; `source` overrides the source + repository's `moverDefaults.throttle` and `destination` the destination + repository's, field by field (a field left unset keeps that repository's + default). Absent: both sides use their repository's defaults. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object mover: description: Mover (Job pod) overrides for the replication run. diff --git a/deploy/examples/39-snapshot-replication.yaml b/deploy/examples/39-snapshot-replication.yaml index 24e34a3d..66e276c1 100644 --- a/deploy/examples/39-snapshot-replication.yaml +++ b/deploy/examples/39-snapshot-replication.yaml @@ -105,6 +105,27 @@ spec: # Whether kopia POLICIES ride along: none (default — a Kopiur-managed # destination keeps retention CR-driven), copy, or copyOverwrite. policies: none + # Bandwidth/ops caps for THIS replication's runs, one block per side. + # `kopia snapshot migrate` has no speed flags, so each side is applied as + # `kopia repository throttle set` on that side's connection before the + # migrate runs. Each side overrides THAT side's repository's + # `moverDefaults.throttle` field by field: a knob set here wins, a knob left + # unset keeps the repository's value. Every set knob must be >= 1. + # + # Caveat worth knowing before you tune these down: byte caps only bite on + # COLD backend traffic (cached content bypasses the limiter), and on + # small-object workloads the real throughput lands far below the nominal + # rate. Treat them as a ceiling for large transfers and measure. + throttle: + # Reading out of nas-primary — on the LAN, so leave it roomy. + source: + downloadBytesPerSecond: 209715200 # 200 MiB/s + # Writing to the off-site bucket — the scarce link, and the one worth + # capping. Unset knobs (downloadBytesPerSecond, readOpsPerSecond) fall + # back to the destination Repository's own moverDefaults.throttle. + destination: + uploadBytesPerSecond: 10485760 # 10 MiB/s + writeOpsPerSecond: 100 # What happens to already-replicated copies on later runs. Exactly one of: # none: never prune (the default when `pruning` is omitted) # mirrorSource: delete a copy when its snapshot vanished from the source diff --git a/deploy/examples/42-repository-seed-from-repository.yaml b/deploy/examples/42-repository-seed-from-repository.yaml index 0a0f209c..e425e30d 100644 --- a/deploy/examples/42-repository-seed-from-repository.yaml +++ b/deploy/examples/42-repository-seed-from-repository.yaml @@ -33,7 +33,8 @@ # absent namespace resolves in this CR's own namespace); `seed.migrate` is # migrate-mode-only tuning and is rejected alongside a `backend:` source; # `policies` is the same enum SnapshotReplication uses and defaults to `none` -# (an explicit `--no-policies`). +# (an explicit `--no-policies`); `seed.migrate.throttle` is the same per-side +# {source, destination} shape SnapshotReplication's `spec.migrate.throttle` uses. # # The `# --8<-- [start:…]` / `# --8<-- [end:…]` lines are snippet-section markers # (docs/repositories.md pulls the Repository on its own). Plain YAML comments — @@ -101,6 +102,32 @@ spec: # window; imported kopia policies could delete manifests behind the # operator's back. policies: none + # PER-SIDE bandwidth/ops caps for THIS seed. `kopia snapshot migrate` has + # no speed flags of its own, so each side is applied as `kopia repository + # throttle set` on that side's connection before the copy runs — and a + # migrate seed is two repositories under two kopia configs, so the two caps + # are independent. `source` is the REPLICA being read; `destination` is + # THIS repository, the one being written. + # + # Each side OVERRIDES that side's repository's `moverDefaults.throttle` + # field by field: a knob set here wins, a knob left unset keeps that + # repository's value. Only applied while the seed is armed — later connects + # to this (now-initialized) repository are capped by `moverDefaults.throttle` + # alone. + # + # Every knob set must be >= 1 (a 0 is rejected: kopia's "no limit" is the + # ABSENT knob). Caps bind only COLD backend traffic, and on small objects + # bite far harder than the nominal rate — set them as a generous ceiling + # for the bulk transfer, and remember a seed already runs against + # failurePolicy.activeDeadlineSeconds below. + throttle: + source: + # Protect the off-site link this replica lives behind. + downloadBytesPerSecond: 10485760 # 10 MiB/s + readOpsPerSecond: 200 + destination: + # Writing into the new primary — usually the roomier side. + uploadBytesPerSecond: 209715200 # 200 MiB/s allowEmptySource: false # Copy the SOURCE repository's credential Secrets into the seeding Job's # namespace for the duration of the run. Needed whenever the source's Secrets diff --git a/deploy/helm/kopiur/crds/clusterrepositories.yaml b/deploy/helm/kopiur/crds/clusterrepositories.yaml index 6e064dd1..0b85edf4 100644 --- a/deploy/helm/kopiur/crds/clusterrepositories.yaml +++ b/deploy/helm/kopiur/crds/clusterrepositories.yaml @@ -3007,6 +3007,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/deploy/helm/kopiur/crds/repositories.yaml b/deploy/helm/kopiur/crds/repositories.yaml index 790b7667..b217e606 100644 --- a/deploy/helm/kopiur/crds/repositories.yaml +++ b/deploy/helm/kopiur/crds/repositories.yaml @@ -2954,6 +2954,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/deploy/helm/kopiur/crds/restores.yaml b/deploy/helm/kopiur/crds/restores.yaml index 73d541d2..172997d0 100644 --- a/deploy/helm/kopiur/crds/restores.yaml +++ b/deploy/helm/kopiur/crds/restores.yaml @@ -944,13 +944,17 @@ spec: description: |- When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a - `target.populator` a PVC already claims it), NOT when the Restore was created. If the - referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate - falls through unverified and the anchor can be stamped as early as creation. Stamped - once and then honored verbatim, so the window survives controller restarts and Job pod - retries; cleared when a populator re-opens resolution for a re-created claim, so that - claim gets the full window again. Absent means the window has not opened yet (or no - `policy.waitTimeout` is configured, in which case there is no window to anchor). + `target.populator` a PVC already claims it), NOT when the Restore was created. The + window does NOT open while the referenced `Repository` object or `fromPolicy` + `SnapshotPolicy` doesn't exist: the restore parks in `Pending` + (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It + DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so + `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose + mover Job already launched. Stamped once and then honored verbatim, so the window + survives controller restarts and Job pod retries; cleared when a populator re-opens + resolution for a re-created claim, so that claim gets the full window again. Absent + means the window has not opened yet (or no `policy.waitTimeout` is configured, in + which case there is no window to anchor). nullable: true type: string type: object diff --git a/deploy/helm/kopiur/crds/snapshotreplications.yaml b/deploy/helm/kopiur/crds/snapshotreplications.yaml index 197e684b..3bfb4893 100644 --- a/deploy/helm/kopiur/crds/snapshotreplications.yaml +++ b/deploy/helm/kopiur/crds/snapshotreplications.yaml @@ -107,6 +107,75 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + migrate` has no speed flags, so each side is applied as `kopia repository + throttle set` on that side's connection; `source` overrides the source + repository's `moverDefaults.throttle` and `destination` the destination + repository's, field by field (a field left unset keeps that repository's + default). Absent: both sides use their repository's defaults. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object mover: description: Mover (Job pod) overrides for the replication run. diff --git a/docs/dev/api-conventions.md b/docs/dev/api-conventions.md index 69141cf2..e6cb9300 100644 --- a/docs/dev/api-conventions.md +++ b/docs/dev/api-conventions.md @@ -49,6 +49,8 @@ Use `k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, Condition}` Every credential/policy/identity/schedule surface is a **sub-object**, not a leaf field. Optionals: `#[serde(default, skip_serializing_if = "Option::is_none")] pub x: Option`. Bools that default false: `#[serde(default, skip_serializing_if = "std::ops::Not::not")]`. Vecs: `#[serde(default, skip_serializing_if = "Vec::is_empty")]`. +Exception: `manualRun.completedAt` (`ReplicationManualRunStatus` and `maintenance::ManualRunStatus`) deliberately omits `skip_serializing_if` — a non-terminal-phase status patch must emit an explicit JSON `null` so the RFC-7386 merge-patch clears a stale timestamp left standing from a prior run; see the field comments and #394. + ## 4a. Schema defaults (`#[schemars(default = …)]`) A schemars `default` becomes a CRD `default:` that the **API server materializes diff --git a/docs/field-reference.md b/docs/field-reference.md index 9ba76f97..8c69b31f 100644 --- a/docs/field-reference.md +++ b/docs/field-reference.md @@ -858,6 +858,32 @@ Externally tagged — set **exactly one** of: `nfs` · `pvc`. | `latestOnly` | boolean | — | Copy only each source identity's most recent snapshot instead of its full history (`kopia snapshot migrate --latest`). Default `false` — a seed exists to recover history, so the full copy is the sane default; set this when you only need the latest restore point back quickly. | | `parallel` | integer | —
min 0 | `--parallel`: snapshots migrated concurrently (kopia default `1` — sequential). Must be >= 1 when set. | | `policies` | enum: none \| copy \| copyOverwrite | `none` | Whether the source's kopia **policies** are copied along with the snapshots. Defaults to `PolicyCopyMode::None` (an explicit `--no-policies`), not kopia's own copy-by-default: retention in a kopiur-managed repository is driven by `Snapshot` CRs, and importing the source's kopia-side policies could delete manifests behind the operator's back. | +| `throttle` | [object](#repository-spec-seed-migrate-throttle) | — | Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens two repositories under two kopia connections and `snapshot migrate` has no speed flags of its own, so each side is applied as `kopia repository throttle set` on that side's connection:
* `source` caps the REPLICA — the repository named by `spec.seed.from.repository`, opened read-only — overriding **its** `moverDefaults.throttle`; * `destination` caps THIS repository, the one being seeded, overriding **its own** `moverDefaults.throttle`.
Each side overrides field by field: a knob set here wins, a knob left unset keeps that repository's default. Absent: both sides use their repository's defaults. Applies only while the seed is armed — an ordinary connect to the now-initialized repository is capped by `moverDefaults.throttle` alone. | + +###### `spec.seed.migrate.throttle` { #repository-spec-seed-migrate-throttle } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `destination` | [object](#repository-spec-seed-migrate-throttle-destination) | — | Caps for the DESTINATION (write) side, overriding the destination repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the destination connection the migrate writes through. | +| `source` | [object](#repository-spec-seed-migrate-throttle-source) | — | Caps for the SOURCE (read) side, overriding the source repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the migrate's read-only source connection — accepted there, so a read-only source is throttled like any other. | + +###### `spec.seed.migrate.throttle.destination` { #repository-spec-seed-migrate-throttle-destination } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | + +###### `spec.seed.migrate.throttle.source` { #repository-spec-seed-migrate-throttle-source } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | ##### `spec.seed.sync` { #repository-spec-seed-sync } @@ -1890,6 +1916,32 @@ Externally tagged — set **exactly one** of: `nfs` · `pvc`. | `latestOnly` | boolean | — | Copy only each source identity's most recent snapshot instead of its full history (`kopia snapshot migrate --latest`). Default `false` — a seed exists to recover history, so the full copy is the sane default; set this when you only need the latest restore point back quickly. | | `parallel` | integer | —
min 0 | `--parallel`: snapshots migrated concurrently (kopia default `1` — sequential). Must be >= 1 when set. | | `policies` | enum: none \| copy \| copyOverwrite | `none` | Whether the source's kopia **policies** are copied along with the snapshots. Defaults to `PolicyCopyMode::None` (an explicit `--no-policies`), not kopia's own copy-by-default: retention in a kopiur-managed repository is driven by `Snapshot` CRs, and importing the source's kopia-side policies could delete manifests behind the operator's back. | +| `throttle` | [object](#clusterrepository-spec-seed-migrate-throttle) | — | Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens two repositories under two kopia connections and `snapshot migrate` has no speed flags of its own, so each side is applied as `kopia repository throttle set` on that side's connection:
* `source` caps the REPLICA — the repository named by `spec.seed.from.repository`, opened read-only — overriding **its** `moverDefaults.throttle`; * `destination` caps THIS repository, the one being seeded, overriding **its own** `moverDefaults.throttle`.
Each side overrides field by field: a knob set here wins, a knob left unset keeps that repository's default. Absent: both sides use their repository's defaults. Applies only while the seed is armed — an ordinary connect to the now-initialized repository is capped by `moverDefaults.throttle` alone. | + +###### `spec.seed.migrate.throttle` { #clusterrepository-spec-seed-migrate-throttle } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `destination` | [object](#clusterrepository-spec-seed-migrate-throttle-destination) | — | Caps for the DESTINATION (write) side, overriding the destination repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the destination connection the migrate writes through. | +| `source` | [object](#clusterrepository-spec-seed-migrate-throttle-source) | — | Caps for the SOURCE (read) side, overriding the source repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the migrate's read-only source connection — accepted there, so a read-only source is throttled like any other. | + +###### `spec.seed.migrate.throttle.destination` { #clusterrepository-spec-seed-migrate-throttle-destination } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | + +###### `spec.seed.migrate.throttle.source` { #clusterrepository-spec-seed-migrate-throttle-source } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | ##### `spec.seed.sync` { #clusterrepository-spec-seed-sync } @@ -3077,7 +3129,7 @@ Externally tagged — set **exactly one** of: `pvcConsumer` · `snapshot` · `wo | `sourceKind` | string | — | The pinned source kind (`SnapshotRef`/`FromPolicy`/`Identity`); backs the `SOURCE` printer column. | | `target` | [object](#restore-status-target) | — | Resolved target details (the PVC written to / populator handshake). | | `timing` | [object](#restore-status-timing) | — | Start/end timestamps for the restore run. | -| `waitStartedAt` | string | — | When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a `target.populator` a PVC already claims it), NOT when the Restore was created. If the referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate falls through unverified and the anchor can be stamped as early as creation. Stamped once and then honored verbatim, so the window survives controller restarts and Job pod retries; cleared when a populator re-opens resolution for a re-created claim, so that claim gets the full window again. Absent means the window has not opened yet (or no `policy.waitTimeout` is configured, in which case there is no window to anchor). | +| `waitStartedAt` | string | — | When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a `target.populator` a PVC already claims it), NOT when the Restore was created. The window does NOT open while the referenced `Repository` object or `fromPolicy` `SnapshotPolicy` doesn't exist: the restore parks in `Pending` (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose mover Job already launched. Stamped once and then honored verbatim, so the window survives controller restarts and Job pod retries; cleared when a populator re-opens resolution for a re-created claim, so that claim gets the full window again. Absent means the window has not opened yet (or no `policy.waitTimeout` is configured, in which case there is no window to anchor). | #### `status.conditions[]` { #restore-status-conditions } @@ -3757,6 +3809,32 @@ Externally tagged — set **exactly one** of: `pvcConsumer` · `snapshot` · `wo | --- | --- | --- | --- | | `parallel` | integer | —
min 0 | `--parallel`: number of snapshots migrated concurrently (kopia default `1` — sequential). Must be >= 1 when set. | | `policies` | enum: none \| copy \| copyOverwrite | `none` | Whether kopia **policies** attached to the copied sources are also copied to the destination. Defaults to `PolicyCopyMode::None`. | +| `throttle` | [object](#snapshotreplication-spec-migrate-throttle) | — | Bandwidth/ops caps for THIS replication's runs, per side. `snapshot migrate` has no speed flags, so each side is applied as `kopia repository throttle set` on that side's connection; `source` overrides the source repository's `moverDefaults.throttle` and `destination` the destination repository's, field by field (a field left unset keeps that repository's default). Absent: both sides use their repository's defaults. | + +##### `spec.migrate.throttle` { #snapshotreplication-spec-migrate-throttle } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `destination` | [object](#snapshotreplication-spec-migrate-throttle-destination) | — | Caps for the DESTINATION (write) side, overriding the destination repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the destination connection the migrate writes through. | +| `source` | [object](#snapshotreplication-spec-migrate-throttle-source) | — | Caps for the SOURCE (read) side, overriding the source repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the migrate's read-only source connection — accepted there, so a read-only source is throttled like any other. | + +###### `spec.migrate.throttle.destination` { #snapshotreplication-spec-migrate-throttle-destination } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | + +###### `spec.migrate.throttle.source` { #snapshotreplication-spec-migrate-throttle-source } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | #### `spec.mover` { #snapshotreplication-spec-mover } diff --git a/docs/gitops.md b/docs/gitops.md index b1f13489..0d50841e 100644 --- a/docs/gitops.md +++ b/docs/gitops.md @@ -105,7 +105,10 @@ exactly what they remove. `dataSourceRef` (see [deploy-or-restore](restores.md#deploy-or-restore-gitops)). Its `waitTimeout` window opens when the restore can first proceed, not when you applied it, so a populator that sits in Git for months does not arrive with its - window already spent. + window already spent. The same holds for apply ordering within one commit: a + `Restore` whose `Repository` or `SnapshotPolicy` hasn't landed yet parks in + `Pending` (`RestoreReferentMissing`) with the window still closed, and un-parks + by itself once the referenced object appears. `spec.seed` is also **mutable**: editing it mid-seed lets the running Job finish, discards its result as stale, and relaunches for the live spec — so a GitOps diff --git a/docs/install.md b/docs/install.md index 009d825f..ead5ce49 100644 --- a/docs/install.md +++ b/docs/install.md @@ -262,6 +262,8 @@ The 9 CRDs ship in the chart's special `crds/` directory. Helm treats that direc > ``` > > A GitOps flow (Flux/Argo with a `CreateReplace` sync policy) applies CRD changes automatically. Anyone managing CRDs entirely out of band just applies `deploy/crds/` themselves — `helm install` skips the ones that already exist. +> +> Skipping this is usually **silent**, not loud: when a release adds a new spec field, an apiserver still running the old schema **prunes** that field out of every object you apply. The object admits cleanly, `kubectl get -o yaml` shows no trace of the field, and the feature you configured simply never happens. If a newly-documented field seems to be ignored, check the live CRD before anything else (`kubectl get crd .kopiur.home-operations.com -o yaml | grep `); `kubectl kopiur doctor` flags known-stale schemas too. The CRDs and RBAC shipped by the chart are **generated** by `cargo xtask gen-crds` / `cargo xtask gen-rbac` and checked in under `deploy/crds/` and `deploy/rbac/`. Those xtasks are the source of truth. diff --git a/docs/reference/crds/repository.md b/docs/reference/crds/repository.md index a76144ab..258855cc 100644 --- a/docs/reference/crds/repository.md +++ b/docs/reference/crds/repository.md @@ -50,6 +50,23 @@ holding zero snapshots. `failurePolicy.activeDeadlineSeconds` defaults to **8640 (24 h) while seeding, versus the 120 s a routine connect gets. See [Repositories → `seed`](../../repositories.md#seed--initialize-a-new-repository-from-a-replica). +Capping the copy differs by mode. Blob mode uses `sync.maxDownloadSpeedBytesPerSecond` +/ `maxUploadSpeedBytesPerSecond` — `kopia repository sync-to` takes real speed +flags. Migrate mode's `kopia snapshot migrate` takes **none**, so +`migrate.throttle` is applied as `kopia repository throttle set` on each +connection before the copy runs, **per side**: `throttle.source` caps the +replica's read connection and `throttle.destination` this repository's write +connection, each a [`throttle`](shared-types.md#moverdefaults) block +(`uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, +`writeOpsPerSecond`; every set knob must be `>= 1`, webhook-enforced). Each side +**overrides that side's repository's `moverDefaults.throttle` field by field** — +a knob set here wins, a knob left unset keeps the repository's value — and only +while the seed is armed; later connects are capped by `moverDefaults.throttle` +alone. Two repositories, two connections, two independent blocks: the replica cap +says nothing about this repository's and vice versa. Applying a cap to the +read-only replica connection is supported (kopia accepts `throttle set` there). +See [Throttling a seed](../../repositories.md#throttling-a-seed). + ### `moverDefaults` Base mover configuration — security context, pod security context, resources, diff --git a/docs/reference/crds/restore.md b/docs/reference/crds/restore.md index b6742ec3..e6367c97 100644 --- a/docs/reference/crds/restore.md +++ b/docs/reference/crds/restore.md @@ -36,7 +36,7 @@ The rest mirror `kopia snapshot restore`'s own flags one-to-one and are all tri- ### `policy` -How the restore reacts to a missing snapshot. `onMissingSnapshot` is `Fail` (fail-closed; the default for explicit `snapshotRef`/`identity` sources, so an explicit restore can never silently no-op) or `Continue` (proceed with an empty, deploy-or-restore volume; the default for `fromPolicy`). `waitTimeout` (e.g. `5m`) bounds how long the restore waits for the source snapshot to appear before giving up. The window opens when the restore can first proceed — its repository is `Ready`, and for `target.populator` a PVC claims it — and that instant is recorded in `status.waitStartedAt`; it is **not** measured from `metadata.creationTimestamp`. See [Restores → `waitTimeout`](../../restores.md#waittimeout--wait-before-giving-up). +How the restore reacts to a missing snapshot. `onMissingSnapshot` is `Fail` (fail-closed; the default for explicit `snapshotRef`/`identity` sources, so an explicit restore can never silently no-op) or `Continue` (proceed with an empty, deploy-or-restore volume; the default for `fromPolicy`). `waitTimeout` (e.g. `5m`) bounds how long the restore waits for the source snapshot to appear before giving up. The window opens when the restore can first proceed — its repository is `Ready`, and for `target.populator` a PVC claims it — and that instant is recorded in `status.waitStartedAt`; it is **not** measured from `metadata.creationTimestamp`. It does not open at all while the referenced `Repository` object or `fromPolicy` `SnapshotPolicy` is missing (the restore parks `Pending`/`RestoreReferentMissing`), but it does still open for a `snapshotRef` whose `Snapshot` row hasn't appeared yet, so `onMissingSnapshot` can fire for a ref that never does. See [Restores → `waitTimeout`](../../restores.md#waittimeout--wait-before-giving-up). ### `mover` diff --git a/docs/reference/crds/shared-types.md b/docs/reference/crds/shared-types.md index b0dd53fa..e914a4d7 100644 --- a/docs/reference/crds/shared-types.md +++ b/docs/reference/crds/shared-types.md @@ -100,10 +100,44 @@ cache. See [movers](../../movers.md). mover Job so finished Jobs self-GC. Defaults to 1h when neither the repo nor the recipe sets one. - **`throttle`** — repository throttle limits (`kopia repository throttle set`) - applied by every mover after it connects, so a run doesn't saturate the link or - hammer the object store: `uploadBytesPerSecond`, `downloadBytesPerSecond`, - `readOpsPerSecond`, `writeOpsPerSecond` (each unset leaves kopia's current - limit). + applied by every mover after it connects to this repository, so a run doesn't + saturate the link or hammer the object store: `uploadBytesPerSecond`, + `downloadBytesPerSecond`, `readOpsPerSecond`, `writeOpsPerSecond` (each unset + leaves kopia's current limit). Honored by the transfer movers — bootstrap, + backup, restore, maintenance, verification, repository replication, snapshot + replication and seed. kopia's limits are **per connection**, so a run that opens two + repositories caps each from *that* repository's own `moverDefaults`: a + [`SnapshotReplication`](snapshot-replication.md) caps its source connection + from the source repo's defaults and its destination connection from the + destination repo's, each overridable per CR via + [`spec.migrate.throttle`](snapshot-replication.md#migrate). A migrate-mode + [`seed`](repository.md#seed) is the same shape: the replica's connection is + capped from the *replica's* defaults and this repository's from its own, each + overridable while the seed is armed via `spec.seed.migrate.throttle.source` / + `.destination`. + + Three things it does **not** cap today: the interactive + [`serve`/browse](../../server.md) session (a read-only UI session, not a batch + transfer); a `RepositoryReplication`'s **destination** side, because `kopia + repository sync-to` copies blobs without opening the destination as a + repository — its caps live on [`spec.sync`](repository-replication.md) instead; + and **batched snapshot deletions and pins**, whose work specs leave the throttle + block empty, so a repository-wide cap does not currently reach them (deletion + load is instead bounded by per-repository batching and the cluster-wide + `maxConcurrentDeleteJobs` backstop). + +/// warning | A byte cap bites much harder than its number suggests + +A `*BytesPerSecond` cap is enforced against **cold** backend traffic only — +content already in the mover's kopia cache is read without touching the limiter, +so a warm re-run can look completely unthrottled. And on small-object workloads +the effective throughput lands far below the nominal rate: a measured 2 MB/s cap +took ~14 s to move a 28 KiB cold repository, while caps of 10 MB/s and up often +did not bind at all at that size. Treat these as a **ceiling for large transfers** +— set them generously and verify against your own data, rather than tuning them +down to a number that looks safe. + +/// The `securityContext`/`podSecurityContext`/`resources`/`cache` fields resolve by field-wise merge: a repo-wide default composes with a partial per-recipe diff --git a/docs/reference/crds/snapshot-replication.md b/docs/reference/crds/snapshot-replication.md index 1ff614f7..61db81aa 100644 --- a/docs/reference/crds/snapshot-replication.md +++ b/docs/reference/crds/snapshot-replication.md @@ -33,6 +33,9 @@ Tuning for the underlying `kopia snapshot migrate`: | --- | --- | | `parallel` | Snapshots migrated concurrently (kopia default `1` — sequential; must be `>= 1` when set, webhook-enforced). The main knob for large first runs. | | `policies` | Whether kopia **policy** objects ride along: `none` (default — Kopiur pins retention CR-side, so imported kopia policies are usually unwanted), `copy` (copy where absent), `copyOverwrite` (copy and overwrite). | +| `throttle.source` / `throttle.destination` | Bandwidth/ops caps for this replication's runs, **per side** — each a [`throttle`](shared-types.md#moverdefaults) block (`uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, `writeOpsPerSecond`; every set knob must be `>= 1`, webhook-enforced). Each side **overrides that side's repository's `moverDefaults.throttle` field by field** — a knob set here wins, a knob left unset keeps the repository's value. Absent: both sides use their repository's defaults. See [Throttling a replication](../../snapshot-replication.md#throttling-a-replication). | + +`kopia snapshot migrate` has **no speed flags of its own**, so `throttle` is not passed to the migrate command: each side is applied as `kopia repository throttle set` on that side's connection *before* the migrate runs, and kopia persists the limits into that connection's config for the migrate to pick up. Two repositories, two connections, two independent blocks — the source cap says nothing about the destination and vice versa. Applying a cap to the read-only source connection is supported (kopia accepts `throttle set` there), so a `mode: ReadOnly` source is throttled like any other. ### `pruning` diff --git a/docs/repositories.md b/docs/repositories.md index 2f4d41f6..c3ffcc32 100644 --- a/docs/repositories.md +++ b/docs/repositories.md @@ -231,6 +231,95 @@ Both modes preserve each snapshot's `username@hostname:path` identity and its times, so seeded history stays restorable by `Restore.source.identity` and by `fromPolicy`. +### Throttling a seed + +A seed is the single heaviest transfer Kopiur ever performs — a whole repository, +pulled across a link that other people are also using, usually on the worst day +you have had all year. Both modes can be capped, but they take different knobs +because they are different kopia operations. + +**Blob mode** copies at the storage layer, so it uses `sync-to`'s own speed +flags: `seed.sync.maxDownloadSpeedBytesPerSecond` and +`maxUploadSpeedBytesPerSecond`. + +**Migrate mode** has no speed flags at all — `kopia snapshot migrate` offers +none. The only lever is `kopia repository throttle set` on each *connection*, so +the cap is expressed **per side**, because a migrate seed is two repositories: + +```yaml +seed: + from: + repository: { kind: ClusterRepository, name: offsite } + migrate: + throttle: + source: # reading OUT of the replica (the off-site link) + downloadBytesPerSecond: 10485760 # 10 MiB/s + readOpsPerSecond: 100 + destination: # writing INTO this repository (on-LAN, be generous) + uploadBytesPerSecond: 209715200 # 200 MiB/s +``` + +Each side takes the same four knobs as a repository's +[`moverDefaults.throttle`](reference/crds/shared-types.md#moverdefaults): +`uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, +`writeOpsPerSecond`. Every knob you set must be at least `1` (a `0` is rejected at +admission — kopia's "no limit" is the *absent* knob, not a zero). + +**Two layers, merged field by field.** Each repository can already declare +`moverDefaults.throttle`, which every mover that touches it honors. +`seed.migrate.throttle` sits on top of that, per side: + +- `throttle.source` overrides the **replica's** defaults — the repository named by + `seed.from.repository`, not this one; +- `throttle.destination` overrides **this** repository's defaults; +- the merge is per knob: a knob you set here wins, a knob you leave unset keeps + the repository's value. Pinning one knob never drops the repository's others; +- omit `migrate.throttle` entirely and each side simply uses its own repository's + defaults. + +The two sides never mix — kopia's limits are **per connection**, and a migrate +seed opens both repositories under two separate kopia configurations. The +override applies only while the seed is **armed**: every later connect to the +now-initialized repository is capped by `moverDefaults.throttle` alone, so a +number chosen for a one-time copy does not go on constraining routine work. + +If any of the applications fails, the bootstrap **fails** rather than proceeding +uncapped: saturating exactly the link you asked to protect is worse than not +running. The mover logs `applied repository throttle` once per capped connection +— three times for a healthy first migrate seed (the replica, this repository's +seed-local connect, and the post-seed reconnect), and four when it is *resuming* +an interrupted attempt, whose opening probe connect finds the repository already +there and so gets capped too: + +```console +$ kubectl -n billing logs job/rebuilt-nas-bootstrap | grep 'applied repository throttle' +``` + +/// warning | A byte cap bites much harder than its number suggests + +Byte-rate caps apply to **cold** backend traffic only — content already in the +mover's kopia cache is served without touching the limiter, so a warm re-run can +look entirely unthrottled. And on small-object workloads the effective throughput +lands far below the nominal number: a measured 2 MB/s cap took ~14 s to move a +28 KiB cold repository, while caps of 10 MB/s and above often did not bind at all +at that size. Set these as a **ceiling for large transfers**, generously, and +measure against your own data. A seed already runs against a 24 h deadline; a cap +picked to "feel safe" is a good way to make that deadline the thing that fails. + +/// + +/// warning | New CRD fields need `kubectl apply`, not just `helm upgrade` + +`seed.migrate.throttle` is a new CRD field. Helm's `crds/` directory is +**install-only** — a `helm upgrade` never updates CRD schemas — so on a helm-CLI +upgrade an apiserver running the old schema **silently prunes** the field from +your manifest: the object admits cleanly, `kubectl get -o yaml` shows no +`throttle`, and the seed runs uncapped with nothing to see. Apply the CRDs first +(`kubectl apply --server-side -f deploy/crds/`), or use a GitOps flow with a +`CreateReplace` CRD policy. See [CRD lifecycle](install.md#crd-lifecycle). + +/// + ### `create` and `seed` together | You set | What happens | @@ -252,6 +341,7 @@ times, so seeded history stays restorable by `Restore.source.identity` and by | `seed.migrate.parallel` | kopia's `1` | Migrate mode only: snapshots migrated concurrently. | | `seed.migrate.latestOnly` | `false` | Copy only each identity's newest snapshot instead of its full history. | | `seed.migrate.policies` | `none` | Whether the source's **kopia-side** policies come along. Default is an explicit `--no-policies`, unlike kopia's own copy-by-default: retention here is driven by `Snapshot` CRs, and imported kopia policies could delete manifests behind the operator's back. | +| `seed.migrate.throttle.source` / `.destination` | each repository's `moverDefaults.throttle` | Migrate mode only: bandwidth/ops caps for the seed run, **per side** — `source` for the replica, `destination` for this repository. Each overrides that side's repository's defaults field by field; every set knob must be `>= 1`. Armed seeds only. See [Throttling a seed](#throttling-a-seed). | | `seed.allowEmptySource` | `false` | Accept a source holding zero snapshots. Left at `false`, an empty source **fails the bootstrap** and retries. | | `seed.failurePolicy.activeDeadlineSeconds` | `86400` (24 h) | Wall-clock cap for the seeding Job. | | `seed.failurePolicy.backoffLimit` | as `bootstrap` | Pod retries within one seeding Job. | diff --git a/docs/restores.md b/docs/restores.md index e22b6a0f..ea05b535 100644 --- a/docs/restores.md +++ b/docs/restores.md @@ -197,7 +197,9 @@ The empty-volume path applies to `fromPolicy`/`identity` sources on **every** ba `waitTimeout` (a Go-style duration, e.g. `5m`) opens a grace window during which "no matching snapshot yet" means *wait and re-check* instead of giving up. `onMissingSnapshot` applies only once the window closes. Use it when the Restore may be applied before the thing that produces its snapshot — a schedule about to fire, a GitOps apply ordering, a populator claim racing the first backup. -The window opens when the restore can first actually **proceed**, not when you applied it: Kopiur stamps that instant into `status.waitStartedAt` and measures `waitTimeout` from there. Two things have to be true — the restore's repository has reached `Ready`, once the controller can see it (the gate falls through unverified if the referenced `Repository`/`SnapshotPolicy` object doesn't exist yet), and (for `target.populator`) a PVC already claims the Restore via `dataSourceRef`. Until then the clock has not started. +The window opens when the restore can first actually **proceed**, not when you applied it: Kopiur stamps that instant into `status.waitStartedAt` and measures `waitTimeout` from there. Two things have to be true — the restore's repository has reached `Ready`, and (for `target.populator`) a PVC already claims the Restore via `dataSourceRef`. Until then the clock has not started. + +The window also does **not** open while the objects Kopiur reads the repository from are missing: if the referenced `Repository` object, or the `SnapshotPolicy` a `fromPolicy` source names, doesn't exist yet, the restore parks in `Pending` with `ReferentAvailable=False reason=RestoreReferentMissing` and `status.waitStartedAt` stays unstamped (it re-checks every 15 s, so it un-parks on its own once you apply the missing object). Two cases deliberately **do** open the window unverified: a `snapshotRef` whose `Snapshot` row doesn't exist yet — that row is precisely what the window is waiting for, and `onMissingSnapshot` has to be able to fire for a ref that never appears — and a restore whose mover Job has already launched, which keeps the deadline it was dispatched with. /// warning | The window is not measured from `metadata.creationTimestamp` diff --git a/docs/scenarios/dr-with-replicated-repository.md b/docs/scenarios/dr-with-replicated-repository.md index 56dc3b55..2e32ef19 100644 --- a/docs/scenarios/dr-with-replicated-repository.md +++ b/docs/scenarios/dr-with-replicated-repository.md @@ -234,7 +234,10 @@ and re-anchors the next cron slot, which is what you want mid-incident. whole time waiting for the repository — and its `waitTimeout` window opens at that moment, not at creation, so a long seed does not spend it (see [Restores → `waitTimeout`](../restores.md#waittimeout--wait-before-giving-up) - and `status.waitStartedAt`). + and `status.waitStartedAt`). That holds from the very first second of the + bring-up: while the `Repository` (or the `SnapshotPolicy` a `fromPolicy` + restore names) has not been applied yet, the `Restore` parks with + `RestoreReferentMissing` and the window is not open either. ## Observability diff --git a/docs/snapshot-replication.md b/docs/snapshot-replication.md index 6c0e5552..7369349d 100644 --- a/docs/snapshot-replication.md +++ b/docs/snapshot-replication.md @@ -53,6 +53,7 @@ $ kubectl -n billing get snapshots -l kopiur.home-operations.com/origin=replicat | `selection.latestOnly` | `true` = only each identity's most recent snapshot (cheap seed); default `false` = full history. | | `migrate.parallel` | Snapshots migrated concurrently (kopia default: 1, sequential) — the main knob for large first runs. | | `migrate.policies` | Whether kopia **policies** ride along: `none` (default — a Kopiur-managed destination keeps retention CR-driven), `copy`, or `copyOverwrite`. | +| `migrate.throttle.source` / `.destination` | Bandwidth/ops caps for this replication's runs, one block **per side** — see [Throttling a replication](#throttling-a-replication). Each overrides that side's repository's `moverDefaults.throttle` field by field. | | `pruning` | What happens to already-made copies on later runs: exactly one of `none` / `mirrorSource` / `retention` — see [Pruning](#pruning-the-copies). | | `mover` | Per-run mover overrides (resources, scheduling, security context). Inherits the source repository's `moverDefaults`. `inheritSecurityContextFrom` is rejected here, as for `RepositoryReplication` — there is no workload to inherit from. | | `credentialProjection` | Opt in to [credential projection](movers.md#let-kopiur-project-the-credentials-secret-recommended-for-shared-repos) for a `ClusterRepository` source/destination whose Secret lives elsewhere. | @@ -110,6 +111,90 @@ selection: Matching zero identities is a **successful no-op** (`NoIdentitiesMatched`), not an error — a fresh source simply has nothing to copy yet. Incomplete (interrupted) source snapshots are never copied. +## Throttling a replication + +A replication is often the heaviest thing Kopiur does to your network: it reads a +whole history out of one repository and writes it into another, frequently across +a WAN link that other people are also using. `spec.migrate.throttle` caps it — +**per side**, because a replication is two repositories. + +```yaml +migrate: + throttle: + source: # reading out of nas-primary (on-LAN, be generous) + downloadBytesPerSecond: 209715200 # 200 MiB/s + destination: # writing to the off-site bucket (the scarce link) + uploadBytesPerSecond: 10485760 # 10 MiB/s + writeOpsPerSecond: 100 +``` + +Each side takes the same four knobs as a repository's +[`moverDefaults.throttle`](reference/crds/shared-types.md#moverdefaults): +`uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, +`writeOpsPerSecond`. Every knob you set must be at least `1` (a `0` is rejected at +admission — kopia's "no limit" is the *absent* knob, not a zero). + +**Two layers, merged field by field.** Each repository can already declare +`moverDefaults.throttle`, which every mover that touches it honors. `migrate.throttle` +sits on top of that, per side: + +- `throttle.source` overrides the **source** repository's defaults; + `throttle.destination` overrides the **destination** repository's. +- The merge is per knob: a knob you set here wins, a knob you leave unset keeps + the repository's value. Pinning one knob never drops the repository's others. +- Omit `migrate.throttle` entirely and each side simply uses its own repository's + defaults. + +The two sides never mix. A cap on the source constrains only the read connection, +because kopia's limits are **per connection** and a replication opens two of them +under two separate kopia configurations. + +/// note | Why the cap isn't a flag on `snapshot migrate` + +`kopia snapshot migrate` has no speed options at all. The only lever kopia offers +is `kopia repository throttle set`, which writes the limits into a *connection's* +config — so Kopiur connects each side, applies that side's limits, and lets the +migrate (which reopens those configs) inherit them. That is also why each side +needs its own block: there is nothing repository-wide to inherit. Applying a cap +to the read-only source connection is fine — kopia accepts `throttle set` there — +so a `mode: ReadOnly` source is throttled like any other. + +If either application fails, the run **fails** rather than proceeding uncapped: +saturating exactly the link you asked to protect is worse than not running. The +mover logs `applied repository throttle` once per capped connection (twice for a +healthy replication), which is the quickest way to confirm the limits landed: + +```console +$ kubectl -n billing logs job/nas-primary-to-offsite-srepl-1765440000 | grep 'applied repository throttle' +``` + +/// + +/// warning | A byte cap bites much harder than its number suggests + +Byte-rate caps apply to **cold** backend traffic only — content already in the +mover's kopia cache is served without touching the limiter, so a warm re-run can +look entirely unthrottled. And on small-object workloads the effective throughput +lands far below the nominal number: a measured 2 MB/s cap took ~14 s to move a +28 KiB cold repository, while caps of 10 MB/s and above often did not bind at all +at that size. Set these as a **ceiling for large transfers**, generously, and +measure against your own data — a number picked to "feel safe" can turn a nightly +replication into one that never finishes. + +/// + +/// warning | New CRD fields need `kubectl apply`, not just `helm upgrade` + +`migrate.throttle` is a new CRD field. Helm's `crds/` directory is **install-only** +— a `helm upgrade` never updates CRD schemas — so on a helm-CLI upgrade an +apiserver running the old schema **silently prunes** the field from your manifest: +the object admits cleanly, `kubectl get -o yaml` shows no `throttle`, and the runs +stay uncapped with nothing to see. Apply the CRDs first (`kubectl apply +--server-side -f deploy/crds/`), or use a GitOps flow with a `CreateReplace` CRD +policy. See [CRD lifecycle](install.md#crd-lifecycle). + +/// + ## Pruning the copies `pruning` is exactly one of three modes; **absent means `none`**. Whatever the mode, pruning only ever considers snapshots **this replication created** (the copy CRs it labels) — never the destination's own directly-written snapshots, and never another replication's copies. @@ -137,7 +222,7 @@ Matching zero identities is a **successful no-op** (`NoIdentitiesMatched`), not - **A dedicated mover ServiceAccount.** Replication movers create/patch/delete `Snapshot` CRs (the copies), which the ordinary backup mover must never be able to do — so they run as the dedicated `kopiur-snapshot-replication-mover` ServiceAccount with its own narrowly-scoped Role, generated alongside the rest of the [RBAC](rbac.md). - **No destination maintenance behind your back.** The mover always disables kopia's auto-maintenance; the destination's own [`Maintenance`](maintenance.md) remains the only compaction that runs there. - **Every run is counted.** `kopiur_replication_runs_total{kind,trigger,outcome}` records each finished run, so "the nightly copy has been failing" is alertable without watching conditions. `trigger` separates `cron` from the [requested](#run-it-now) runs. -- **Size the first run.** A full-history first replication of a large repository moves everything once (idempotent thereafter). Raise `migrate.parallel`, consider `latestOnly: true` for seeding, and note the mover Job's deadline can be tuned via `mover`/`failurePolicy` knobs on big estates. +- **Size the first run.** A full-history first replication of a large repository moves everything once (idempotent thereafter). Raise `migrate.parallel`, consider `latestOnly: true` for seeding, and note the mover Job's deadline can be tuned via `mover`/`failurePolicy` knobs on big estates. If that first run is what worries the network, cap it with [`migrate.throttle`](#throttling-a-replication) rather than by shrinking the selection — but leave headroom, or the deadline becomes the thing that fails. ## See also diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index ea861e65..94915d11 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -435,6 +435,7 @@ $ kubectl describe restore -n | PVC stuck `Pending` (populator) | Volume-populator handshake not completing. | Need Kubernetes ≥ 1.24; install `volume-data-source-validator` to see the real event. See [Restores → deploy-or-restore](restores.md#deploy-or-restore-gitops). | | `prime-*` PVCs piling up, `Bound`, mounted by nothing | ≤ 0.7.x: a populator `Restore` re-created over an **already-bound** claim restored into a prime PVC that could never be adopted, and leaked it — one full copy of the data per re-created `Restore`. | Upgrade: Kopiur now completes that case as a no-op (`Ready=True reason=TargetAlreadyBound`) and reaps the orphans (`OrphanedPrimePvcReaped` event). List them with `kubectl get pvc -A -l kopiur.home-operations.com/op=restore-populate`; any whose claiming PVC is gone are collected with their `Restore`. | | Populator `Restore` says `TargetAlreadyBound` and restores nothing | Working as intended: its claiming PVC is already **Bound**, and a volume populator can only fill an **unbound** claim. | To really restore into it, delete the **PVC** (keep its `dataSourceRef`) and let it be re-created. Deleting the `Restore` just re-triggers the no-op. | +| Stuck `Pending`, `ReferentAvailable=False reason=RestoreReferentMissing` | The object Kopiur reads the restore's repository from doesn't exist: the referenced `Repository`/`ClusterRepository`, or the `SnapshotPolicy` a `fromPolicy` source names. Kopiur can't verify the backend, so it parks and — deliberately — does **not** start the `waitTimeout` window (`status.waitStartedAt` stays unset). | Create the object named in the condition message (or repoint the `Restore`). The park re-checks every 15 s and clears itself. A `snapshotRef` whose `Snapshot` row is missing is a *different* case and does not park — it waits out `waitTimeout` and then applies `onMissingSnapshot`. | | `identity` source rejected at admission | `source.identity` requires an explicit `spec.repository`. | Add `spec.repository`. | | Stuck `Pending`, `MoverPermitted=False` | The restore mover requests an elevated context (root / `privilegedMode`, container- or pod-level, possibly inherited). | Annotate the namespace (above), or drop the elevation from `spec.mover`. | | Restored files **owned by `65532`** / unreadable by the app | The mover wrote them as its own UID (no `mover.securityContext`). | Set `Restore.spec.mover.securityContext.runAsUser/Group` to the app's UID (or `inheritSecurityContextFrom`). |