From 886d775cc25a5eae1ec314cc6155747677ab6e7e Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 09:53:57 -0700 Subject: [PATCH 01/15] fix(replication): clear stale manualRun.completedAt with an explicit null (#394) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `completedAt` carried `skip_serializing_if = "Option::is_none"`, so a non-terminal manualRun patch simply omitted the key. An RFC-7386 merge patch never removes what it omits, so the PREVIOUS run's completion stamp stayed visible under a fresh `Pending`/`Running`. In the two replication controllers this is worse than a stale display: they build the noop guard's `current` by re-serializing the same typed status, so `desired` (key absent) never matched `current` (stale value) and every queued pass re-fired a PATCH the apiserver no-op'd — a guard that cannot converge. Maintenance's `patch_manual_run` is unguarded, so there it was display only. Fix is serde-level in the two shared status types: drop the `skip_serializing_if`, keep `#[serde(default)]`. A non-terminal phase now emits `"completedAt": null` and the merge patch DELETES the stale key. No controller changes — every non-terminal site already sets `None`. Zero CRD drift: the rationale is a plain `//` comment, not rustdoc, because doc comments become the CRD `description` and serde mechanics are not the user's business. --- Cargo.lock | 1 + crates/api/src/common/mod.rs | 19 +++++- crates/api/src/common/tests.rs | 28 ++++++++- crates/api/src/maintenance.rs | 73 ++++++++++++++++++++++- crates/controller/Cargo.toml | 3 + crates/controller/src/io/tests.rs | 74 ++++++++++++++++++++++++ crates/controller/src/replication_run.rs | 15 +++-- crates/e2e/tests/replication.rs | 55 ++++++++++++++++++ 8 files changed, 259 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ebfb926..22f109a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1755,6 +1755,7 @@ dependencies = [ "futures", "http", "http-body-util", + "json-patch", "k8s-openapi", "kopiur-api", "kopiur-kopia", diff --git a/crates/api/src/common/mod.rs b/crates/api/src/common/mod.rs index e0fd48da..64fffdcf 100644 --- a/crates/api/src/common/mod.rs +++ b/crates/api/src/common/mod.rs @@ -1273,7 +1273,24 @@ pub struct ReplicationManualRunStatus { #[serde(default, skip_serializing_if = "Option::is_none")] pub phase: Option, /// RFC3339 instant the run reached a terminal phase. - #[serde(default, skip_serializing_if = "Option::is_none")] + // Deliberately serialized EVEN WHEN `None` — no `skip_serializing_if`, unlike + // every other optional in this struct. A non-terminal phase must emit + // `"completedAt": null` so the RFC-7386 merge-patch the controller sends + // DELETES the previous run's stamp. Omitting the key instead leaves the old + // instant standing, which is both a lie in `kubectl get -o yaml` and — + // because the replication controllers build the noop guard's `current` by + // re-serializing this very struct — a guard that never converges: `desired` + // would omit the key while `current` carries the stale value, so every + // queued pass re-fires a PATCH the apiserver no-ops (#394). + // + // This depends on `patch_status` sending `kube::api::Patch::Merge` + // (`crates/controller/src/io/apply.rs`). Under `Patch::Apply` an explicit + // null does NOT delete, and this contract silently breaks. + // + // Kept a plain comment rather than rustdoc on purpose: doc comments become + // the CRD `description` (`kubectl explain`, docs/field-reference.md), and + // serde mechanics are not the user's business. + #[serde(default)] pub completed_at: Option, } diff --git a/crates/api/src/common/tests.rs b/crates/api/src/common/tests.rs index 10c3dc34..80b24945 100644 --- a/crates/api/src/common/tests.rs +++ b/crates/api/src/common/tests.rs @@ -1523,7 +1523,31 @@ fn replication_manual_run_status_roundtrips_camel_case() { assert_eq!(json["completedAt"], "2026-06-11T12:01:42Z"); let back: ReplicationManualRunStatus = serde_json::from_value(json).unwrap(); assert_eq!(back, st); - // Absent fields serialize away entirely (no explicit nulls in status). + // The contract (#394): `requestedAt`/`phase` serialize away when absent, but + // `completedAt` ALWAYS emits — as an explicit null when there is no + // completion instant — so the merge-patch DELETES a previous run's stamp + // instead of leaving it standing over a fresh non-terminal phase. let empty = serde_json::to_value(ReplicationManualRunStatus::default()).unwrap(); - assert_eq!(empty, serde_json::json!({})); + assert_eq!(empty, serde_json::json!({ "completedAt": null })); + // …and that explicit null reads back as "no completion instant", not as a + // decode error (the round trip both directions). + let back: ReplicationManualRunStatus = serde_json::from_value(empty).unwrap(); + assert_eq!(back, ReplicationManualRunStatus::default()); + assert!(back.completed_at.is_none()); + + // A non-terminal run keeps its other fields AND clears the stamp. + let running = serde_json::to_value(ReplicationManualRunStatus { + requested_at: Some("2026-06-11T13:00:00Z".into()), + phase: Some(ReplicationManualRunPhase::Running), + completed_at: None, + }) + .unwrap(); + assert_eq!( + running, + serde_json::json!({ + "requestedAt": "2026-06-11T13:00:00Z", + "phase": "Running", + "completedAt": null, + }) + ); } diff --git a/crates/api/src/maintenance.rs b/crates/api/src/maintenance.rs index d0e3f5c2..30ec7ca8 100644 --- a/crates/api/src/maintenance.rs +++ b/crates/api/src/maintenance.rs @@ -609,7 +609,21 @@ pub struct ManualRunStatus { #[serde(default, skip_serializing_if = "Option::is_none")] pub phase: Option, /// RFC3339 instant the run reached a terminal phase. - #[serde(default, skip_serializing_if = "Option::is_none")] + // Deliberately serialized EVEN WHEN `None` (no `skip_serializing_if`), for + // the same reason as `common::ReplicationManualRunStatus::completed_at`: a + // non-terminal phase emits `"completedAt": null` so the RFC-7386 merge-patch + // DELETES the previous run's stamp instead of leaving it standing over a + // fresh `Running` (#394). Maintenance patches `manualRun` unconditionally + // (no noop guard), so here the stake is a truthful status rather than a + // non-converging write loop. + // + // This depends on `patch_status` sending `kube::api::Patch::Merge` + // (`crates/controller/src/io/apply.rs`). Under `Patch::Apply` an explicit + // null does NOT delete, and this contract silently breaks. + // + // Kept a plain comment rather than rustdoc on purpose: doc comments become + // the CRD `description` (`kubectl explain`, docs/field-reference.md). + #[serde(default)] pub completed_at: Option, } @@ -677,6 +691,63 @@ mod tests { } } + #[test] + fn manual_run_status_roundtrips_camel_case_and_nulls_a_missing_completion() { + // Parsed the cluster's way (YAML -> serde_json::Value -> typed), which + // is the only path that proves the camelCase wire names land. + let status: MaintenanceStatus = from_yaml( + "manualRun:\n requestedAt: 2026-06-11T12:00:00Z\n mode: full\n phase: Succeeded\n completedAt: 2026-06-11T12:01:42Z\n", + ); + let manual = status.manual_run.expect("manualRun decodes"); + assert_eq!(manual.requested_at.as_deref(), Some("2026-06-11T12:00:00Z")); + assert_eq!(manual.mode, Some(ManualRunMode::Full)); + assert_eq!(manual.phase, Some(ManualRunPhase::Succeeded)); + assert_eq!(manual.completed_at.as_deref(), Some("2026-06-11T12:01:42Z")); + let json = serde_json::to_value(&manual).unwrap(); + assert_eq!(json["requestedAt"], "2026-06-11T12:00:00Z"); + assert_eq!(json["mode"], "full"); + assert_eq!(json["phase"], "Succeeded"); + assert_eq!(json["completedAt"], "2026-06-11T12:01:42Z"); + + // #394: a non-terminal run emits an EXPLICIT null completedAt, so the + // RFC-7386 merge-patch deletes the previous run's stamp rather than + // leaving it standing over a fresh `Running`. + let running = serde_json::to_value(ManualRunStatus { + requested_at: Some("2026-06-11T13:00:00Z".into()), + mode: Some(ManualRunMode::Quick), + phase: Some(ManualRunPhase::Running), + completed_at: None, + }) + .unwrap(); + assert_eq!( + running, + serde_json::json!({ + "requestedAt": "2026-06-11T13:00:00Z", + "mode": "quick", + "phase": "Running", + "completedAt": null, + }) + ); + // The explicit null reads back as "no completion instant". + let back: ManualRunStatus = serde_json::from_value(running).unwrap(); + assert!(back.completed_at.is_none()); + assert_eq!( + serde_json::to_value(ManualRunStatus::default()).unwrap(), + serde_json::json!({ "completedAt": null }) + ); + + // A status that never requested a run still has NO manualRun key — + // the parent field keeps its own skip_serializing_if. + let never: MaintenanceStatus = from_yaml("observedGeneration: 3\n"); + assert!(never.manual_run.is_none()); + assert!( + serde_json::to_value(&never) + .unwrap() + .get("manualRun") + .is_none() + ); + } + #[test] fn lease_identity_is_hostname_safe_and_stable() { let (user, host) = kopia_lease_identity("kopiur/media/My_App.x"); diff --git a/crates/controller/Cargo.toml b/crates/controller/Cargo.toml index 41ad9f50..bfdbeec2 100644 --- a/crates/controller/Cargo.toml +++ b/crates/controller/Cargo.toml @@ -51,6 +51,9 @@ rand = "0.10" [dev-dependencies] serial_test = { workspace = true } +# Apply RFC-7386 merge patches the way the apiserver does, so the status-patch +# convergence tests can replay a reconcile loop instead of asserting one pass. +json-patch = { workspace = true } # Collect mock request bodies in the leader-election protocol tests. http-body-util = "0.1" # `test-util` enables paused tokio time (`start_paused`) so the renewal-loss diff --git a/crates/controller/src/io/tests.rs b/crates/controller/src/io/tests.rs index 26601d57..1e1552ef 100644 --- a/crates/controller/src/io/tests.rs +++ b/crates/controller/src/io/tests.rs @@ -820,6 +820,80 @@ fn status_patch_noop_ignores_volatile_message_only_when_message_matches() { assert!(status_patch_is_noop(Some(¤t), &desired)); } +/// The stored status a replication has after a run that finished: the previous +/// request, terminal, with its completion stamp. +fn finished_manual_run() -> serde_json::Value { + serde_json::json!({ + "manualRun": { + "requestedAt": "2026-06-11T12:00:00Z", + "phase": "Succeeded", + "completedAt": "2026-06-11T12:01:42Z", + }, + }) +} + +/// The `desired` payload `patch_manual_run` builds for a FOLLOW-UP request that +/// has not started — built from the typed struct exactly as the controllers do. +fn pending_manual_run_patch() -> serde_json::Value { + let manual = kopiur_api::common::ReplicationManualRunStatus { + requested_at: Some("2026-06-11T13:00:00Z".into()), + phase: Some(kopiur_api::common::ReplicationManualRunPhase::Pending), + completed_at: None, + }; + serde_json::json!({ "manualRun": manual }) +} + +#[test] +fn non_terminal_manual_run_patch_carries_an_explicit_null_completed_at() { + // #394, the fix itself at the layer that consumes it: `Value::Index` maps a + // MISSING key to `Null`, so this has to ask for the key with `.get()` — + // present-and-null is the whole point, and absent is the bug. + let desired = pending_manual_run_patch(); + assert_eq!( + desired["manualRun"].get("completedAt"), + Some(&serde_json::Value::Null), + "a non-terminal manualRun must NAME completedAt so the merge-patch \ + deletes the previous run's stamp; got {desired}" + ); +} + +#[test] +fn manual_run_patch_converges_after_clearing_a_stale_completed_at() { + // The #394 trap is a LOOP, not one pass, so this replays one: patch, let the + // apiserver apply RFC-7386, rebuild `current` the way the replication + // controllers do (re-serialize the TYPED status off the refreshed object), + // and demand the second pass write nothing. + // + // Without the explicit null, `desired` simply omits `completedAt`; a merge + // patch never removes what it omits, so the stale stamp SURVIVES, the + // rebuilt `current` still differs from `desired`, and every queued pass + // re-fires a PATCH the apiserver no-ops — forever. + let desired = pending_manual_run_patch(); + let mut stored = finished_manual_run(); + assert!( + !status_patch_is_noop(Some(&stored), &desired), + "answering a NEW request must write" + ); + + // The apiserver applies the merge patch (this is `Patch::Merge` — an + // explicit null DELETES the key). + json_patch::merge(&mut stored, &desired); + assert!( + stored["manualRun"].get("completedAt").is_none(), + "the merge-patch must have deleted the stale stamp; got {stored}" + ); + + // Next reconcile: `current` is the typed status of the refreshed object, + // re-serialized — the exact round trip `patch_manual_run` performs. + let typed: kopiur_api::common::ReplicationManualRunStatus = + serde_json::from_value(stored["manualRun"].clone()).expect("stored manualRun decodes"); + let current = serde_json::json!({ "manualRun": typed }); + assert!( + status_patch_is_noop(Some(¤t), &desired), + "the guard must CONVERGE once the stamp is cleared; current {current}, desired {desired}" + ); +} + #[test] fn terminal_gate_only_on_failed_at_current_generation() { use kopiur_api::RepositoryPhase; diff --git a/crates/controller/src/replication_run.rs b/crates/controller/src/replication_run.rs index 3296e070..8f80dc92 100644 --- a/crates/controller/src/replication_run.rs +++ b/crates/controller/src/replication_run.rs @@ -620,11 +620,16 @@ mod tests { ReplicationManualRunPhase::Pending, ReplicationManualRunPhase::Running, ] { - assert!( - manual_run_status(&req, phase.clone(), now) - .completed_at - .is_none(), - "{phase:?} has not completed" + let st = manual_run_status(&req, phase.clone(), now); + assert!(st.completed_at.is_none(), "{phase:?} has not completed"); + // #394: and it must SERIALIZE as an explicit null, not vanish — the + // merge-patch only deletes a previous run's stamp when the key is + // present with a null value. + let json = serde_json::to_value(&st).unwrap(); + assert_eq!( + json.get("completedAt"), + Some(&serde_json::Value::Null), + "{phase:?} must clear a stale completedAt; got {json}" ); } } diff --git a/crates/e2e/tests/replication.rs b/crates/e2e/tests/replication.rs index 38a082fd..82b87fad 100644 --- a/crates/e2e/tests/replication.rs +++ b/crates/e2e/tests/replication.rs @@ -381,6 +381,10 @@ async fn repository_replication_s3_to_s3_uses_destination_scoped_credentials() { /// It pins the halves a unit test cannot reach: the requested run rides the /// ordinary mover path (a real `sync-to`, tagged `run-trigger: manual` on its /// Job), and `status.manualRun` answers the exact timestamp requested. +/// +/// It then pins issue #394 against a REAL apiserver: a follow-up request parked +/// as `Pending` must have the finished run's `completedAt` merge-patched away, +/// which only the explicit-null serialization achieves. #[tokio::test] #[ignore = "requires the e2e harness (mise run //crates/e2e:test): kind + built images + helm install"] async fn repository_replication_runs_on_demand_from_the_run_requested_annotation() { @@ -533,5 +537,56 @@ async fn repository_replication_runs_on_demand_from_the_run_requested_annotation "the requested run's Job must be tagged run-trigger: manual; got {triggers:?}" ); + // Issue #394: a FOLLOW-UP request must not inherit the finished run's + // `completedAt`. The non-terminal patch now serializes an explicit null, and + // an RFC-7386 merge-patch DELETES the key it is nulling — so the stale stamp + // disappears from the stored object rather than sitting under a `Pending`. + // + // Suspending is what makes the park deterministic (#380 records an + // unanswered request as `Pending` and stops). Ordering hazard: the + // reconciler checks `spec.suspend` BEFORE the run request, but annotating an + // un-suspended CR can spawn the Job and record `Running` before the suspend + // lands — so both fields ride ONE patch. + let second_request = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + repls + .patch( + name, + &PatchParams::default(), + &Patch::Merge(serde_json::json!({ + "metadata": { "annotations": { + kopiur_api::consts::RUN_REQUESTED_ANNOTATION: second_request + } }, + "spec": { "suspend": true } + })), + ) + .await + .expect("suspend and re-request in one patch"); + + let parked = wait_until( + "the follow-up request parks as Pending on the suspended replication", + default_timeout(), + poll_interval(), + || { + let repls = repls.clone(); + let want = second_request.clone(); + async move { + let s = status_json(&repls, name).await; + Ok(s.get("manualRun") + .filter(|m| { + m.get("phase").and_then(|p| p.as_str()) == Some("Pending") + && m.get("requestedAt").and_then(|r| r.as_str()) == Some(want.as_str()) + }) + .cloned()) + } + }, + ) + .await + .expect("the suspended replication should record the new request as Pending"); + assert!( + parked.get("completedAt").is_none(), + "a non-terminal manualRun must carry NO completedAt — the previous run's \ + stamp must have been merge-patched away, not left standing; got {parked}" + ); + let _ = repls.delete(name, &DeleteParams::default()).await; } From 0b22424bc1dc60fa0949b2075da1eac4f71a473d Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 10:22:45 -0700 Subject: [PATCH 02/15] fix(restore): park the readiness gate when the repository referent is missing (#393) --- crates/api/src/consts.rs | 30 ++ crates/api/src/gates.rs | 76 ++++ crates/api/src/restore.rs | 18 +- crates/controller/src/consts.rs | 10 +- crates/controller/src/io/tests.rs | 11 + crates/controller/src/restore/mod.rs | 288 ++++++++++++--- crates/controller/src/restore/plan.rs | 151 ++++++++ crates/controller/src/restore/tests.rs | 151 ++++++++ crates/e2e/mise.toml | 2 +- crates/e2e/src/consts.rs | 5 + crates/e2e/tests/restore.rs | 336 ++++++++++++++++++ deploy/crds/all-crds.yaml | 18 +- deploy/crds/restores.yaml | 18 +- deploy/helm/kopiur/crds/restores.yaml | 18 +- docs/field-reference.md | 2 +- docs/gitops.md | 5 +- docs/reference/crds/restore.md | 2 +- docs/restores.md | 4 +- .../dr-with-replicated-repository.md | 5 +- docs/troubleshooting.md | 1 + 20 files changed, 1068 insertions(+), 83 deletions(-) diff --git a/crates/api/src/consts.rs b/crates/api/src/consts.rs index 31dd8a45..ead7794d 100644 --- a/crates/api/src/consts.rs +++ b/crates/api/src/consts.rs @@ -386,6 +386,36 @@ pub const SOURCE_PVC_AVAILABLE_CONDITION: &str = "SourcePvcAvailable"; /// `reason`/Event reason for [`SOURCE_PVC_AVAILABLE_CONDITION`] = `False`. pub const SOURCE_PVC_MISSING_REASON: &str = "SourcePvcMissing"; +/// `Restore` condition recording whether the object this restore's repository +/// is DERIVED from exists. Set `False` (with +/// [`RESTORE_REFERENT_MISSING_REASON`]) when the readiness gate cannot even +/// look up the repository because a referent is absent: the explicit +/// `spec.repository` object itself, or the `source.fromPolicy` `SnapshotPolicy` +/// the repository ref is read from (issue #393). +/// +/// Its own condition rather than the `Ready` one: the registry's coarse +/// [`crate::gates::StructuralGate::trips`] filter keys on condition+polarity, +/// so registering a `Ready=False` row would make every unrelated `Ready=False` +/// reason on a `Snapshot`/`Restore` read as an unknown gate from a newer +/// operator. Only this gate ever writes this condition, so it cannot misfire. +/// +/// Structural ([`crate::gates::RESTORE_REFERENT_MISSING_GATE`]): a referent +/// that is never created never self-heals, and while it is missing the restore +/// sits at `phase: Pending` with its `policy.waitTimeout` window deliberately +/// NOT started — the phase-invisible park shape of #359. +pub const RESTORE_REFERENT_AVAILABLE_CONDITION: &str = "ReferentAvailable"; +/// `reason`/Event reason for [`RESTORE_REFERENT_AVAILABLE_CONDITION`] = +/// `False`. Distinct from [`REPOSITORY_NOT_READY_REASON`] on purpose: that one +/// means "the repository object exists and its backend is unreachable", which +/// would be a lie for a `SnapshotPolicy` that was never applied. +pub const RESTORE_REFERENT_MISSING_REASON: &str = "RestoreReferentMissing"; +/// `reason` for [`RESTORE_REFERENT_AVAILABLE_CONDITION`] = `True`: the referent +/// appeared on a later pass (the clear-side counterpart of +/// [`RESTORE_REFERENT_MISSING_REASON`]; controller-internal remediation copy, +/// like the `Snapshot` gate's `SourcePvcFound`). Written ONLY when the +/// condition already exists — the healthy wire never grows the condition. +pub const RESTORE_REFERENT_FOUND_REASON: &str = "RestoreReferentFound"; + /// `Snapshot` condition: this deletion is HELD by the mass-deletion breaker /// (`Repository`/`ClusterRepository` `spec.deletionProtection.threshold`) /// until acknowledged via [`ALLOW_MASS_DELETION_ANNOTATION`] on the diff --git a/crates/api/src/gates.rs b/crates/api/src/gates.rs index a479f0c0..a79f1908 100644 --- a/crates/api/src/gates.rs +++ b/crates/api/src/gates.rs @@ -528,6 +528,33 @@ pub const SEED_MOVER_TOO_OLD_GATE: StructuralGate = StructuralGate { severity: GateSeverity::Warn, }; +/// A `Restore` whose repository referent does not exist (issue #393): the +/// explicit `spec.repository` object, or the `source.fromPolicy` +/// `SnapshotPolicy` the repository ref is derived from. +/// +/// The readiness gate cannot verify a repository it cannot even look up, so the +/// restore parks at `phase: Pending` and — critically — its +/// `policy.waitTimeout` window is NOT opened (`status.waitStartedAt` stays +/// unstamped) until the referent appears and its repository becomes `Ready`. +/// Before #393 the gate fell through unverified here, so a slow-arriving +/// referent silently spent the window; for a `fromPolicy` source, whose +/// `onMissingSnapshot` defaults to `Continue`, a spent window means an EMPTY +/// volume. +/// +/// WARN, not Fail, for the reason [`POLICY_REPOSITORY_NOT_READY_GATE`] is: +/// referents applied moments apart by GitOps are the ordinary case and resolve +/// themselves within a requeue or two, so this must explain the park rather +/// than independently turn a diagnostic red. What it must never be is INVISIBLE +/// — a restore parked on a `SnapshotPolicy` that was never applied looks +/// identical, by phase, to one that is simply still resolving. +pub const RESTORE_REFERENT_MISSING_GATE: StructuralGate = StructuralGate { + applies_to: GateScope::SnapshotOrRestore, + condition: consts::RESTORE_REFERENT_AVAILABLE_CONDITION, + blocked_status: CONDITION_FALSE, + reason: consts::RESTORE_REFERENT_MISSING_REASON, + severity: GateSeverity::Warn, +}; + /// Every human-actionable structural gate kopiur's reconcilers can park an /// object on. /// @@ -559,6 +586,7 @@ pub const STRUCTURAL_GATES: &[StructuralGate] = &[ SCHEDULE_FANOUT_CAPPED_GATE, POLICY_REPOSITORY_NOT_READY_GATE, SOURCE_PVC_MISSING_GATE, + RESTORE_REFERENT_MISSING_GATE, SEED_SOURCE_NOT_READY_GATE, SEED_SOURCE_AUTH_CONFLICT_GATE, SEEDING_GATE, @@ -893,6 +921,13 @@ mod tests { GateScope::Snapshot, GateSeverity::Fail, ), + ( + consts::RESTORE_REFERENT_AVAILABLE_CONDITION, + CONDITION_FALSE, + consts::RESTORE_REFERENT_MISSING_REASON, + GateScope::SnapshotOrRestore, + GateSeverity::Warn, + ), ( consts::SEEDED_CONDITION, CONDITION_FALSE, @@ -967,6 +1002,47 @@ mod tests { } } + /// The #393 park must be visible to `doctor` on a `Restore` — and must not + /// make any OTHER condition read as a gate. + /// + /// The registry's coarse [`StructuralGate::trips`] filter is + /// reason-agnostic, so a row registered on a condition other reconcilers + /// also write (`Ready`, `Resolved`, …) would make every unrelated reason on + /// that condition report as "a gate from a newer operator". This row's + /// condition is therefore written by exactly one gate and one reason. + #[test] + fn the_restore_referent_gate_is_restore_scoped_and_owns_its_condition() { + let row = STRUCTURAL_GATES + .iter() + .find(|g| g.reason == consts::RESTORE_REFERENT_MISSING_REASON) + .expect("the #393 referent-missing park must be registered"); + // Reachable from a Restore (the only scope whose `covers_restore` is true). + assert!(row.applies_to.covers_restore()); + assert_eq!(row.applies_to, GateScope::SnapshotOrRestore); + // Warn: referents applied moments apart by GitOps resolve themselves. + assert_eq!(row.severity, GateSeverity::Warn); + assert_eq!(row.condition, consts::RESTORE_REFERENT_AVAILABLE_CONDITION); + assert!(row.trips(consts::RESTORE_REFERENT_AVAILABLE_CONDITION, "False")); + assert!(!row.trips(consts::RESTORE_REFERENT_AVAILABLE_CONDITION, "True")); + // Exactly one row owns this condition, so the coarse filter can only + // fire for the one reason this build writes on it. + assert_eq!( + STRUCTURAL_GATES + .iter() + .filter(|g| g.condition == consts::RESTORE_REFERENT_AVAILABLE_CONDITION) + .count(), + 1 + ); + // ...and it is NOT the `Ready` condition, which every Restore writes with + // many reasons — registering there would misreport all of them. + assert_ne!(row.condition, consts::READY_CONDITION); + assert!( + !STRUCTURAL_GATES + .iter() + .any(|g| g.condition == consts::READY_CONDITION) + ); + } + #[test] fn every_scope_classifier_is_consistent() { // Each scope covers exactly one of the four kind families a consumer diff --git a/crates/api/src/restore.rs b/crates/api/src/restore.rs index 2fc73666..0dd3814f 100644 --- a/crates/api/src/restore.rs +++ b/crates/api/src/restore.rs @@ -426,13 +426,17 @@ pub struct RestoreStatus { pub timing: Option, /// When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which /// the restore could actually proceed (its repository reached `Ready`, and for a - /// `target.populator` a PVC already claims it), NOT when the Restore was created. If the - /// referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate - /// falls through unverified and the anchor can be stamped as early as creation. Stamped - /// once and then honored verbatim, so the window survives controller restarts and Job pod - /// retries; cleared when a populator re-opens resolution for a re-created claim, so that - /// claim gets the full window again. Absent means the window has not opened yet (or no - /// `policy.waitTimeout` is configured, in which case there is no window to anchor). + /// `target.populator` a PVC already claims it), NOT when the Restore was created. The + /// window does NOT open while the referenced `Repository` object or `fromPolicy` + /// `SnapshotPolicy` doesn't exist: the restore parks in `Pending` + /// (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It + /// DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so + /// `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose + /// mover Job already launched. Stamped once and then honored verbatim, so the window + /// survives controller restarts and Job pod retries; cleared when a populator re-opens + /// resolution for a re-created claim, so that claim gets the full window again. Absent + /// means the window has not opened yet (or no `policy.waitTimeout` is configured, in + /// which case there is no window to anchor). #[serde(default, skip_serializing_if = "Option::is_none")] pub wait_started_at: Option, /// Bytes/files restored so far, patched periodically by the mover. diff --git a/crates/controller/src/consts.rs b/crates/controller/src/consts.rs index df3a16b2..31eaf5d7 100644 --- a/crates/controller/src/consts.rs +++ b/crates/controller/src/consts.rs @@ -25,7 +25,8 @@ pub use kopiur_api::consts::{ OP_RESTORE_TARGET, ORIGIN_LABEL, PRIVILEGED_MOVER_NOT_PERMITTED_REASON, PRIVILEGED_MOVERS_ANNOTATION, READY_CONDITION, RECONCILING_CONDITION, REPOSITORIES_READY_CONDITION, REPOSITORY_NOT_READY_REASON, REPOSITORY_READ_ONLY_REASON, - REPOSITORY_UID_LABEL, REPOSITORY_WRITABLE_CONDITION, RUN_MODE_ANNOTATION, + REPOSITORY_UID_LABEL, REPOSITORY_WRITABLE_CONDITION, RESTORE_REFERENT_AVAILABLE_CONDITION, + RESTORE_REFERENT_FOUND_REASON, RESTORE_REFERENT_MISSING_REASON, RUN_MODE_ANNOTATION, RUN_REQUESTED_ANNOTATION, SCHEDULE_FANOUT_CAPPED_CONDITION, SCHEDULE_LABEL, SCHEDULE_RUNNABLE_CONDITION, SKIP_SNAPSHOT_CLEANUP_ANNOTATION, SNAPSHOT_CLEANUP_FINALIZER, SNAPSHOT_ID_LABEL, SOURCE_PVC_AVAILABLE_CONDITION, SOURCE_PVC_MISSING_REASON, @@ -287,6 +288,13 @@ pub const ORPHANED_PRIME_REAPED_REASON: &str = "OrphanedPrimePvcReaped"; /// `action` for the already-bound no-op / orphan-reap Events: to actually restore into the /// claim, delete it and let it be recreated (keeping its `dataSourceRef`). pub const RECREATE_CLAIM_TO_RESTORE_ACTION: &str = "RecreateClaimToRestore"; +/// Event `action` (remediation hint) for +/// [`RESTORE_REFERENT_MISSING_REASON`]: create the referenced object, or repoint +/// the `Restore` at one that exists. Published on the park TRANSITION only (the +/// park itself re-patches an identical status every 15s, which is a server-side +/// no-op) — it is what replaces the error metric + Warning Event the pre-#393 +/// unverified fall-through produced downstream. +pub const CREATE_RESTORE_REFERENT_ACTION: &str = "CreateRestoreReferentOrFixTheReference"; /// `Snapshot` condition tracking kopia-side pin reconciliation (ADR-0005 §13(c)). /// `True`/`False` mirrors `status.pinned` once a SnapshotPin mover Job ran; diff --git a/crates/controller/src/io/tests.rs b/crates/controller/src/io/tests.rs index 1e1552ef..ca98c93a 100644 --- a/crates/controller/src/io/tests.rs +++ b/crates/controller/src/io/tests.rs @@ -3534,6 +3534,17 @@ const GATE_WRITERS: &[(&str, bool, &str, &str)] = &[ crate::consts::SOURCE_PVC_MISSING_REASON, "snapshot::handle_missing_source_pvc (computed polarity)", ), + // `restore::park_on_missing_referent` via + // io::upsert_gate(&RESTORE_REFERENT_MISSING_GATE, ...) — the tri-state + // readiness gate's `Undetermined` arm (#393); cleared (True, reason + // `RestoreReferentFound`) by `restore::proceed_past_gate` / + // `plan::cleared_referent_conditions` once the referent exists. + ( + crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION, + false, + crate::consts::RESTORE_REFERENT_MISSING_REASON, + "restore::park_on_missing_referent", + ), // `repository::park_on_seed_source` + // `cluster_repository::park_cluster_on_seed_source`, both via // io::upsert_gate(&SEED_SOURCE_NOT_READY_GATE, ...); cleared by diff --git a/crates/controller/src/restore/mod.rs b/crates/controller/src/restore/mod.rs index d0dc7167..ab30c697 100644 --- a/crates/controller/src/restore/mod.rs +++ b/crates/controller/src/restore/mod.rs @@ -262,10 +262,16 @@ async fn reconcile_inner(restore: &Restore, ctx: &Context) -> Result { // so a transient outage would permanently fail it. A not-Ready repository // therefore DEFERS (requeue); it never fakes a missing-snapshot outcome and // never launches a doomed Job. - if let Some(action) = - gate_on_repository_readiness(ctx, restore, &api, &namespace, &name).await? - { - return Ok(action); + // + // Matched exhaustively (CLAUDE.md), and both parking arms return BEFORE + // `ensure_wait_anchor` below — which is the whole of #393: a restore whose + // repository could not even be looked up must not open (and start spending) + // its `waitTimeout` window. + match gate_on_repository_readiness(ctx, restore, &api, &namespace, &name).await? { + RepositoryGate::Proceed => {} + RepositoryGate::Held(action) | RepositoryGate::Undetermined(action) => { + return Ok(action); + } } // The claiming PVC, looked up ONCE per pass and shared by everything below that needs @@ -3372,21 +3378,24 @@ async fn resolve_restore_repository( /// derivation rule (explicit `spec.repository` wins; `snapshotRef`/`fromPolicy` /// derive from the referent; a raw `identity` source has nothing to derive from). /// -/// Returns `Ok(None)` whenever the ref cannot be determined: a missing -/// `snapshotRef`/`fromPolicy` referent, a Snapshot with neither pin nor repository -/// owner, or `identity` without the (required) explicit `spec.repository`. -/// Deliberately NON-FATAL — a `snapshotRef` whose Snapshot CR does not exist yet is -/// a supported shape (`resolve_snapshot` parks it on the `waitTimeout` window), so -/// erroring here would break it; the other absences produce their canonical -/// `MissingDependency`/`Validation` surfaces downstream. The gate simply does not -/// engage until the repository is known. +/// Returns a [`RepoRefLookup`], not an `Option`: "the ref cannot be determined" +/// is several different situations and they do NOT get the same treatment +/// (issue #393 — the old `Option` collapsed a `SnapshotPolicy` that was never +/// applied together with a snapshot row the restore is legitimately waiting for, +/// and let the gate fall through unverified for both). Each `get_opt` result is +/// matched rather than chained through `and_then`, so "row missing" stays +/// distinct from "row present but underivable"; the classification itself is +/// pure and unit-tested ([`classify_snapshot_lookup`], [`classify_policy_lookup`]). +/// +/// Still deliberately NON-FATAL: nothing here errors. The gate decides what each +/// shape means. async fn restore_repository_ref( ctx: &Context, restore: &Restore, namespace: &str, -) -> Result> { +) -> Result { if let Some(rref) = &restore.spec.repository { - return Ok(Some((rref.clone(), namespace.to_string()))); + return Ok(RepoRefLookup::Derived(rref.clone(), namespace.to_string())); } match &restore.spec.source { RestoreSource::SnapshotRef(sref) => { @@ -3395,36 +3404,60 @@ async fn restore_repository_ref( // — the same base `resolve_restore_repository` uses. let snap_ns = sref.namespace.as_deref().unwrap_or(namespace); let snap_api: Api = Api::namespaced(ctx.client.clone(), snap_ns); - Ok(snap_api - .get_opt(&sref.name) - .await? - .as_ref() - .and_then(repository_ref_from_snapshot) - .map(|rref| (rref, snap_ns.to_string()))) + let snap = snap_api.get_opt(&sref.name).await?; + Ok(classify_snapshot_lookup(snap.as_ref(), snap_ns)) } RestoreSource::FromPolicy(c) => { use kopiur_api::SnapshotPolicy; let cfg_ns = c.namespace.as_deref().unwrap_or(namespace); let cfg_api: Api = Api::namespaced(ctx.client.clone(), cfg_ns); - Ok(cfg_api.get_opt(&c.name).await?.and_then(|cfg| { - // Multi-repo fromPolicy with no explicit selection: the - // readiness gate cannot know which repository to wait on, and - // this contract is deliberately NON-FATAL — return None (never - // guess repository #1); `resolve_restore_repository` fails - // closed downstream with the valid choices listed. - kopiur_api::single_repository_ref(&cfg.spec) - .ok() - .map(|r| (r.clone(), cfg_ns.to_string())) - })) + let cfg = cfg_api.get_opt(&c.name).await?; + Ok(classify_policy_lookup(cfg.as_ref(), cfg_ns, &c.name)) } - RestoreSource::Identity(_) => Ok(None), + // A raw identity source has nothing to derive from, and `spec.repository` + // is REQUIRED for it (checked above and refused downstream): a spec + // problem, never a missing object. + RestoreSource::Identity(_) => Ok(RepoRefLookup::NotDerivable), } } +/// Where the repository-readiness gate leaves a restore on this pass — a +/// TRI-STATE, because "the repository is not Ready" and "kopiur cannot tell +/// whether it is Ready" are different answers with different consequences for +/// the `waitTimeout` window (issue #393). +/// +/// [`Self::Held`] and [`Self::Undetermined`] both park and both requeue; they +/// are separate variants because they are separate *facts*, written with +/// different reasons, and only one of them is a verified statement about a +/// repository. Both stop the reconcile BEFORE [`ensure_wait_anchor`], so +/// neither opens the window. +/// +/// [`WaitWindow`] deliberately gains no variant for this: a parked restore never +/// reaches the window machinery at all, so there is no third window state to +/// model — the window is simply not open yet. +/// +/// **Match this enum EXHAUSTIVELY.** Never `matches!(…)` it and never add a +/// `_ =>` arm — a fourth outcome must force its caller to decide, at compile +/// time, whether it proceeds or parks. COMPILER-ONLY guard: `cargo xtask +/// check-phases` scans only the `*Phase` enums in `kopiur-api`. +enum RepositoryGate { + /// The gate does not hold this restore: reconcile continues (and the + /// `waitTimeout` window may open). + Proceed, + /// VERIFIED not ready: the repository object exists and its phase is not + /// `Ready` (the backend is unreachable). Park + requeue. + Held(Action), + /// UNDETERMINED: an object the readiness check needs does not exist — the + /// `Repository`/`ClusterRepository` itself, or the `fromPolicy` + /// `SnapshotPolicy` its ref is derived from. Park + requeue; nothing is + /// claimed about the backend. + Undetermined(Action), +} + /// The Restore peer of the Snapshot reconciler's repository-readiness gate: hold a /// not-yet-launched restore in `Pending` (`Ready=False`/`RepositoryNotReady`, /// non-terminal) while its repository is not `Ready`, and requeue on the same 15s -/// cadence. Returns `Some(requeue)` while gated, `None` to proceed. +/// cadence. Returns a [`RepositoryGate`] the caller matches exhaustively. /// /// Two semantic notes, both deliberate: /// - **`spec.mode: ReadOnly` is a different axis.** The Snapshot reconciler's mode @@ -3441,6 +3474,25 @@ async fn restore_repository_ref( /// a repository outage can never fail a restore — or fake a `Continue` empty-volume /// outcome — by silently burning the window while the backend is down. /// +/// **Tri-state (issue #393).** "Ready", "not Ready" and "cannot tell" are three +/// answers, and the gate used to give two: every cannot-tell shape fell through +/// as if the repository had been verified, so `ensure_wait_anchor` stamped the +/// window against nothing. Now a MISSING referent — the `Repository` object +/// itself, or the `fromPolicy` `SnapshotPolicy` its ref derives from — parks as +/// [`RepositoryGate::Undetermined`] with its own reason, no anchor stamped. Two +/// cannot-tell shapes still deliberately proceed: a `snapshotRef` whose +/// `Snapshot` row does not exist yet (that IS what the window waits for, and +/// `onMissingSnapshot: Fail` must be able to fire for a typo'd ref) and a +/// referent that exists but derives no single repository (a spec bug for +/// downstream validation to fail closed on). See [`RepoRefLookup`]. +/// +/// **Wake-up contract.** The `Repository`/`ClusterRepository` → `Restore` watches +/// cover only an explicit `spec.repository` (`watch::repository_to_restores`); +/// there is NO `SnapshotPolicy` → `Restore` watch. An `Undetermined` park +/// therefore un-parks on its own 15s requeue, not on an event — which is why +/// every `Undetermined` arm must return a requeue. (Adding the missing watch is +/// a possible follow-up; the requeue is correct either way.) +/// /// Only a restore that has NOT launched its mover Job is gated /// ([`restore_awaiting_launch`]); a `Restoring` restore's live Job is tracked to /// terminal, never re-gated, mirroring the Snapshot reconciler's ordering. (Narrow @@ -3453,53 +3505,193 @@ async fn gate_on_repository_readiness( api: &Api, namespace: &str, name: &str, -) -> Result> { +) -> Result { if !restore_awaiting_launch(restore.status.as_ref().and_then(|s| s.phase.as_ref())) { - return Ok(None); + return proceed_past_gate(restore, api, name).await; } - let Some((rref, base_ns)) = restore_repository_ref(ctx, restore, namespace).await? else { - return Ok(None); + let (rref, base_ns) = match restore_repository_ref(ctx, restore, namespace).await? { + RepoRefLookup::Derived(rref, base_ns) => (rref, base_ns), + // Both cannot-determine shapes that must NOT park — see the fn doc. + RepoRefLookup::SnapshotRowMissing | RepoRefLookup::NotDerivable => { + return proceed_past_gate(restore, api, name).await; + } + RepoRefLookup::ReferentMissing { + kind, + namespace: referent_ns, + name: referent, + } => { + return park_on_missing_referent( + ctx, + restore, + api, + name, + kind, + referent_ns.as_deref(), + &referent, + ) + .await; + } }; match io::repository_ready_cached(ctx, &rref, &base_ns).await { - Ok(true) => Ok(None), + Ok(true) => proceed_past_gate(restore, api, name).await, Ok(false) => { // patch-if-changed for byte-stability: the parked status is identical // across the 15s requeues, so a re-park is a server-side no-op — no // watch event, no self-trigger (the reconcile hot-loop rule). let current = serde_json::to_value(&restore.status).ok(); + // The repository object EXISTS, so any `ReferentAvailable=False` from + // an earlier park is stale — clear it in this same patch rather than a + // second conditions write (a merge patch replaces the whole array). + let conditions = cleared_referent_conditions(restore) + .unwrap_or_else(|| existing_conditions(restore)); io::patch_status_if_changed( api, name, current.as_ref(), - restore_ready_status( + restore_ready_status_on( restore, + &conditions, RestorePhase::Pending, crate::consts::REPOSITORY_NOT_READY_REASON, &repository_not_ready_restore_message(&rref.name), ), ) .await?; - Ok(Some(Action::requeue(std::time::Duration::from_secs(15)))) + Ok(RepositoryGate::Held(Action::requeue( + std::time::Duration::from_secs(15), + ))) + } + // The repository OBJECT is missing (not merely unreachable): the readiness + // check answered nothing, so this is UNDETERMINED, not "proceed". Pre-#393 + // it fell through here and the wait window opened against an unverified + // repository (#393). + Err(Error::MissingDependency(_)) => { + park_on_missing_referent( + ctx, + restore, + api, + name, + io::repo_kind_str(rref.kind), + repository_referent_namespace(&rref, &base_ns), + &rref.name, + ) + .await } - // The repository OBJECT is missing (not merely unreachable): not this - // gate's call — fall through so the resolve/dispatch paths surface their - // canonical `MissingDependency` hold, exactly as before the gate existed. - Err(Error::MissingDependency(_)) => Ok(None), Err(e) => Err(e), } } +/// The namespace a missing repository referent was looked up in — `None` for a +/// `ClusterRepository`, which is cluster-scoped and whose message must not +/// invent one. +fn repository_referent_namespace<'a>(rref: &'a RepositoryRef, base_ns: &'a str) -> Option<&'a str> { + match rref.kind { + kopiur_api::common::RepositoryKind::Repository => { + Some(rref.namespace.as_deref().unwrap_or(base_ns)) + } + kopiur_api::common::RepositoryKind::ClusterRepository => None, + } +} + +/// Leave the gate open, clearing a stale +/// [`crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION`] +/// `= False` from an earlier park first. +/// +/// The clear is guarded on the condition actually being present-and-not-True, so +/// the overwhelmingly common path (no such condition was ever written) costs +/// nothing and the healthy wire never grows the condition. Without it the +/// registry row — which is deliberately age-independent — would keep reporting a +/// restore that proceeded hours ago as blocked. +async fn proceed_past_gate( + restore: &Restore, + api: &Api, + name: &str, +) -> Result { + if let Some(conditions) = cleared_referent_conditions(restore) { + io::patch_status(api, name, serde_json::json!({ "conditions": conditions })).await?; + } + Ok(RepositoryGate::Proceed) +} + +/// Park a restore whose repository referent does not exist (issue #393): +/// `Pending` + the [`kopiur_api::gates::RESTORE_REFERENT_MISSING_GATE`] condition, +/// requeue 15s, and NO wait anchor — the caller returns before +/// [`ensure_wait_anchor`]. +/// +/// The Warning Event fires only on the park TRANSITION, gated by the same +/// status-changed check as the patch: the park re-patches a byte-identical +/// status every 15s (a server-side no-op), and Eventing that would be a +/// once-per-requeue spam. It compensates for what the pre-#393 fall-through +/// produced downstream — a `MissingDependency` error that `error_policy_for` +/// counted and Evented — which a clean park would otherwise silently remove. +async fn park_on_missing_referent( + ctx: &Context, + restore: &Restore, + api: &Api, + name: &str, + kind: &str, + referent_namespace: Option<&str>, + referent: &str, +) -> Result { + let msg = referent_missing_restore_message(kind, referent_namespace, referent); + let conditions = io::upsert_gate( + &existing_conditions(restore), + &kopiur_api::gates::RESTORE_REFERENT_MISSING_GATE, + &msg, + restore.metadata.generation, + ); + let current = serde_json::to_value(&restore.status).ok(); + let changed = io::patch_status_if_changed( + api, + name, + current.as_ref(), + restore_ready_status_on( + restore, + &conditions, + RestorePhase::Pending, + crate::consts::RESTORE_REFERENT_MISSING_REASON, + &msg, + ), + ) + .await?; + if changed { + io::publish_warning_event( + ctx, + restore, + crate::consts::RESTORE_REFERENT_MISSING_REASON, + crate::consts::CREATE_RESTORE_REFERENT_ACTION, + &msg, + ) + .await; + } + Ok(RepositoryGate::Undetermined(Action::requeue( + std::time::Duration::from_secs(15), + ))) +} + /// Open the `waitTimeout` window if it is not open yet, and return where it stands for /// THIS pass (#380). /// /// The window is stamped ONCE, into `status.waitStartedAt`, on the first pass that gets /// PAST [`gate_on_repository_readiness`] **and** — for a `target.populator` — has a PVC /// claiming this Restore (`consumer`). "Past the gate" is what the code enforces, and it is -/// weaker than "the repository is Ready": the gate also declines to hold a restore whose -/// mover Job is already live or settled ([`restore_awaiting_launch`]), and one whose -/// `Repository` OBJECT is missing outright. Stamping in those cases is harmless — a live -/// Job keeps the deadline it was dispatched with — and the ordering still guarantees the -/// thing that matters: a restore parked waiting for its backend never opens a window. +/// still slightly weaker than "the repository is Ready", but the gap is now deliberate and +/// narrow (#393). The gate no longer lets a MISSING referent through: a `Repository` +/// object that does not exist, or a `fromPolicy` `SnapshotPolicy` that does not exist, +/// parks as [`RepositoryGate::Undetermined`] and never reaches this function. +/// +/// What still gets past the gate unverified, on purpose: +/// - a restore whose mover Job is already live or settled ([`restore_awaiting_launch`]) — +/// a live Job keeps the deadline it was dispatched with; +/// - a `snapshotRef` whose `Snapshot` ROW does not exist yet — that row is exactly what +/// the window is waiting for, and parking instead would prevent `onMissingSnapshot: +/// Fail` from ever firing for a typo'd ref; +/// - a referent that exists but derives no single repository (a spec bug, failed closed +/// downstream by `resolve_restore_repository`). +/// +/// The ordering still guarantees the thing that matters, and now guarantees it for the +/// slow-arriving-referent case too: a restore parked waiting for its backend — or for the +/// object that names it — never opens a window. /// /// Both halves of the condition are about a restore that cannot yet do anything measuring /// a window it will need later: diff --git a/crates/controller/src/restore/plan.rs b/crates/controller/src/restore/plan.rs index 5dcf7298..7904210b 100644 --- a/crates/controller/src/restore/plan.rs +++ b/crates/controller/src/restore/plan.rs @@ -411,6 +411,157 @@ pub(super) fn existing_conditions(restore: &Restore) -> Vec { .map(|s| s.conditions.clone()) .unwrap_or_default() } + +/// What the repository-readiness gate could learn about the repository a restore +/// will connect to (issue #393) — the return of `super::restore_repository_ref`. +/// +/// The point of the enum is that "no repository ref" is THREE different +/// situations that the pre-#393 `Option<(RepositoryRef, String)>` collapsed into +/// one `None`, and only some of them may fall through the gate unverified: +/// +/// - [`Self::Derived`] — the ref is known; the gate goes on to check readiness. +/// - [`Self::SnapshotRowMissing`] — a `snapshotRef` whose `Snapshot` CR does not +/// exist (yet). Deliberately falls through: the `waitTimeout` window exists +/// precisely to wait for that row, and `onMissingSnapshot: Fail` must be able +/// to fire for a typo'd ref. Parking here would break both. +/// - [`Self::ReferentMissing`] — a `fromPolicy` whose `SnapshotPolicy` does not +/// exist. Nothing downstream can wait for it usefully, and letting the gate +/// fall through stamps `status.waitStartedAt` against a repository nobody +/// verified — the #393 bug. +/// - [`Self::NotDerivable`] — the referent EXISTS but names no single repository +/// (a `Snapshot` with neither pin nor repository owner, a multi-repository +/// `fromPolicy` with no explicit selection, a raw `identity` source). That is a +/// spec problem, not a missing object: it falls through to the downstream +/// validation that fails closed and lists the valid choices. Parking would +/// hide a permanent misconfiguration behind a "waiting…" message. +/// +/// **Match this enum EXHAUSTIVELY.** Never `matches!(…)` it and never add a +/// `_ =>` arm: the compiler's exhaustiveness check is the only thing that makes +/// a fifth shape decide, at compile time, whether it may spend a restore's wait +/// window. This is a COMPILER-ONLY guard: `cargo xtask check-phases` scans only +/// the `*Phase` enums in `kopiur-api`, so nothing but exhaustiveness protects it +/// — which is exactly why the rule is written down here (same convention as the +/// `unreadable_phase` named predicate in `repository_replication`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum RepoRefLookup { + /// The repository ref, plus the namespace it resolves relative to. + Derived(kopiur_api::common::RepositoryRef, String), + /// `snapshotRef` naming a `Snapshot` CR that does not exist — the shape the + /// `waitTimeout` window is FOR. The gate must not engage. + SnapshotRowMissing, + /// A referent object the repository ref is derived FROM does not exist. + /// `namespace` is `None` for a cluster-scoped referent. + ReferentMissing { + /// The referent's kind, for the message (`SnapshotPolicy`, …). + kind: &'static str, + /// The namespace it was looked up in; `None` when cluster-scoped. + namespace: Option, + /// The referent's name. + name: String, + }, + /// The referent exists but yields no single repository ref (or there is + /// nothing to derive from at all): a spec problem for downstream validation. + NotDerivable, +} + +/// Classify a `snapshotRef` lookup: the fetched `Snapshot` (`None` ⇒ the row +/// does not exist) into a [`RepoRefLookup`]. Pure. +/// +/// A missing ROW is [`RepoRefLookup::SnapshotRowMissing`] — a supported, +/// waited-for shape. A row that exists but carries no derivable repository +/// (neither `status.resolved.repository`, nor `spec.repository`, nor a +/// repository `ownerReference`) is [`RepoRefLookup::NotDerivable`]: the object +/// is there, so nothing will appear later to fix it. +pub(super) fn classify_snapshot_lookup( + snapshot: Option<&kopiur_api::Snapshot>, + snapshot_namespace: &str, +) -> RepoRefLookup { + match snapshot { + None => RepoRefLookup::SnapshotRowMissing, + Some(snap) => match kopiur_api::snapshot::repository_ref_for(snap) { + Some(rref) => RepoRefLookup::Derived(rref, snapshot_namespace.to_string()), + None => RepoRefLookup::NotDerivable, + }, + } +} + +/// Classify a `fromPolicy` lookup: the fetched `SnapshotPolicy` (`None` ⇒ the +/// object does not exist) into a [`RepoRefLookup`]. Pure. +/// +/// A missing POLICY is [`RepoRefLookup::ReferentMissing`] — the gate parks. A +/// policy that exists but fans out over several repositories with no explicit +/// selection is [`RepoRefLookup::NotDerivable`]: the gate cannot know which +/// repository to wait on and must never guess repository #1, so it falls through +/// to `resolve_restore_repository`, which fails closed listing the valid choices. +pub(super) fn classify_policy_lookup( + policy: Option<&kopiur_api::SnapshotPolicy>, + policy_namespace: &str, + policy_name: &str, +) -> RepoRefLookup { + match policy { + None => RepoRefLookup::ReferentMissing { + kind: "SnapshotPolicy", + namespace: Some(policy_namespace.to_string()), + name: policy_name.to_string(), + }, + Some(cfg) => match kopiur_api::single_repository_ref(&cfg.spec) { + Ok(rref) => RepoRefLookup::Derived(rref.clone(), policy_namespace.to_string()), + Err(_) => RepoRefLookup::NotDerivable, + }, + } +} + +/// Message for a restore parked because a referent it derives its repository +/// from does not exist (issue #393). Pure so the text is unit-asserted. +/// +/// Says what is missing (kind + namespaced name), why that blocks the restore, +/// that the `waitTimeout` window is deliberately NOT running meanwhile, and how +/// to clear it. +pub(super) fn referent_missing_restore_message( + kind: &str, + namespace: Option<&str>, + name: &str, +) -> String { + let target = match namespace { + Some(ns) => format!("{ns}/{name}"), + None => name.to_string(), + }; + format!( + "waiting for {kind} `{target}` to exist: the repository this restore connects to is \ + derived from it, so kopiur cannot verify the backend is reachable and will not launch \ + the restore. The `policy.waitTimeout` window is NOT running while this is missing \ + (status.waitStartedAt stays unstamped) — it opens once the {kind} exists and its \ + repository becomes `Ready`. Create the {kind} (or repoint the Restore at one that \ + exists) to proceed." + ) +} + +/// A stale [`crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION`] = `False` +/// left by an earlier park, flipped back to `True` — or `None` when there is +/// nothing to clear (the healthy wire never grows the condition). Pure. +/// +/// Without this the park's gate condition outlives the park: the registry row is +/// age-independent, so `kubectl kopiur doctor` would keep reporting a restore +/// that has long since proceeded as blocked on a referent that has existed for +/// hours. Mirrors how the `MoverPermitted`/`CredentialsAvailable` gates clear. +pub(super) fn cleared_referent_conditions(restore: &Restore) -> Option> { + use crate::consts::{RESTORE_REFERENT_AVAILABLE_CONDITION, RESTORE_REFERENT_FOUND_REASON}; + let existing = existing_conditions(restore); + if !existing + .iter() + .any(|c| c.type_ == RESTORE_REFERENT_AVAILABLE_CONDITION && c.status != "True") + { + return None; + } + Some(io::upsert_condition( + &existing, + RESTORE_REFERENT_AVAILABLE_CONDITION, + true, + RESTORE_REFERENT_FOUND_REASON, + "the referent the restore derives its repository from now exists", + restore.metadata.generation, + )) +} /// Where the `waitTimeout` window is anchored: `status.waitStartedAt` once /// [`super::ensure_wait_anchor`] has stamped it, else `created_epoch`. /// diff --git a/crates/controller/src/restore/tests.rs b/crates/controller/src/restore/tests.rs index b765e582..5bb8a21b 100644 --- a/crates/controller/src/restore/tests.rs +++ b/crates/controller/src/restore/tests.rs @@ -142,6 +142,157 @@ fn repository_not_ready_restore_message_says_what_why_how() { ); } +// --- #393: the readiness gate's tri-state repository lookup --------------- + +/// A `Snapshot` fixture, parsed the cluster's way. `pin` is the +/// `spec.repository` mint-time pin (`None` ⇒ nothing derivable at all). +fn snapshot_fixture(pin: Option<&str>) -> kopiur_api::Snapshot { + let mut spec = serde_json::json!({ "sources": [ { "pvc": { "name": "data" } } ] }); + if let Some(name) = pin { + spec["repository"] = serde_json::json!({ "kind": "Repository", "name": name }); + } + serde_json::from_value(serde_json::json!({ + "apiVersion": "kopiur.home-operations.com/v1alpha1", + "kind": "Snapshot", + "metadata": { "name": "b", "namespace": "apps" }, + "spec": spec, + })) + .expect("Snapshot fixture") +} + +/// A `SnapshotPolicy` fixture: one repository (`repository`) or a +/// multi-repository fan-out (`repositories`), which has no single ref. +fn policy_fixture(repositories: &[&str]) -> kopiur_api::SnapshotPolicy { + let mut spec = serde_json::json!({ "sources": [ { "pvc": { "name": "data" } } ] }); + match repositories { + [one] => spec["repository"] = serde_json::json!({ "kind": "Repository", "name": one }), + many => { + spec["repositories"] = serde_json::json!( + many.iter() + .map(|n| serde_json::json!({ "kind": "Repository", "name": n })) + .collect::>() + ) + } + } + serde_json::from_value(serde_json::json!({ + "apiVersion": "kopiur.home-operations.com/v1alpha1", + "kind": "SnapshotPolicy", + "metadata": { "name": "cfg", "namespace": "apps" }, + "spec": spec, + })) + .expect("SnapshotPolicy fixture") +} + +/// The distinction the pre-#393 chained `get_opt(..).and_then(..)` erased: a +/// `Snapshot` ROW that does not exist is a supported, waited-for shape; a row +/// that exists but names no repository is a spec problem. +#[test] +fn snapshot_lookup_separates_a_missing_row_from_an_underivable_one() { + // Missing row: the shape the `waitTimeout` window is FOR. The gate must not + // engage, or `onMissingSnapshot: Fail` could never fire for a typo'd ref. + assert_eq!( + classify_snapshot_lookup(None, "apps"), + RepoRefLookup::SnapshotRowMissing + ); + // Row present WITH a pin: derived, relative to the snapshot's namespace. + let pinned = snapshot_fixture(Some("nas")); + match classify_snapshot_lookup(Some(&pinned), "apps") { + RepoRefLookup::Derived(rref, ns) => { + assert_eq!(rref.name, "nas"); + assert_eq!(ns, "apps"); + } + other => panic!("a pinned Snapshot must derive its repository: {other:?}"), + } + // Row present WITHOUT a pin (no status, no spec.repository, no repository + // owner): the object is there, so nothing will appear later to fix it — + // falls through to downstream validation rather than parking forever. + assert_eq!( + classify_snapshot_lookup(Some(&snapshot_fixture(None)), "apps"), + RepoRefLookup::NotDerivable + ); +} + +/// The other half: a `SnapshotPolicy` that does not exist parks the gate; one +/// that exists but fans out over several repositories does not (the gate must +/// never guess repository #1). +#[test] +fn policy_lookup_separates_a_missing_policy_from_a_multi_repo_one() { + assert_eq!( + classify_policy_lookup(None, "apps", "cfg"), + RepoRefLookup::ReferentMissing { + kind: "SnapshotPolicy", + namespace: Some("apps".into()), + name: "cfg".into(), + } + ); + let single = policy_fixture(&["nas"]); + match classify_policy_lookup(Some(&single), "apps", "cfg") { + RepoRefLookup::Derived(rref, ns) => { + assert_eq!(rref.name, "nas"); + assert_eq!(ns, "apps"); + } + other => panic!("a single-repository policy must derive its repository: {other:?}"), + } + // Multi-repo with no explicit selection: `resolve_restore_repository` fails + // closed downstream listing the valid choices — parking would hide that. + assert_eq!( + classify_policy_lookup(Some(&policy_fixture(&["a", "b"])), "apps", "cfg"), + RepoRefLookup::NotDerivable + ); +} + +#[test] +fn referent_missing_message_says_what_why_and_how() { + let msg = referent_missing_restore_message("SnapshotPolicy", Some("apps"), "cfg"); + // WHAT is missing, namespaced. + assert!(msg.contains("SnapshotPolicy `apps/cfg`"), "{msg}"); + // WHY it blocks: the repository is derived from it and cannot be verified. + assert!(msg.contains("derived from it"), "{msg}"); + // The #393 promise itself: the window is NOT running meanwhile. + assert!(msg.contains("waitTimeout"), "{msg}"); + assert!(msg.contains("status.waitStartedAt"), "{msg}"); + // HOW to clear it. + assert!(msg.contains("Create the SnapshotPolicy"), "{msg}"); + // A cluster-scoped referent must not be given an invented namespace. + let cluster = referent_missing_restore_message("ClusterRepository", None, "offsite"); + assert!(cluster.contains("ClusterRepository `offsite`"), "{cluster}"); + assert!(!cluster.contains('/'), "{cluster}"); + // The reason is distinct from the not-Ready one: a SnapshotPolicy that was + // never applied is not an unreachable backend. + assert_ne!( + crate::consts::RESTORE_REFERENT_MISSING_REASON, + crate::consts::REPOSITORY_NOT_READY_REASON + ); + assert_eq!( + crate::consts::RESTORE_REFERENT_MISSING_REASON, + "RestoreReferentMissing" + ); +} + +/// The park's gate condition must not outlive the park: the registry row is +/// age-independent, so a stale `ReferentAvailable=False` would keep `kubectl +/// kopiur doctor` reporting a restore that proceeded hours ago as blocked. +#[test] +fn the_referent_gate_condition_clears_only_when_it_is_stale() { + use crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION; + // Never parked: nothing to clear, and the healthy wire must not GROW the + // condition (a write per pass would be pure churn). + assert!(cleared_referent_conditions(&restore_with_condition("Resolved", "True")).is_none()); + // Parked: flipped back to True in place, keeping the array a single row. + let parked = restore_with_condition(RESTORE_REFERENT_AVAILABLE_CONDITION, "False"); + let cleared = cleared_referent_conditions(&parked).expect("a stale park must clear"); + let row = cleared + .iter() + .find(|c| c.type_ == RESTORE_REFERENT_AVAILABLE_CONDITION) + .expect("the condition survives, flipped"); + assert_eq!(row.status, "True"); + assert_eq!(row.reason, crate::consts::RESTORE_REFERENT_FOUND_REASON); + // Already cleared: idempotent, so the clear cannot flip-flop with the + // condition writers that rebuild from the reconcile-start copy. + let healthy = restore_with_condition(RESTORE_REFERENT_AVAILABLE_CONDITION, "True"); + assert!(cleared_referent_conditions(&healthy).is_none()); +} + #[test] fn populator_state_depends_on_target_variant() { use kopiur_api::PopulatorTarget; diff --git a/crates/e2e/mise.toml b/crates/e2e/mise.toml index ca80f5c1..191f62fc 100644 --- a/crates/e2e/mise.toml +++ b/crates/e2e/mise.toml @@ -108,7 +108,7 @@ docker exec "$NODE" sh -c ' # PVC ROOT is the kopia repo — a subdir under one shared PVC gives no isolation. # Each scenario binds its own PVC over one of these dirs. KEEP IN LOCKSTEP with # crates/e2e/src/consts.rs::REPO_SUBPATHS. - for s in moverdefaults scc-shadow recmeta nsdel-orphan nsdel-delete pin pinrestore populator populator2 popempty popempty2 readonly kstatus verify vfydeep vfyinh vfygate repl-src repl-dst repl-s3-src repl-run-src repl-run-dst projgate hooks gfs errh colocation colocation-off colocation-missing copymethod copymethod-csi copymethod-explicit idxhealth epochparams epochparams-default staging-recover staging-timeout staging-override staging-mismatch massdel-cascade massdel-breaker massdel-prune massdel-single massdel-nooverlap massdel-throttle massdel-outage massdel-heldprune adopt-prune adopt-ignore polcasc-retain polcasc-delete polcasc-simul recrestore waitanchor multipvc-fanout multipvc-group unchanged-dedup unchanged-default srepl-src srepl-dst srepl-coex-src srepl-coex-dst srepl-sel-src srepl-sel-dst srepl-susp-src srepl-susp-dst srepl-run-src srepl-run-dst mrepo-a mrepo-b mrepo-ret-a mrepo-ret-b mrepo-down-a mrepo-down-b mrepo-rest-a mrepo-rest-b seed-primary seed-mirror seed-blob seed-mig-src seed-mig-dst seed-empty-src seed-empty-dst; do + for s in moverdefaults scc-shadow recmeta nsdel-orphan nsdel-delete pin pinrestore populator populator2 popempty popempty2 readonly kstatus verify vfydeep vfyinh vfygate repl-src repl-dst repl-s3-src repl-run-src repl-run-dst projgate hooks gfs errh colocation colocation-off colocation-missing copymethod copymethod-csi copymethod-explicit idxhealth epochparams epochparams-default staging-recover staging-timeout staging-override staging-mismatch massdel-cascade massdel-breaker massdel-prune massdel-single massdel-nooverlap massdel-throttle massdel-outage massdel-heldprune adopt-prune adopt-ignore polcasc-retain polcasc-delete polcasc-simul recrestore waitanchor waitref multipvc-fanout multipvc-group unchanged-dedup unchanged-default srepl-src srepl-dst srepl-coex-src srepl-coex-dst srepl-sel-src srepl-sel-dst srepl-susp-src srepl-susp-dst srepl-run-src srepl-run-dst mrepo-a mrepo-b mrepo-ret-a mrepo-ret-b mrepo-down-a mrepo-down-b mrepo-rest-a mrepo-rest-b seed-primary seed-mirror seed-blob seed-mig-src seed-mig-dst seed-empty-src seed-empty-dst; do mkdir -p "/kopiur-e2e/repos/$s" done # Source dir with one UNREADABLE file for the errorHandling.ignoreFileErrors diff --git a/crates/e2e/src/consts.rs b/crates/e2e/src/consts.rs index 4e151d9d..aa3d53ed 100644 --- a/crates/e2e/src/consts.rs +++ b/crates/e2e/src/consts.rs @@ -144,6 +144,11 @@ pub const REPO_SUBPATHS: &[&str] = &[ // (repository Ready), not at the Restore's creation — needs its own repo, created // SUSPENDED so it is cold until the scenario releases it. "waitanchor", + // #393: the readiness gate must PARK (no `waitStartedAt`) while the repository + // referent does not exist. Distinct from `waitanchor`, whose repository + // pre-exists (suspended): here the `Repository` and the `SnapshotPolicy` are + // created MID-TEST, so the shard must start with no repository at all. + "waitref", // SnapshotReplication (issue #368, crates/e2e/tests/snapshot_replication.rs): // logical fs→fs replication. Each scenario needs its OWN source AND destination // kopia repository so copy/prune/idempotency counts never leak between scenarios: diff --git a/crates/e2e/tests/restore.rs b/crates/e2e/tests/restore.rs index bbbba75b..b4ade596 100644 --- a/crates/e2e/tests/restore.rs +++ b/crates/e2e/tests/restore.rs @@ -1808,3 +1808,339 @@ async fn restore_wait_window_opens_when_the_repository_becomes_ready() { cleanup_restore(&restores, WA_RESTORE).await; } + +// --- #393: the readiness gate is TRI-STATE — a MISSING referent parks ------- + +/// Isolated repo for the two referent-park scenarios: unlike [`WA_SUBPATH`], whose +/// `Repository` pre-exists (created suspended), here the repository is created +/// **mid-test** — the whole point is a `Restore` that arrives before the objects it +/// derives its repository from. Subpath in `consts::REPO_SUBPATHS` + the mise +/// node-seed list (LOCKSTEP). +const WR_SUBPATH: &str = "waitref"; +/// Scenario 1: the `Repository` the Restore names explicitly. +const WR_REPO: &str = "e2e-wr-repo"; +const WR_RESTORE: &str = "e2e-r-waitref"; +/// Scenario 2: a `fromPolicy` Restore with NO `spec.repository`, so the repository +/// ref is derived from this `SnapshotPolicy` — which is what goes missing. +const WR_CFG: &str = "e2e-wr-cfg"; +const WR_CFG_REPO: &str = "e2e-wr-cfg-repo"; +const WR_CFG_RESTORE: &str = "e2e-r-waitref-policy"; +/// The `waitTimeout` both scenarios configure. Long enough that the window cannot +/// close mid-test (which would turn "the anchor stamped" into a race), and it only +/// ever starts running after the assertions are done. +const WR_WINDOW: &str = "10m"; +/// How long the park is observed to HOLD with no anchor — comfortably more than the +/// gate's 15s requeue, so a gate that stamps `waitStartedAt` on a later pass (rather +/// than only on the first) still fails this. +const WR_PARK_PROBE_SECS: u64 = 45; + +/// The `Restore` under test for the referent scenarios: a direct `target.pvc` with a +/// `waitTimeout`, plus the caller's `source`/`repository` fields. +fn waitref_restore_json(name: &str, extra_spec: serde_json::Value) -> serde_json::Value { + let mut spec = serde_json::json!({ + "policy": { "waitTimeout": WR_WINDOW }, + "target": { "pvc": { "name": format!("{name}-dst"), "capacity": "1Gi", "accessModes": ["ReadWriteOnce"] } } + }); + if let Some(extra) = extra_spec.as_object() { + for (k, v) in extra { + spec[k] = v.clone(); + } + } + serde_json::json!({ + "apiVersion": "kopiur.home-operations.com/v1alpha1", + "kind": "Restore", + "metadata": { "name": name, "namespace": E2E_NAMESPACE }, + "spec": spec + }) +} + +/// Poll until `status.waitStartedAt` is stamped, returning it. +async fn wait_anchor_stamped(restores: &Api, name: &str) -> anyhow::Result { + wait_until( + &format!("{name} stamps status.waitStartedAt"), + default_timeout(), + poll_interval(), + || { + let restores = restores.clone(); + let name = name.to_string(); + async move { + let s = status_json(&restores, &name).await; + Ok(s.get("waitStartedAt") + .and_then(|v| v.as_str()) + .map(str::to_string)) + } + }, + ) + .await +} + +/// #393 — the wait window must not open against a `Repository` that does not exist. +/// +/// Pre-fix the readiness gate conflated "verified Ready" with three cannot-determine +/// shapes: a missing `Repository` OBJECT made `repository_ready_cached` return +/// `MissingDependency`, the gate fell through as if the repository had been checked, +/// and `ensure_wait_anchor` stamped `status.waitStartedAt` ≈ the Restore's creation. +/// A `Repository` that then took longer than `waitTimeout` to arrive (a GitOps apply +/// ordering, a DR bring-up) found the window already spent. +/// +/// Post-fix the gate parks: `Pending`, `ReferentAvailable=False` / +/// `RestoreReferentMissing`, and NO anchor — asserted over a stable poll so a gate +/// that merely stamps one pass later still fails. Creating the `Repository` releases +/// it (this path also has a watch), and only THEN does the anchor appear. +#[tokio::test] +#[ignore = "requires the e2e harness (mise run //crates/e2e:test)"] +async fn restore_parks_without_wait_anchor_until_its_repository_exists() { + let Some(world) = World::connect().await else { + return; + }; + world + .ensure(&[Need::Filesystem]) + .await + .expect("fixtures ready"); + let client = world.client().clone(); + common::ensure_repo(&client, WR_SUBPATH).await; + + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + let restores: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + + // Clean slate: the scenario is only meaningful while the Repository is ABSENT. + cleanup_restore(&restores, WR_RESTORE).await; + let _ = repos.delete(WR_REPO, &DeleteParams::default()).await; + wait_until( + "the waitref Repository is gone", + default_timeout(), + poll_interval(), + || { + let repos = repos.clone(); + async move { Ok(repos.get_opt(WR_REPO).await?.is_none().then_some(())) } + }, + ) + .await + .expect("the previous run's waitref Repository should be deleted"); + + // A Restore naming a Repository that does not exist. The source snapshotRef is + // deliberately a name that never appears: nothing about it must matter until the + // repository shows up. + restores + .create( + &PostParams::default(), + &cr(waitref_restore_json( + WR_RESTORE, + serde_json::json!({ + "repository": { "kind": "Repository", "name": WR_REPO }, + "source": { "snapshotRef": { "name": "e2e-wr-never" } } + }), + )), + ) + .await + .expect("create the Restore whose Repository does not exist yet"); + + wait_condition(&restores, WR_RESTORE, "ReferentAvailable", "False") + .await + .expect("a Restore whose Repository object is missing must park on the referent gate"); + let parked = status_json(&restores, WR_RESTORE).await; + assert_eq!( + parked.get("phase").and_then(|v| v.as_str()), + Some("Pending"), + "the park is non-terminal: {parked}" + ); + assert_eq!( + condition_reason(&parked, "ReferentAvailable"), + "RestoreReferentMissing", + "the park must use its OWN reason, not RepositoryNotReady (which would claim \ + the backend was checked): {parked}" + ); + assert_eq!( + condition_reason(&parked, "Ready"), + "RestoreReferentMissing", + "the Ready condition carries the same reason: {parked}" + ); + let message = condition_field(&parked, "ReferentAvailable", "message"); + assert!( + message.contains(WR_REPO), + "the message must name the missing object: {message}" + ); + assert!( + parked.get("waitStartedAt").is_none(), + "the wait window must NOT open while the Repository object does not exist: {parked}" + ); + + // ...and it STAYS closed across several requeues (the pre-fix bug stamped on the + // very first pass, but a partial fix could stamp on a later one). + tokio::time::sleep(std::time::Duration::from_secs(WR_PARK_PROBE_SECS)).await; + let still_parked = status_json(&restores, WR_RESTORE).await; + assert!( + still_parked.get("waitStartedAt").is_none(), + "the park must hold the window closed for as long as the Repository is absent: \ + {still_parked}" + ); + assert_eq!( + still_parked.get("phase").and_then(|v| v.as_str()), + Some("Pending"), + "still parked, still non-terminal: {still_parked}" + ); + + // Create the Repository: it bootstraps, reaches Ready, and the gate opens. + repos + .create( + &PostParams::default(), + &cr(common::repository_json( + WR_REPO, + WR_SUBPATH, + serde_json::json!({ "maintenance": { "enabled": false } }), + )), + ) + .await + .expect("create the waitref Repository"); + wait_phase(&repos, WR_REPO, "Ready") + .await + .expect("the waitref Repository should reach Ready"); + + // The anchor stamps only NOW — the same assertion shape as the #380 scenario. + let opened = wait_anchor_stamped(&restores, WR_RESTORE).await.expect( + "the released Restore must stamp status.waitStartedAt once its repository is Ready", + ); + assert!( + chrono::DateTime::parse_from_rfc3339(&opened).is_ok(), + "waitStartedAt {opened} must be RFC3339" + ); + // The gate condition clears rather than lingering as a phantom block for + // `kubectl kopiur doctor`. + wait_condition(&restores, WR_RESTORE, "ReferentAvailable", "True") + .await + .expect("the referent gate must clear once the Repository exists"); + + cleanup_restore(&restores, WR_RESTORE).await; + let _ = repos.delete(WR_REPO, &DeleteParams::default()).await; +} + +/// #393 — the same park for a `fromPolicy` source whose `SnapshotPolicy` is missing, +/// and the un-park path that has NO watch behind it. +/// +/// This Restore sets no `spec.repository`, so the repository ref is DERIVED from the +/// `SnapshotPolicy` — and there is no `SnapshotPolicy` → `Restore` watch +/// (`watch::repository_to_restores` covers only an explicit `spec.repository`). The +/// park therefore has to un-park on its own 15s requeue, which is exactly what this +/// scenario proves: an `Undetermined` arm that forgot to requeue would park here +/// FOREVER, and every phase-based check would report the Restore as merely "Pending". +#[tokio::test] +#[ignore = "requires the e2e harness (mise run //crates/e2e:test)"] +async fn restore_parks_when_frompolicy_names_a_missing_policy() { + let Some(world) = World::connect().await else { + return; + }; + world + .ensure(&[Need::Filesystem]) + .await + .expect("fixtures ready"); + let client = world.client().clone(); + common::ensure_repo(&client, WR_SUBPATH).await; + + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + let configs: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + let restores: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + + // Clean slate: the SnapshotPolicy must be ABSENT when the Restore is created. + cleanup_restore(&restores, WR_CFG_RESTORE).await; + let _ = configs.delete(WR_CFG, &DeleteParams::default()).await; + wait_until( + "the waitref SnapshotPolicy is gone", + default_timeout(), + poll_interval(), + || { + let configs = configs.clone(); + async move { Ok(configs.get_opt(WR_CFG).await?.is_none().then_some(())) } + }, + ) + .await + .expect("the previous run's waitref SnapshotPolicy should be deleted"); + + restores + .create( + &PostParams::default(), + &cr(waitref_restore_json( + WR_CFG_RESTORE, + // No `spec.repository`: the repository is derived from the policy, + // which is the referent under test. + serde_json::json!({ "source": { "fromPolicy": { "name": WR_CFG } } }), + )), + ) + .await + .expect("create the Restore whose SnapshotPolicy does not exist yet"); + + wait_condition(&restores, WR_CFG_RESTORE, "ReferentAvailable", "False") + .await + .expect("a fromPolicy Restore whose SnapshotPolicy is missing must park"); + let parked = status_json(&restores, WR_CFG_RESTORE).await; + assert_eq!( + parked.get("phase").and_then(|v| v.as_str()), + Some("Pending"), + "the park is non-terminal: {parked}" + ); + assert_eq!( + condition_reason(&parked, "ReferentAvailable"), + "RestoreReferentMissing", + "{parked}" + ); + let message = condition_field(&parked, "ReferentAvailable", "message"); + assert!( + message.contains("SnapshotPolicy") && message.contains(WR_CFG), + "the message must name the missing SnapshotPolicy, not a repository: {message}" + ); + assert!( + parked.get("waitStartedAt").is_none(), + "the wait window must NOT open while the SnapshotPolicy does not exist: {parked}" + ); + tokio::time::sleep(std::time::Duration::from_secs(WR_PARK_PROBE_SECS)).await; + let still_parked = status_json(&restores, WR_CFG_RESTORE).await; + assert!( + still_parked.get("waitStartedAt").is_none(), + "the park must hold the window closed for as long as the policy is absent: \ + {still_parked}" + ); + + // Provide the referent (and the repository it names). NOTHING watches + // SnapshotPolicy on behalf of a Restore, so the un-park below can only come from + // the gate's own 15s requeue. + if repos.get_opt(WR_CFG_REPO).await.ok().flatten().is_none() { + let _ = repos + .create( + &PostParams::default(), + &cr(common::repository_json( + WR_CFG_REPO, + WR_SUBPATH, + serde_json::json!({ "maintenance": { "enabled": false } }), + )), + ) + .await; + } + wait_phase(&repos, WR_CFG_REPO, "Ready") + .await + .expect("the waitref policy Repository should reach Ready"); + configs + .create( + &PostParams::default(), + &cr(common::snapshot_policy_json( + E2E_NAMESPACE, + WR_CFG, + "Repository", + WR_CFG_REPO, + serde_json::json!({}), + )), + ) + .await + .expect("create the waitref SnapshotPolicy"); + + // Un-parked by the requeue: the anchor finally opens. + wait_anchor_stamped(&restores, WR_CFG_RESTORE).await.expect( + "the parked Restore must un-park on its own requeue once the SnapshotPolicy exists \ + — there is no SnapshotPolicy watch to wake it", + ); + wait_condition(&restores, WR_CFG_RESTORE, "ReferentAvailable", "True") + .await + .expect("the referent gate must clear once the SnapshotPolicy exists"); + + cleanup_restore(&restores, WR_CFG_RESTORE).await; + let _ = configs.delete(WR_CFG, &DeleteParams::default()).await; + let _ = repos.delete(WR_CFG_REPO, &DeleteParams::default()).await; +} diff --git a/deploy/crds/all-crds.yaml b/deploy/crds/all-crds.yaml index b166c59d..0335f6b8 100644 --- a/deploy/crds/all-crds.yaml +++ b/deploy/crds/all-crds.yaml @@ -11075,13 +11075,17 @@ spec: description: |- When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a - `target.populator` a PVC already claims it), NOT when the Restore was created. If the - referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate - falls through unverified and the anchor can be stamped as early as creation. Stamped - once and then honored verbatim, so the window survives controller restarts and Job pod - retries; cleared when a populator re-opens resolution for a re-created claim, so that - claim gets the full window again. Absent means the window has not opened yet (or no - `policy.waitTimeout` is configured, in which case there is no window to anchor). + `target.populator` a PVC already claims it), NOT when the Restore was created. The + window does NOT open while the referenced `Repository` object or `fromPolicy` + `SnapshotPolicy` doesn't exist: the restore parks in `Pending` + (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It + DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so + `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose + mover Job already launched. Stamped once and then honored verbatim, so the window + survives controller restarts and Job pod retries; cleared when a populator re-opens + resolution for a re-created claim, so that claim gets the full window again. Absent + means the window has not opened yet (or no `policy.waitTimeout` is configured, in + which case there is no window to anchor). nullable: true type: string type: object diff --git a/deploy/crds/restores.yaml b/deploy/crds/restores.yaml index 73d541d2..172997d0 100644 --- a/deploy/crds/restores.yaml +++ b/deploy/crds/restores.yaml @@ -944,13 +944,17 @@ spec: description: |- When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a - `target.populator` a PVC already claims it), NOT when the Restore was created. If the - referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate - falls through unverified and the anchor can be stamped as early as creation. Stamped - once and then honored verbatim, so the window survives controller restarts and Job pod - retries; cleared when a populator re-opens resolution for a re-created claim, so that - claim gets the full window again. Absent means the window has not opened yet (or no - `policy.waitTimeout` is configured, in which case there is no window to anchor). + `target.populator` a PVC already claims it), NOT when the Restore was created. The + window does NOT open while the referenced `Repository` object or `fromPolicy` + `SnapshotPolicy` doesn't exist: the restore parks in `Pending` + (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It + DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so + `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose + mover Job already launched. Stamped once and then honored verbatim, so the window + survives controller restarts and Job pod retries; cleared when a populator re-opens + resolution for a re-created claim, so that claim gets the full window again. Absent + means the window has not opened yet (or no `policy.waitTimeout` is configured, in + which case there is no window to anchor). nullable: true type: string type: object diff --git a/deploy/helm/kopiur/crds/restores.yaml b/deploy/helm/kopiur/crds/restores.yaml index 73d541d2..172997d0 100644 --- a/deploy/helm/kopiur/crds/restores.yaml +++ b/deploy/helm/kopiur/crds/restores.yaml @@ -944,13 +944,17 @@ spec: description: |- When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a - `target.populator` a PVC already claims it), NOT when the Restore was created. If the - referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate - falls through unverified and the anchor can be stamped as early as creation. Stamped - once and then honored verbatim, so the window survives controller restarts and Job pod - retries; cleared when a populator re-opens resolution for a re-created claim, so that - claim gets the full window again. Absent means the window has not opened yet (or no - `policy.waitTimeout` is configured, in which case there is no window to anchor). + `target.populator` a PVC already claims it), NOT when the Restore was created. The + window does NOT open while the referenced `Repository` object or `fromPolicy` + `SnapshotPolicy` doesn't exist: the restore parks in `Pending` + (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It + DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so + `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose + mover Job already launched. Stamped once and then honored verbatim, so the window + survives controller restarts and Job pod retries; cleared when a populator re-opens + resolution for a re-created claim, so that claim gets the full window again. Absent + means the window has not opened yet (or no `policy.waitTimeout` is configured, in + which case there is no window to anchor). nullable: true type: string type: object diff --git a/docs/field-reference.md b/docs/field-reference.md index 5a4a997f..9f5bbe11 100644 --- a/docs/field-reference.md +++ b/docs/field-reference.md @@ -3077,7 +3077,7 @@ Externally tagged — set **exactly one** of: `pvcConsumer` · `snapshot` · `wo | `sourceKind` | string | — | The pinned source kind (`SnapshotRef`/`FromPolicy`/`Identity`); backs the `SOURCE` printer column. | | `target` | [object](#restore-status-target) | — | Resolved target details (the PVC written to / populator handshake). | | `timing` | [object](#restore-status-timing) | — | Start/end timestamps for the restore run. | -| `waitStartedAt` | string | — | When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a `target.populator` a PVC already claims it), NOT when the Restore was created. If the referenced `Repository`/`SnapshotPolicy` object doesn't exist yet, the readiness gate falls through unverified and the anchor can be stamped as early as creation. Stamped once and then honored verbatim, so the window survives controller restarts and Job pod retries; cleared when a populator re-opens resolution for a re-created claim, so that claim gets the full window again. Absent means the window has not opened yet (or no `policy.waitTimeout` is configured, in which case there is no window to anchor). | +| `waitStartedAt` | string | — | When the `policy.waitTimeout` window OPENED (RFC3339) — the first reconcile on which the restore could actually proceed (its repository reached `Ready`, and for a `target.populator` a PVC already claims it), NOT when the Restore was created. The window does NOT open while the referenced `Repository` object or `fromPolicy` `SnapshotPolicy` doesn't exist: the restore parks in `Pending` (`ReferentAvailable=False`, reason `RestoreReferentMissing`) and stays unstamped. It DOES still open for a `snapshotRef` whose `Snapshot` row doesn't exist yet (so `onMissingSnapshot` can fire for a ref that never appears) and for a restore whose mover Job already launched. Stamped once and then honored verbatim, so the window survives controller restarts and Job pod retries; cleared when a populator re-opens resolution for a re-created claim, so that claim gets the full window again. Absent means the window has not opened yet (or no `policy.waitTimeout` is configured, in which case there is no window to anchor). | #### `status.conditions[]` { #restore-status-conditions } diff --git a/docs/gitops.md b/docs/gitops.md index b1f13489..0d50841e 100644 --- a/docs/gitops.md +++ b/docs/gitops.md @@ -105,7 +105,10 @@ exactly what they remove. `dataSourceRef` (see [deploy-or-restore](restores.md#deploy-or-restore-gitops)). Its `waitTimeout` window opens when the restore can first proceed, not when you applied it, so a populator that sits in Git for months does not arrive with its - window already spent. + window already spent. The same holds for apply ordering within one commit: a + `Restore` whose `Repository` or `SnapshotPolicy` hasn't landed yet parks in + `Pending` (`RestoreReferentMissing`) with the window still closed, and un-parks + by itself once the referenced object appears. `spec.seed` is also **mutable**: editing it mid-seed lets the running Job finish, discards its result as stale, and relaunches for the live spec — so a GitOps diff --git a/docs/reference/crds/restore.md b/docs/reference/crds/restore.md index b6742ec3..e6367c97 100644 --- a/docs/reference/crds/restore.md +++ b/docs/reference/crds/restore.md @@ -36,7 +36,7 @@ The rest mirror `kopia snapshot restore`'s own flags one-to-one and are all tri- ### `policy` -How the restore reacts to a missing snapshot. `onMissingSnapshot` is `Fail` (fail-closed; the default for explicit `snapshotRef`/`identity` sources, so an explicit restore can never silently no-op) or `Continue` (proceed with an empty, deploy-or-restore volume; the default for `fromPolicy`). `waitTimeout` (e.g. `5m`) bounds how long the restore waits for the source snapshot to appear before giving up. The window opens when the restore can first proceed — its repository is `Ready`, and for `target.populator` a PVC claims it — and that instant is recorded in `status.waitStartedAt`; it is **not** measured from `metadata.creationTimestamp`. See [Restores → `waitTimeout`](../../restores.md#waittimeout--wait-before-giving-up). +How the restore reacts to a missing snapshot. `onMissingSnapshot` is `Fail` (fail-closed; the default for explicit `snapshotRef`/`identity` sources, so an explicit restore can never silently no-op) or `Continue` (proceed with an empty, deploy-or-restore volume; the default for `fromPolicy`). `waitTimeout` (e.g. `5m`) bounds how long the restore waits for the source snapshot to appear before giving up. The window opens when the restore can first proceed — its repository is `Ready`, and for `target.populator` a PVC claims it — and that instant is recorded in `status.waitStartedAt`; it is **not** measured from `metadata.creationTimestamp`. It does not open at all while the referenced `Repository` object or `fromPolicy` `SnapshotPolicy` is missing (the restore parks `Pending`/`RestoreReferentMissing`), but it does still open for a `snapshotRef` whose `Snapshot` row hasn't appeared yet, so `onMissingSnapshot` can fire for a ref that never does. See [Restores → `waitTimeout`](../../restores.md#waittimeout--wait-before-giving-up). ### `mover` diff --git a/docs/restores.md b/docs/restores.md index e22b6a0f..ea05b535 100644 --- a/docs/restores.md +++ b/docs/restores.md @@ -197,7 +197,9 @@ The empty-volume path applies to `fromPolicy`/`identity` sources on **every** ba `waitTimeout` (a Go-style duration, e.g. `5m`) opens a grace window during which "no matching snapshot yet" means *wait and re-check* instead of giving up. `onMissingSnapshot` applies only once the window closes. Use it when the Restore may be applied before the thing that produces its snapshot — a schedule about to fire, a GitOps apply ordering, a populator claim racing the first backup. -The window opens when the restore can first actually **proceed**, not when you applied it: Kopiur stamps that instant into `status.waitStartedAt` and measures `waitTimeout` from there. Two things have to be true — the restore's repository has reached `Ready`, once the controller can see it (the gate falls through unverified if the referenced `Repository`/`SnapshotPolicy` object doesn't exist yet), and (for `target.populator`) a PVC already claims the Restore via `dataSourceRef`. Until then the clock has not started. +The window opens when the restore can first actually **proceed**, not when you applied it: Kopiur stamps that instant into `status.waitStartedAt` and measures `waitTimeout` from there. Two things have to be true — the restore's repository has reached `Ready`, and (for `target.populator`) a PVC already claims the Restore via `dataSourceRef`. Until then the clock has not started. + +The window also does **not** open while the objects Kopiur reads the repository from are missing: if the referenced `Repository` object, or the `SnapshotPolicy` a `fromPolicy` source names, doesn't exist yet, the restore parks in `Pending` with `ReferentAvailable=False reason=RestoreReferentMissing` and `status.waitStartedAt` stays unstamped (it re-checks every 15 s, so it un-parks on its own once you apply the missing object). Two cases deliberately **do** open the window unverified: a `snapshotRef` whose `Snapshot` row doesn't exist yet — that row is precisely what the window is waiting for, and `onMissingSnapshot` has to be able to fire for a ref that never appears — and a restore whose mover Job has already launched, which keeps the deadline it was dispatched with. /// warning | The window is not measured from `metadata.creationTimestamp` diff --git a/docs/scenarios/dr-with-replicated-repository.md b/docs/scenarios/dr-with-replicated-repository.md index 56dc3b55..2e32ef19 100644 --- a/docs/scenarios/dr-with-replicated-repository.md +++ b/docs/scenarios/dr-with-replicated-repository.md @@ -234,7 +234,10 @@ and re-anchors the next cron slot, which is what you want mid-incident. whole time waiting for the repository — and its `waitTimeout` window opens at that moment, not at creation, so a long seed does not spend it (see [Restores → `waitTimeout`](../restores.md#waittimeout--wait-before-giving-up) - and `status.waitStartedAt`). + and `status.waitStartedAt`). That holds from the very first second of the + bring-up: while the `Repository` (or the `SnapshotPolicy` a `fromPolicy` + restore names) has not been applied yet, the `Restore` parks with + `RestoreReferentMissing` and the window is not open either. ## Observability diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 9f9f0f73..b379d1a9 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -434,6 +434,7 @@ $ kubectl describe restore -n | PVC stuck `Pending` (populator) | Volume-populator handshake not completing. | Need Kubernetes ≥ 1.24; install `volume-data-source-validator` to see the real event. See [Restores → deploy-or-restore](restores.md#deploy-or-restore-gitops). | | `prime-*` PVCs piling up, `Bound`, mounted by nothing | ≤ 0.7.x: a populator `Restore` re-created over an **already-bound** claim restored into a prime PVC that could never be adopted, and leaked it — one full copy of the data per re-created `Restore`. | Upgrade: Kopiur now completes that case as a no-op (`Ready=True reason=TargetAlreadyBound`) and reaps the orphans (`OrphanedPrimePvcReaped` event). List them with `kubectl get pvc -A -l kopiur.home-operations.com/op=restore-populate`; any whose claiming PVC is gone are collected with their `Restore`. | | Populator `Restore` says `TargetAlreadyBound` and restores nothing | Working as intended: its claiming PVC is already **Bound**, and a volume populator can only fill an **unbound** claim. | To really restore into it, delete the **PVC** (keep its `dataSourceRef`) and let it be re-created. Deleting the `Restore` just re-triggers the no-op. | +| Stuck `Pending`, `ReferentAvailable=False reason=RestoreReferentMissing` | The object Kopiur reads the restore's repository from doesn't exist: the referenced `Repository`/`ClusterRepository`, or the `SnapshotPolicy` a `fromPolicy` source names. Kopiur can't verify the backend, so it parks and — deliberately — does **not** start the `waitTimeout` window (`status.waitStartedAt` stays unset). | Create the object named in the condition message (or repoint the `Restore`). The park re-checks every 15 s and clears itself. A `snapshotRef` whose `Snapshot` row is missing is a *different* case and does not park — it waits out `waitTimeout` and then applies `onMissingSnapshot`. | | `identity` source rejected at admission | `source.identity` requires an explicit `spec.repository`. | Add `spec.repository`. | | Stuck `Pending`, `MoverPermitted=False` | The restore mover requests an elevated context (root / `privilegedMode`, container- or pod-level, possibly inherited). | Annotate the namespace (above), or drop the elevation from `spec.mover`. | | Restored files **owned by `65532`** / unreadable by the app | The mover wrote them as its own UID (no `mover.securityContext`). | Set `Restore.spec.mover.securityContext.runAsUser/Group` to the app's UID (or `inheritSecurityContextFrom`). | From dd2378214eb936b466a7b03813b50705bf9ad3ef Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 10:41:30 -0700 Subject: [PATCH 03/15] fix(restore): carry the cleared referent condition through the pass (#393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The readiness gate's clear wrote `ReferentAvailable=True` and then let the rest of the pass keep using the reconcile-start copy. The four gate parks in `run_restore_mover` (MissingCaBundle, MissingServiceAccount, PrivilegedMover, MissingCredentials) rebuild `conditions` from that copy and patch it UNCONDITIONALLY, so they wrote the stale `False` straight back: two resourceVersion-bumping writes per iteration, each waking the watch and re-enqueueing, collapsing error-policy backoff into a tight loop for as long as the missing Secret/SA/ConfigMap went unfixed — exactly the GitOps bring-up this feature serves. Those parks are byte-identical no-ops today only because nothing wrote conditions ahead of them. `RepositoryGate::Proceed` now carries the `Cow` the rest of the pass must build on (borrowed when nothing was cleared, so the common path pays no clone), and `reconcile_inner` rebinds `restore` to it — the caller cannot get a `&Restore` to continue with without taking the carried one. The cleared-but- stale combination has one construction site, the pure and total `carried_after_clear`, so it is unwritable elsewhere. --- crates/controller/src/restore/mod.rs | 65 ++++++++++++++++----- crates/controller/src/restore/plan.rs | 46 +++++++++++++++ crates/controller/src/restore/tests.rs | 81 ++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 14 deletions(-) diff --git a/crates/controller/src/restore/mod.rs b/crates/controller/src/restore/mod.rs index ab30c697..78ddfef1 100644 --- a/crates/controller/src/restore/mod.rs +++ b/crates/controller/src/restore/mod.rs @@ -267,12 +267,25 @@ async fn reconcile_inner(restore: &Restore, ctx: &Context) -> Result { // `ensure_wait_anchor` below — which is the whole of #393: a restore whose // repository could not even be looked up must not open (and start spending) // its `waitTimeout` window. - match gate_on_repository_readiness(ctx, restore, &api, &namespace, &name).await? { - RepositoryGate::Proceed => {} + // + // The gate hands back THE `Restore` the rest of this pass must use, and + // `restore` is rebound to it: unchanged in the ordinary case, and a carried + // copy holding the cleared conditions when the gate cleared a stale + // `ReferentAvailable=False`. Every condition writer below rebuilds the array + // from this `restore`, and four of them (the gate parks in + // `run_restore_mover`) patch UNCONDITIONALLY — continuing from the + // reconcile-start copy would re-write the `False` just cleared and alternate + // the two writes on every requeue, forever. See `restore_with_conditions`. + let gated; + let restore = match gate_on_repository_readiness(ctx, restore, &api, &namespace, &name).await? { + RepositoryGate::Proceed(carried) => { + gated = carried; + gated.as_ref() + } RepositoryGate::Held(action) | RepositoryGate::Undetermined(action) => { return Ok(action); } - } + }; // The claiming PVC, looked up ONCE per pass and shared by everything below that needs // it: the wait-window anchor (a populator with no claim cannot proceed, so its window @@ -3440,10 +3453,20 @@ async fn restore_repository_ref( /// `_ =>` arm — a fourth outcome must force its caller to decide, at compile /// time, whether it proceeds or parks. COMPILER-ONLY guard: `cargo xtask /// check-phases` scans only the `*Phase` enums in `kopiur-api`. -enum RepositoryGate { +enum RepositoryGate<'a> { /// The gate does not hold this restore: reconcile continues (and the /// `waitTimeout` window may open). - Proceed, + /// + /// It carries **the `Restore` the rest of the pass must use** — borrowed + /// unchanged in the ordinary case, and an owned copy holding the cleared + /// conditions when this pass cleared a stale `ReferentAvailable=False` + /// ([`restore_with_conditions`]). Returning the object rather than a + /// "…and also remember to apply this" side value is deliberate: the caller + /// cannot obtain a `&Restore` to continue with WITHOUT taking the carried + /// one, so the clear cannot be silently dropped by a later edit. Dropping it + /// would let the unconditional downstream gate parks re-write the stale + /// `False` and alternate two writes forever — see [`restore_with_conditions`]. + Proceed(std::borrow::Cow<'a, Restore>), /// VERIFIED not ready: the repository object exists and its phase is not /// `Ready` (the backend is unreachable). Park + requeue. Held(Action), @@ -3499,13 +3522,13 @@ enum RepositoryGate { /// crash window: a Job created moments before the controller died — before the /// `Restoring` patch landed — is re-gated as `Pending`/`Resolving`; harmless, the /// Job runs to terminal on its own and is observed once the gate opens.) -async fn gate_on_repository_readiness( +async fn gate_on_repository_readiness<'a>( ctx: &Context, - restore: &Restore, + restore: &'a Restore, api: &Api, namespace: &str, name: &str, -) -> Result { +) -> Result> { if !restore_awaiting_launch(restore.status.as_ref().and_then(|s| s.phase.as_ref())) { return proceed_past_gate(restore, api, name).await; } @@ -3602,15 +3625,29 @@ fn repository_referent_namespace<'a>(rref: &'a RepositoryRef, base_ns: &'a str) /// nothing and the healthy wire never grows the condition. Without it the /// registry row — which is deliberately age-independent — would keep reporting a /// restore that proceeded hours ago as blocked. -async fn proceed_past_gate( - restore: &Restore, +/// +/// The cleared array is not just written to the server: the returned +/// [`RepositoryGate::Proceed`] carries a `Restore` holding it, which is the +/// object the rest of the pass must build on. The downstream gate parks rebuild +/// `conditions` from the `Restore` they are handed and patch it +/// UNCONDITIONALLY, so continuing from the reconcile-start copy would re-write +/// the `False` this just cleared — see [`restore_with_conditions`] for the write +/// loop that prevents. Nothing was cleared ⇒ the original is borrowed, no clone. +async fn proceed_past_gate<'a>( + restore: &'a Restore, api: &Api, name: &str, -) -> Result { - if let Some(conditions) = cleared_referent_conditions(restore) { +) -> Result> { + let cleared = cleared_referent_conditions(restore); + if let Some(conditions) = &cleared { io::patch_status(api, name, serde_json::json!({ "conditions": conditions })).await?; } - Ok(RepositoryGate::Proceed) + // One construction site, pure and total: a cleared pass CANNOT continue from + // the stale object, because that combination is unwritable here. + Ok(RepositoryGate::Proceed(carried_after_clear( + restore, + cleared.as_deref(), + ))) } /// Park a restore whose repository referent does not exist (issue #393): @@ -3632,7 +3669,7 @@ async fn park_on_missing_referent( kind: &str, referent_namespace: Option<&str>, referent: &str, -) -> Result { +) -> Result> { let msg = referent_missing_restore_message(kind, referent_namespace, referent); let conditions = io::upsert_gate( &existing_conditions(restore), diff --git a/crates/controller/src/restore/plan.rs b/crates/controller/src/restore/plan.rs index 7904210b..9c2f2a10 100644 --- a/crates/controller/src/restore/plan.rs +++ b/crates/controller/src/restore/plan.rs @@ -562,6 +562,52 @@ pub(super) fn cleared_referent_conditions(restore: &Restore) -> Option) -> Restore { + let mut carried = restore.clone(); + let mut status = carried.status.take().unwrap_or_default(); + status.conditions = conditions; + carried.status = Some(status); + carried +} + +/// The `Restore` the rest of the pass must build on, given what +/// [`cleared_referent_conditions`] produced: the original borrowed when nothing +/// was cleared (the overwhelmingly common path — no clone), an owned copy +/// carrying the cleared conditions when something was. Pure and TOTAL. +/// +/// The single construction site for the readiness gate's proceed payload, so the +/// "cleared but continued from the stale copy" combination — the one that starts +/// the alternating-write loop described on [`restore_with_conditions`] — has no +/// place to be written. Both arms are unit-asserted. +pub(super) fn carried_after_clear<'a>( + restore: &'a Restore, + cleared: Option<&[Condition]>, +) -> std::borrow::Cow<'a, Restore> { + match cleared { + None => std::borrow::Cow::Borrowed(restore), + Some(conditions) => { + std::borrow::Cow::Owned(restore_with_conditions(restore, conditions.to_vec())) + } + } +} /// Where the `waitTimeout` window is anchored: `status.waitStartedAt` once /// [`super::ensure_wait_anchor`] has stamped it, else `created_epoch`. /// diff --git a/crates/controller/src/restore/tests.rs b/crates/controller/src/restore/tests.rs index 5bb8a21b..f301cb66 100644 --- a/crates/controller/src/restore/tests.rs +++ b/crates/controller/src/restore/tests.rs @@ -293,6 +293,87 @@ fn the_referent_gate_condition_clears_only_when_it_is_stale() { assert!(cleared_referent_conditions(&healthy).is_none()); } +/// The write-loop guard: a pass that clears the referent gate must carry the +/// cleared conditions forward, or the UNCONDITIONAL gate parks downstream +/// (`run_restore_mover`'s `MissingCaBundle`/`MissingServiceAccount`/ +/// `PrivilegedMover`/`MissingCredentials` writes) rebuild the array from the +/// reconcile-start copy and put `ReferentAvailable=False` straight back. +/// +/// That alternation is not cosmetic: both writes bump `resourceVersion`, each +/// wakes the watch and re-enqueues immediately, so the pair repeats forever (two +/// writes + an Event per iteration) for as long as the Secret/SA/ConfigMap is +/// missing — precisely the GitOps bring-up this feature serves. Those parks are +/// byte-identical no-ops today only because nothing writes conditions ahead of +/// them; this test pins the property that keeps that true. +/// +/// The CALLER half of the contract is enforced by the compiler rather than here: +/// `RepositoryGate::Proceed` carries the `Cow` to continue with, so +/// `reconcile_inner` cannot get a `&Restore` for the rest of the pass without +/// taking the carried one. This test pins the mechanism that carrying provides. +#[test] +fn a_cleared_referent_condition_survives_a_downstream_gate_park() { + use crate::consts::RESTORE_REFERENT_AVAILABLE_CONDITION; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; + let status_of = |conds: &[Condition]| { + conds + .iter() + .find(|c| c.type_ == RESTORE_REFERENT_AVAILABLE_CONDITION) + .map(|c| c.status.clone()) + }; + + let parked = restore_with_condition(RESTORE_REFERENT_AVAILABLE_CONDITION, "False"); + let cleared = cleared_referent_conditions(&parked).expect("a stale park clears"); + // Built through the one construction site the gate uses, so this exercises + // the real seam rather than a re-implementation of it. + let carried = carried_after_clear(&parked, Some(&cleared)).into_owned(); + + // What a downstream gate park computes from the CARRIED copy: the clear holds. + let after_park = io::upsert_gate( + &existing_conditions(&carried), + &kopiur_api::gates::MISSING_CREDENTIALS_GATE, + "the credentials Secret is not in the mover namespace", + carried.metadata.generation, + ); + assert_eq!( + status_of(&after_park).as_deref(), + Some("True"), + "carrying the cleared conditions forward must survive an unconditional \ + downstream gate park: {after_park:?}" + ); + // ...and the same park computed from the reconcile-start copy is the write + // that used to alternate with the clear. Asserted so this test fails loudly + // if the clobber ever stops being reproducible (i.e. the guard goes vacuous). + let clobbered = io::upsert_gate( + &existing_conditions(&parked), + &kopiur_api::gates::MISSING_CREDENTIALS_GATE, + "the credentials Secret is not in the mover namespace", + parked.metadata.generation, + ); + assert_eq!( + status_of(&clobbered).as_deref(), + Some("False"), + "the reconcile-start copy still carries the stale park — that is the write \ + the carried copy exists to prevent" + ); + + // The carried copy is also a fixed point: a second pass over it clears + // nothing, so the loop cannot restart from the other side. + assert!(cleared_referent_conditions(&carried).is_none()); + // Carrying preserves everything else about the object (only conditions move). + assert_eq!(carried.metadata.generation, parked.metadata.generation); + assert_eq!(carried.spec, parked.spec); + + // Both arms of the construction site: nothing cleared ⇒ the original is + // BORROWED (the common path pays no clone and changes nothing). + let untouched = restore_with_condition("Resolved", "True"); + let same = carried_after_clear( + &untouched, + cleared_referent_conditions(&untouched).as_deref(), + ); + assert!(matches!(same, std::borrow::Cow::Borrowed(_))); + assert_eq!(same.as_ref().status, untouched.status); +} + #[test] fn populator_state_depends_on_target_variant() { use kopiur_api::PopulatorTarget; From eec7d04d2e1c524bcd891be22c417888e8d83c93 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 11:10:49 -0700 Subject: [PATCH 04/15] feat(kopia): repository set-client read-only/read-write flip (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `KopiaClient::repository_set_client_read_only(bool)` → `kopia repository set-client --read-only|--read-write`, the primitive a read-only-connected source config needs to run a write-requiring verb: flip read-write, run the verb, flip back. Covered by a shim-binary unit test (exact argv, both polarities) and a new `integration_set_client_throttle` test against the mise-pinned kopia 0.23.1, which is the empirical gate for the dependent throttle milestones. It pins: * set-client works on a `--readonly` connection and only rewrites the local config's `readonly` key; * the flip is real, not cosmetic — `repository set-parameters` fails while read-only, succeeds inside the window, and fails again after the flip back; * `throttle set` does NOT actually need the window. It succeeds directly on a read-only connection (it never puts a blob), contradicting the design premise this milestone was written to confirm. The flip is defensive there, not required; if a future kopia changes that, the test fails loudly; * limits persist as `throttlingLimits` and survive the flip back; * the load-bearing claim: `snapshot migrate --source-config` — which has no speed flags of its own — genuinely honors those config-persisted limits. Two measured caveats the dependent milestones must design around, both documented at the test's calibration table: throttling bites only on traffic that reaches the backend (a warm source cache makes even a 1 KB/s cap invisible), and kopia's throttler costs far more than its nominal rate implies (28 KiB takes 14 s at a 2 MB/s cap; caps above ~10 MB/s do not bind at all). --- crates/kopia/src/client/mod.rs | 34 ++ crates/kopia/src/client/tests.rs | 52 +++ .../tests/integration_set_client_throttle.rs | 367 ++++++++++++++++++ 3 files changed, 453 insertions(+) create mode 100644 crates/kopia/tests/integration_set_client_throttle.rs diff --git a/crates/kopia/src/client/mod.rs b/crates/kopia/src/client/mod.rs index 79d7406e..894d5094 100644 --- a/crates/kopia/src/client/mod.rs +++ b/crates/kopia/src/client/mod.rs @@ -1415,6 +1415,40 @@ impl KopiaClient { self.run_ok(&args).await.map(|_| ()) } + /// Flip the CONNECTED repository's client-side read-only bit + /// (`kopia repository set-client --read-only` / `--read-write`), issue #374. + /// + /// Three properties, all pinned against kopia 0.23.1 by + /// `crates/kopia/tests/integration_set_client_throttle.rs`: + /// + /// - **Works on a read-only connection.** `set-client` registers kopia's + /// `repositoryReaderAction`, so it opens fine against a config connected with + /// `--readonly` — unlike [`Self::repository_set_parameters`], which needs a real + /// blob write and hard-errors there. + /// - **The bit lives in the local config, not the repository.** `read_only: true` + /// writes `"readonly": true` into the `KOPIA_CONFIG_PATH` JSON; `false` removes the + /// key. Nothing is written to the backend, so this never mutates shared state and + /// needs no maintenance ownership/lease. + /// - **It is the "flip window" primitive** for a read-only-connected config that must + /// run a write-requiring verb: flip read-write, run the verb, flip back. Note that + /// [`Self::repository_throttle_set`] does *not* need this window — M3a measured it + /// succeeding directly on a `--readonly` connection (it only rewrites the local + /// config's `throttlingLimits`), so the flip is defensive there, not required. + pub async fn repository_set_client_read_only(&self, read_only: bool) -> Result<(), KopiaError> { + let mode = if read_only { + "--read-only" + } else { + "--read-write" + }; + self.run_ok(&[ + "repository".to_string(), + "set-client".to_string(), + mode.to_string(), + ]) + .await + .map(|_| ()) + } + /// Rewrite mutable repository parameters on the CONNECTED repository /// (`kopia repository set-parameters [flags]`), issue #258. No-op when nothing is set. /// diff --git a/crates/kopia/src/client/tests.rs b/crates/kopia/src/client/tests.rs index 2b59a700..54c0d50f 100644 --- a/crates/kopia/src/client/tests.rs +++ b/crates/kopia/src/client/tests.rs @@ -1476,6 +1476,58 @@ fn throttle_args_builds_per_second_flags_and_empties() { assert!(ThrottleArgs::default().args().is_empty()); } +#[cfg(unix)] +#[tokio::test] +async fn set_client_read_only_emits_the_matching_polarity_flag() { + use std::io::Write; + use std::os::unix::fs::PermissionsExt; + + // `repository_set_client_read_only` builds argv only — no parsing — so the + // assertion has to be on what the process is actually SPAWNED with. A shim + // standing in for `kopia` appends its argv to a file (issue #374). + let dir = std::env::temp_dir().join(format!("kopiur-setclient-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let shim = dir.join("kopia"); + let out = dir.join("argv.out"); + { + let mut f = std::fs::File::create(&shim).unwrap(); + write!( + f, + "#!/bin/sh\necho \"$@\" >> \"$KOPIUR_SETCLIENT_OUT\"\nexit 0\n" + ) + .unwrap(); + std::fs::set_permissions(&shim, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + + let client = KopiaClient::builder() + .binary(&shim) + .env("KOPIUR_SETCLIENT_OUT", out.to_str().unwrap()) + .build(); + + // Both polarities: `true` parks the config read-only, `false` opens the + // flip window. kopia spells these as two distinct flags, not a negation of + // one — passing the wrong one silently leaves the bit where it was. + client.repository_set_client_read_only(false).await.unwrap(); + client.repository_set_client_read_only(true).await.unwrap(); + + let lines: Vec = std::fs::read_to_string(&out) + .unwrap() + .lines() + .map(|l| l.trim().to_string()) + .collect(); + // `--no-auto-maintenance` is the unconditional common arg every invocation + // carries; asserting the whole argv keeps it honest rather than substring-matching. + assert_eq!( + lines, + vec![ + "repository set-client --read-write --no-auto-maintenance", + "repository set-client --read-only --no-auto-maintenance", + ] + ); + + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn server_start_args_password_mode() { let spec = ServerStartSpec { diff --git a/crates/kopia/tests/integration_set_client_throttle.rs b/crates/kopia/tests/integration_set_client_throttle.rs new file mode 100644 index 00000000..1d92dba2 --- /dev/null +++ b/crates/kopia/tests/integration_set_client_throttle.rs @@ -0,0 +1,367 @@ +//! Real-kopia integration test for the read-only/read-write client flip and the +//! throttle limits it brackets (issue #374, milestone M3a). +//! +//! Gated behind the `integration` feature and `#[ignore]` by default so the +//! hermetic `cargo test` never invokes the real binary (the `integration` +//! feature lifts the `#[ignore]`, so it is not needed on the command line): +//! +//! ```text +//! cargo test -p kopiur-kopia --features integration --test integration_set_client_throttle +//! ``` +//! +//! The kopia binary comes from mise's PATH — the SAME pin (`v0.23.1`) the mover +//! image ships — so a Renovate bump moves both and this test fails loudly on +//! semantic drift. That is the point: everything asserted here is a premise the +//! replication/seed movers' throttle handling rests on, and none of it is +//! documented by kopia. +//! +//! What it pins against kopia 0.23.1: +//! +//! * `repository set-client --read-only` / `--read-write` works on a config +//! connected `--readonly`, and only rewrites the LOCAL config's `readonly` key +//! (`true` present / key absent when read-write). Nothing reaches the backend. +//! * The flip is real, not cosmetic: `repository set-parameters` — a genuinely +//! write-requiring verb — fails while read-only, succeeds inside the flip +//! window, and fails again after the flip back. +//! * **`repository throttle set` does NOT need the flip window.** It succeeds +//! directly on a `--readonly` connection — contradicting the design premise +//! this milestone was written to confirm (kopia registers it as a write action, +//! but it never puts a blob, so read-only storage never rejects it). The flip +//! is therefore defensive for throttling, not required. If a future kopia turns +//! it into a real write, step (1b) below fails loudly and the flip becomes +//! mandatory. +//! * Limits persist as `throttlingLimits` in the config JSON and survive the flip +//! back to read-only. +//! * The load-bearing claim for the source side: `snapshot migrate +//! --source-config ` genuinely honors those config-persisted +//! limits. `snapshot migrate` has no speed flags at all, so the persisted +//! config is the only lever there is — if this were false, the whole +//! throttle-the-source design would be inert. +//! * The caveat that comes with it: throttling only bites on traffic that +//! actually reaches the backend. A warm source cache makes even a 1 KB/s cap +//! invisible (the migrate serves from local disk and finishes in ~200 ms), so +//! step (6) drops the cache first — which is also what a fresh mover pod looks +//! like. + +#![cfg(unix)] + +use std::collections::BTreeMap; +use std::time::Instant; + +use kopiur_kopia::client::SetParametersArgs; +use kopiur_kopia::{ + ConnectOptions, ConnectSpec, KopiaClient, MigratePolicies, MigrateSources, + SnapshotMigrateOptions, ThrottleArgs, +}; + +const SRC_PASSWORD: &str = "setclient-pass-a"; +const DEST_PASSWORD: &str = "setclient-pass-b"; +const IDENTITY: &str = "throttleuser@throttlehost:/data"; + +/// The cap used for the migrate timing probe, in bytes/sec. +/// +/// Chosen from a measured curve, not guessed. Against this fixture (an 8 KiB +/// incompressible payload → a ~28 KiB source repository) on the pinned binary, +/// with a COLD source cache, `snapshot migrate` takes: +/// +/// | cap B/s | elapsed | +/// |-----------|---------| +/// | unlimited | 0.21 s | +/// | 20 M | 0.21 s | +/// | 10 M | 0.21 s | +/// | 5 M | 2.2 s | +/// | 2 M | 14.2 s | +/// | 1 M | 34.2 s | +/// +/// Note how brutally non-linear that is: kopia's throttler costs far more than +/// the nominal rate implies (28 KiB "should" take 14 ms at 2 MB/s, and takes +/// 14 s), and caps at or above ~10 MB/s do not bind at all here. 2 MB/s is the +/// sweet spot — a ~14 s run, ~5x above the assertion floor and ~70x above the +/// unthrottled time, while a 1 MB/s cap would drag the test to 35 s and lower +/// caps to hours. +const MIGRATE_CAP_BYTES_PER_SEC: i64 = 2_000_000; + +/// The wall-clock floor a throttled migrate must clear, well under the ~14 s +/// measured at [`MIGRATE_CAP_BYTES_PER_SEC`] so a loaded CI box cannot make it +/// flake. The failure it guards is "the throttle was ignored entirely", which +/// lands at ~0.2 s — two orders of magnitude below this floor, so the verdict is +/// never ambiguous. Throttling can only ever make a run SLOWER, so a faster +/// machine cannot push a genuinely throttled run below the bound. +const MIGRATE_MIN_ELAPSED_SECS: u64 = 3; + +fn config_path(config_dir: &std::path::Path) -> String { + config_dir.join("repository.config").display().to_string() +} + +/// A client whose kopia state is isolated inside `config_dir`, same pattern as +/// `integration_migrate.rs`. +fn isolated_client(config_dir: &std::path::Path, password: &str) -> KopiaClient { + KopiaClient::builder() + .binary("kopia") + .env("KOPIA_PASSWORD", password) + .env("KOPIA_CONFIG_PATH", config_path(config_dir)) + .env( + "KOPIA_CACHE_DIRECTORY", + config_dir.join("cache").display().to_string(), + ) + .env( + "KOPIA_LOG_DIR", + config_dir.join("logs").display().to_string(), + ) + .env("KOPIA_CHECK_FOR_UPDATES", "false") + .build() +} + +/// A DESTINATION client able to run `snapshot migrate`: it must NOT pin +/// `KOPIA_CACHE_DIRECTORY`, or migrate's source open reads the destination's +/// cached format blob and fails with "invalid repository password" (proven by +/// `integration_migrate.rs`). Cache isolation comes from `XDG_CACHE_HOME`. +fn dest_client(config_dir: &std::path::Path, password: &str) -> KopiaClient { + KopiaClient::builder() + .binary("kopia") + .env("KOPIA_PASSWORD", password) + .env("KOPIA_CONFIG_PATH", config_path(config_dir)) + .env( + "XDG_CACHE_HOME", + config_dir.join("xdg-cache").display().to_string(), + ) + .env_remove("KOPIA_CACHE_DIRECTORY") + .env( + "KOPIA_LOG_DIR", + config_dir.join("logs").display().to_string(), + ) + .env("KOPIA_CHECK_FOR_UPDATES", "false") + .build() +} + +/// The parsed kopia config JSON for a client's config directory. The read-only +/// bit and the throttling limits both live here — this test asserts on the file +/// because that persistence IS the mechanism the movers depend on. +fn read_config(config_dir: &std::path::Path) -> serde_json::Value { + let raw = std::fs::read_to_string(config_path(config_dir)).expect("read kopia config JSON"); + serde_json::from_str(&raw).expect("kopia config is JSON") +} + +/// A genuinely write-requiring verb, used to prove the read-only bit is really +/// in force (not just recorded). `repository set-parameters` must reach the +/// backend to rewrite `kopia.blobcfg`, so a read-only connection hard-errors. +/// The value varies per call so a rejected attempt can never be confused with +/// kopia skipping a no-change write. +async fn try_set_epoch_advance_on_count( + client: &KopiaClient, + count: i64, +) -> Result<(), kopiur_kopia::KopiaError> { + client + .repository_set_parameters(&SetParametersArgs { + epoch_advance_on_count: Some(count), + ..Default::default() + }) + .await +} + +#[tokio::test] +#[cfg_attr(not(feature = "integration"), ignore)] +async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() { + let src_repo = tempfile::tempdir().unwrap(); + let src_admin_cfg = tempfile::tempdir().unwrap(); + let src_ro_cfg = tempfile::tempdir().unwrap(); + let data_dir = tempfile::tempdir().unwrap(); + let dest_repo = tempfile::tempdir().unwrap(); + let dest_cfg = tempfile::tempdir().unwrap(); + + // ---- fixture: a source repository holding one snapshot of 8 KiB of + // deliberately incompressible-but-DETERMINISTIC bytes (a Knuth-multiplicative + // scramble). Deterministic keeps the calibrated timings in + // `MIGRATE_CAP_BYTES_PER_SEC` reproducible run to run; incompressible stops a + // future compression default from shrinking the payload and quietly robbing + // the timing probe of the bytes it measures. + let payload: Vec = (0..8192u32) + .map(|i| (i.wrapping_mul(2654435761) >> 13) as u8) + .collect(); + std::fs::write(data_dir.path().join("blob.bin"), &payload).unwrap(); + let src_admin = isolated_client(src_admin_cfg.path(), SRC_PASSWORD); + src_admin + .repository_create( + &ConnectSpec::Filesystem { + path: src_repo.path().to_path_buf(), + }, + Default::default(), + &Default::default(), + ) + .await + .expect("source repository create"); + src_admin + .snapshot_create( + data_dir.path().to_str().unwrap(), + &BTreeMap::new(), + Some(IDENTITY), + ) + .await + .expect("source snapshot create"); + + let dest = dest_client(dest_cfg.path(), DEST_PASSWORD); + dest.repository_create( + &ConnectSpec::Filesystem { + path: dest_repo.path().to_path_buf(), + }, + Default::default(), + &Default::default(), + ) + .await + .expect("destination repository create"); + + // ---- the connect the replication/seed movers perform for their SOURCE: + // a separate config, read-only, credentials persisted beside it. + let src_ro = isolated_client(src_ro_cfg.path(), SRC_PASSWORD); + src_ro + .repository_connect_with( + &ConnectSpec::Filesystem { + path: src_repo.path().to_path_buf(), + }, + Default::default(), + ConnectOptions { + readonly: true, + persist_credentials: true, + }, + ) + .await + .expect("source reconnect --readonly --persist-credentials"); + assert_eq!( + read_config(src_ro_cfg.path()).get("readonly"), + Some(&serde_json::Value::Bool(true)), + "a --readonly connect must record `readonly: true` in the config" + ); + + // (1) The read-only bit is REALLY in force: a verb that must write a blob + // fails. This is the control for everything below — without it, "throttle + // set works on read-only" would prove nothing. + let err = try_set_epoch_advance_on_count(&src_ro, 10) + .await + .expect_err("set-parameters must fail on a read-only connection"); + assert!( + err.to_string().contains("read-only") || err.to_string().contains("readonly"), + "expected a read-only storage rejection, got: {err}" + ); + + // (1b) THE PREMISE THIS MILESTONE WAS WRITTEN TO CHECK — and it does not + // hold. `repository throttle set` SUCCEEDS on the read-only connection: it + // only rewrites the local config's `throttlingLimits`, it never puts a blob. + // The flip window is therefore DEFENSIVE for throttling, not required. + // If a future kopia turns this into a real write action, this line fails + // loudly and the dependent milestones must adopt the flip unconditionally. + let benign = ThrottleArgs { + upload_bytes_per_second: Some(1_000_000), + download_bytes_per_second: Some(1_000_000), + ..Default::default() + }; + src_ro + .repository_throttle_set(&benign) + .await + .expect("throttle set succeeds on a READ-ONLY connection (kopia 0.23.1)"); + assert_eq!( + read_config(src_ro_cfg.path())["throttlingLimits"]["maxUploadSpeedBytesPerSecond"], + serde_json::json!(1_000_000), + "throttle set must persist into the local config even when read-only" + ); + + // (2) Open the flip window and prove it is real: the same write-requiring + // verb that failed above now SUCCEEDS. + src_ro + .repository_set_client_read_only(false) + .await + .expect("set-client --read-write"); + assert!( + read_config(src_ro_cfg.path()).get("readonly").is_none(), + "read-write must clear the config's `readonly` key" + ); + try_set_epoch_advance_on_count(&src_ro, 10) + .await + .expect("set-parameters must succeed inside the read-write flip window"); + + // Apply the cap the timing probe measures, inside the window — this is the + // exact flip → throttle set → flip-back sequence the dependent milestones use. + src_ro + .repository_throttle_set(&ThrottleArgs { + upload_bytes_per_second: Some(MIGRATE_CAP_BYTES_PER_SEC), + download_bytes_per_second: Some(MIGRATE_CAP_BYTES_PER_SEC), + ..Default::default() + }) + .await + .expect("throttle set inside the flip window"); + + // (3) Flip back, and (4) assert read-only is genuinely restored — config bit + // AND the behavioral probe, with a DIFFERENT epoch count (12, not the 10 just + // committed) so the rejection cannot be a no-change short-circuit. + src_ro + .repository_set_client_read_only(true) + .await + .expect("set-client --read-only"); + assert_eq!( + read_config(src_ro_cfg.path()).get("readonly"), + Some(&serde_json::Value::Bool(true)), + "the flip back must restore `readonly: true`" + ); + let err = try_set_epoch_advance_on_count(&src_ro, 12) + .await + .expect_err("set-parameters must fail again after the flip back"); + assert!( + err.to_string().contains("read-only") || err.to_string().contains("readonly"), + "expected a read-only storage rejection after the flip back, got: {err}" + ); + + // (5) The limits survived the whole flip sequence, in the config on disk. + let limits = read_config(src_ro_cfg.path())["throttlingLimits"].clone(); + assert_eq!( + limits["maxUploadSpeedBytesPerSecond"], + serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), + "upload cap must survive the flip back: {limits}" + ); + assert_eq!( + limits["maxDownloadSpeedBytesPerSecond"], + serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), + "download cap must survive the flip back: {limits}" + ); + + // (6) THE LOAD-BEARING CLAIM: `snapshot migrate` reopens the source config + // and honors its persisted limits. `snapshot migrate` exposes no speed + // flags, so if this were false there would be no way to throttle a + // seed-migrate at all. + // + // The source cache must be COLD first. kopia only throttles traffic that + // actually reaches the backend, and the migrating process reads the source + // through the cache directory recorded in the SOURCE config + // (`caching.cacheDirectory`, relative to the config file → `/cache`). + // The earlier steps warmed it; leaving it warm makes the migrate serve + // everything from local disk and finish unthrottled in ~200ms — which is + // also the honest caveat for the dependent milestones: throttling only + // bites on real backend traffic, and a mover pod's cache is cold by + // construction, which is what this models. + std::fs::remove_dir_all(src_ro_cfg.path().join("cache")).expect("drop the source cache"); + + let started = Instant::now(); + dest.snapshot_migrate(&SnapshotMigrateOptions { + source_config_path: config_path(src_ro_cfg.path()), + sources: MigrateSources::All, + latest_only: false, + parallel: None, + policies: MigratePolicies::None, + }) + .await + .expect("throttled snapshot migrate"); + let elapsed = started.elapsed(); + + let migrated = dest.snapshot_list_all().await.expect("dest list --all"); + assert_eq!( + migrated.len(), + 1, + "the snapshot must actually arrive: {migrated:?}" + ); + assert_eq!(migrated[0].source.identity(), IDENTITY); + assert!( + elapsed.as_secs() >= MIGRATE_MIN_ELAPSED_SECS, + "migrate finished in {elapsed:?} with the source config capped at \ + {MIGRATE_CAP_BYTES_PER_SEC} B/s — it must have IGNORED the persisted \ + throttle (an unthrottled migrate of this fixture is sub-second). The \ + entire source-side throttle design rests on migrate honoring it." + ); +} From 7c8988766cb10cd8caacf43191705be6e9474229 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 11:17:51 -0700 Subject: [PATCH 05/15] fix(restore): box the carried Restore so the gate enum stays small (#393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dd23782 gave `RepositoryGate::Proceed` a `Cow<'a, Restore>`. A Cow stores its owned variant INLINE, so the whole enum grew to a `Restore` while its sibling variants are one `Action` each — `clippy::large_enum_variant`, and `mise run clippy` (--workspace --all-targets -D warnings) exited 101. Clippy's suggested `Box>` would allocate on the BORROWED path too, and that path is the common one this design deliberately keeps free. Instead `CarriedRestore` boxes only its owned arm: Unchanged(&'a Restore) // no clone, no allocation Cleared(Box) // one Box atop a clone already being made so the gate enum is pointer-sized in both arms. Semantics are unchanged: the no-clone fast path, the 'caller cannot get a &Restore without taking the carried one' property, and the single total construction site all survive, and `get()` is an exhaustive match so a third carrier shape must decide which object the pass continues with. The guard test now pins the carrier variant in both arms; its mutation-detection property was re-verified after the refactor. --- crates/controller/src/restore/mod.rs | 11 ++++-- crates/controller/src/restore/plan.rs | 50 ++++++++++++++++++++++---- crates/controller/src/restore/tests.rs | 14 ++++++-- 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/crates/controller/src/restore/mod.rs b/crates/controller/src/restore/mod.rs index 78ddfef1..a91e605e 100644 --- a/crates/controller/src/restore/mod.rs +++ b/crates/controller/src/restore/mod.rs @@ -280,7 +280,7 @@ async fn reconcile_inner(restore: &Restore, ctx: &Context) -> Result { let restore = match gate_on_repository_readiness(ctx, restore, &api, &namespace, &name).await? { RepositoryGate::Proceed(carried) => { gated = carried; - gated.as_ref() + gated.get() } RepositoryGate::Held(action) | RepositoryGate::Undetermined(action) => { return Ok(action); @@ -3466,7 +3466,14 @@ enum RepositoryGate<'a> { /// one, so the clear cannot be silently dropped by a later edit. Dropping it /// would let the unconditional downstream gate parks re-write the stale /// `False` and alternate two writes forever — see [`restore_with_conditions`]. - Proceed(std::borrow::Cow<'a, Restore>), + /// + /// [`CarriedRestore`], not `Cow`: a `Cow`'s owned variant is stored + /// inline, and `Restore` is large enough that it would bloat every value of + /// this enum past `clippy::large_enum_variant` (denied workspace-wide) while + /// the sibling variants are one `Action` each. `CarriedRestore` boxes only + /// its owned arm, so this stays pointer-sized AND the borrowed path keeps + /// costing nothing. + Proceed(CarriedRestore<'a>), /// VERIFIED not ready: the repository object exists and its phase is not /// `Ready` (the backend is unreachable). Park + requeue. Held(Action), diff --git a/crates/controller/src/restore/plan.rs b/crates/controller/src/restore/plan.rs index 9c2f2a10..c89a3d2a 100644 --- a/crates/controller/src/restore/plan.rs +++ b/crates/controller/src/restore/plan.rs @@ -588,7 +588,43 @@ pub(super) fn restore_with_conditions(restore: &Restore, conditions: Vec`, for one +/// concrete reason: `Restore` is a large struct, so a `Cow` stores the owned +/// variant INLINE and every value of the enum carrying it (here +/// [`super::RepositoryGate`], whose other variants are one `Action` each) grows +/// to that size — `clippy::large_enum_variant`, denied workspace-wide. Boxing +/// the whole `Cow` (clippy's suggestion) would allocate on the borrowed path +/// too, which is the overwhelmingly common one and must stay free. Boxing only +/// [`Self::Cleared`] keeps the carrier pointer-sized in both arms: the unchanged +/// path is still a bare borrow with no clone and no allocation, and the cleared +/// path pays one `Box` on top of a `Restore` clone it was already making. +/// +/// [`Self::get`] is an exhaustive `match`, so a third carrier shape would have +/// to say which object the pass continues with. +#[derive(Debug)] +pub(super) enum CarriedRestore<'a> { + /// Nothing was cleared: the reconcile's own `Restore`, borrowed. + Unchanged(&'a Restore), + /// The gate cleared a stale gate condition: this copy mirrors the status + /// patch it just made, and is what every later writer must rebuild from. + Cleared(Box), +} + +impl CarriedRestore<'_> { + /// The `Restore` to continue the pass with. Exhaustive. + pub(super) fn get(&self) -> &Restore { + match self { + Self::Unchanged(restore) => restore, + Self::Cleared(restore) => restore, + } + } +} + +/// The [`CarriedRestore`] for this pass, given what /// [`cleared_referent_conditions`] produced: the original borrowed when nothing /// was cleared (the overwhelmingly common path — no clone), an owned copy /// carrying the cleared conditions when something was. Pure and TOTAL. @@ -600,14 +636,16 @@ pub(super) fn restore_with_conditions(restore: &Restore, conditions: Vec( restore: &'a Restore, cleared: Option<&[Condition]>, -) -> std::borrow::Cow<'a, Restore> { +) -> CarriedRestore<'a> { match cleared { - None => std::borrow::Cow::Borrowed(restore), - Some(conditions) => { - std::borrow::Cow::Owned(restore_with_conditions(restore, conditions.to_vec())) - } + None => CarriedRestore::Unchanged(restore), + Some(conditions) => CarriedRestore::Cleared(Box::new(restore_with_conditions( + restore, + conditions.to_vec(), + ))), } } + /// Where the `waitTimeout` window is anchored: `status.waitStartedAt` once /// [`super::ensure_wait_anchor`] has stamped it, else `created_epoch`. /// diff --git a/crates/controller/src/restore/tests.rs b/crates/controller/src/restore/tests.rs index f301cb66..2e764466 100644 --- a/crates/controller/src/restore/tests.rs +++ b/crates/controller/src/restore/tests.rs @@ -325,7 +325,12 @@ fn a_cleared_referent_condition_survives_a_downstream_gate_park() { let cleared = cleared_referent_conditions(&parked).expect("a stale park clears"); // Built through the one construction site the gate uses, so this exercises // the real seam rather than a re-implementation of it. - let carried = carried_after_clear(&parked, Some(&cleared)).into_owned(); + let carrier = carried_after_clear(&parked, Some(&cleared)); + assert!( + matches!(carrier, CarriedRestore::Cleared(_)), + "a cleared pass must carry an OWNED copy, never the stale borrow" + ); + let carried = carrier.get().clone(); // What a downstream gate park computes from the CARRIED copy: the clear holds. let after_park = io::upsert_gate( @@ -370,8 +375,11 @@ fn a_cleared_referent_condition_survives_a_downstream_gate_park() { &untouched, cleared_referent_conditions(&untouched).as_deref(), ); - assert!(matches!(same, std::borrow::Cow::Borrowed(_))); - assert_eq!(same.as_ref().status, untouched.status); + assert!( + matches!(same, CarriedRestore::Unchanged(_)), + "the common path must borrow: no clone, no allocation, nothing changed" + ); + assert_eq!(same.get().status, untouched.status); } #[test] From 15303001c5bad929523f6f7c4b9efb80c3eeabd1 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 11:27:13 -0700 Subject: [PATCH 06/15] test(kopia): assert the no-flip throttle sequence keeps the source read-only (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding: the test proved every piece of the source-side sequence but never asserted the invariant the pieces exist to protect. Since `throttle set` turns out NOT to need the read-write window, what M3c/M3d will ship is `connect --readonly` -> `throttle set` -> migrate with no flip — and `throttle set` rewrites the config file. If that rewrite silently dropped the `readonly` key, a replication SOURCE would be left writable and nothing downstream would notice: the next step flipped read-write on purpose, and the step after re-established the bit explicitly. Restructures so the measured path IS the production path: * the cap now lands at step (1b) on the read-only connection, and step (1c) immediately asserts the bit survived — in the config JSON AND behaviorally, via a write-requiring `set-parameters` probe; * steps (2)-(4) keep the flip proof for the verbs that genuinely need the window, and (5) asserts the cap set on the read-only connection came through the flip sequence untouched, so (6) times the config the no-flip sequence wrote; * the timing bound is now self-calibrating: an identical UNTHROTTLED baseline migrate is timed first and the capped run must be >= 10x it, so the verdict no longer depends on machine speed. The 3s absolute floor stays as a backstop against a pathologically slow baseline making the ratio trivial. Both new assertions were mutation-checked against the real binary: forcing the read-only bit off before (1c) fails it with "the config must record `readonly: true`", and a non-binding 20 MB/s cap fails the ratio bound at 190.5ms vs a 189.9ms baseline. Also from the review ledger: pin "set-client writes nothing to the backend" by fingerprinting every blob across the flip instead of asserting it in prose, and truncate the shim's argv capture directory before use so an appending shim cannot inherit stale lines. --- crates/kopia/src/client/mod.rs | 10 +- crates/kopia/src/client/tests.rs | 3 + .../tests/integration_set_client_throttle.rs | 298 ++++++++++++------ 3 files changed, 209 insertions(+), 102 deletions(-) diff --git a/crates/kopia/src/client/mod.rs b/crates/kopia/src/client/mod.rs index 894d5094..c157735f 100644 --- a/crates/kopia/src/client/mod.rs +++ b/crates/kopia/src/client/mod.rs @@ -1427,13 +1427,15 @@ impl KopiaClient { /// blob write and hard-errors there. /// - **The bit lives in the local config, not the repository.** `read_only: true` /// writes `"readonly": true` into the `KOPIA_CONFIG_PATH` JSON; `false` removes the - /// key. Nothing is written to the backend, so this never mutates shared state and - /// needs no maintenance ownership/lease. + /// key. The test fingerprints every blob in the backend across the flip and requires + /// it unchanged, so this never mutates shared state and needs no maintenance + /// ownership/lease. /// - **It is the "flip window" primitive** for a read-only-connected config that must /// run a write-requiring verb: flip read-write, run the verb, flip back. Note that /// [`Self::repository_throttle_set`] does *not* need this window — M3a measured it - /// succeeding directly on a `--readonly` connection (it only rewrites the local - /// config's `throttlingLimits`), so the flip is defensive there, not required. + /// succeeding directly on a `--readonly` connection, and leaving the read-only bit + /// intact, because it only rewrites the local config's `throttlingLimits`. The flip + /// is defensive there, not required. pub async fn repository_set_client_read_only(&self, read_only: bool) -> Result<(), KopiaError> { let mode = if read_only { "--read-only" diff --git a/crates/kopia/src/client/tests.rs b/crates/kopia/src/client/tests.rs index 54c0d50f..bd2a1d96 100644 --- a/crates/kopia/src/client/tests.rs +++ b/crates/kopia/src/client/tests.rs @@ -1486,6 +1486,9 @@ async fn set_client_read_only_emits_the_matching_polarity_flag() { // assertion has to be on what the process is actually SPAWNED with. A shim // standing in for `kopia` appends its argv to a file (issue #374). let dir = std::env::temp_dir().join(format!("kopiur-setclient-{}", std::process::id())); + // The shim APPENDS, so a leftover argv.out from an earlier run in the same + // process-id directory would prepend phantom lines. Start from nothing. + let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); let shim = dir.join("kopia"); let out = dir.join("argv.out"); diff --git a/crates/kopia/tests/integration_set_client_throttle.rs b/crates/kopia/tests/integration_set_client_throttle.rs index 1d92dba2..694ddd7b 100644 --- a/crates/kopia/tests/integration_set_client_throttle.rs +++ b/crates/kopia/tests/integration_set_client_throttle.rs @@ -17,31 +17,41 @@ //! //! What it pins against kopia 0.23.1: //! +//! * **`repository throttle set` does NOT need a read-write window.** It succeeds +//! directly on a `--readonly` connection — contradicting the design premise +//! this milestone was written to confirm (kopia registers it as a write action, +//! but it never puts a blob, so read-only storage never rejects it). The +//! sequence M3c/M3d will actually ship is therefore `connect --readonly` → +//! `throttle set` → migrate, with no flip; that is what steps (1b)→(6) assert, +//! end to end, and it is the sequence the timing probe measures. +//! * **…and that sequence leaves the source READ-ONLY.** `throttle set` rewrites +//! the config file, so "the bit survives the rewrite" is a data-safety +//! invariant for a replication source, not a formality. Step (1c) asserts it +//! both in the config JSON and behaviorally, immediately after the cap lands +//! and before anything flips. //! * `repository set-client --read-only` / `--read-write` works on a config //! connected `--readonly`, and only rewrites the LOCAL config's `readonly` key -//! (`true` present / key absent when read-write). Nothing reaches the backend. +//! (`true` present / key absent when read-write) — the backend is fingerprinted +//! across the flip to pin that nothing reaches it. //! * The flip is real, not cosmetic: `repository set-parameters` — a genuinely -//! write-requiring verb — fails while read-only, succeeds inside the flip -//! window, and fails again after the flip back. -//! * **`repository throttle set` does NOT need the flip window.** It succeeds -//! directly on a `--readonly` connection — contradicting the design premise -//! this milestone was written to confirm (kopia registers it as a write action, -//! but it never puts a blob, so read-only storage never rejects it). The flip -//! is therefore defensive for throttling, not required. If a future kopia turns -//! it into a real write, step (1b) below fails loudly and the flip becomes -//! mandatory. +//! write-requiring verb, and the one that DOES need the window — fails while +//! read-only, succeeds inside the flip window, and fails again after the flip +//! back. //! * Limits persist as `throttlingLimits` in the config JSON and survive the flip -//! back to read-only. +//! sequence untouched. //! * The load-bearing claim for the source side: `snapshot migrate //! --source-config ` genuinely honors those config-persisted //! limits. `snapshot migrate` has no speed flags at all, so the persisted //! config is the only lever there is — if this were false, the whole -//! throttle-the-source design would be inert. +//! throttle-the-source design would be inert. The bound is self-calibrating: +//! the test times its own UNTHROTTLED baseline migrate first and requires the +//! capped one to be at least an order of magnitude slower, so the verdict does +//! not depend on how fast the machine is. //! * The caveat that comes with it: throttling only bites on traffic that //! actually reaches the backend. A warm source cache makes even a 1 KB/s cap //! invisible (the migrate serves from local disk and finishes in ~200 ms), so -//! step (6) drops the cache first — which is also what a fresh mover pod looks -//! like. +//! both timed migrates drop the cache first — which is also what a fresh mover +//! pod looks like. #![cfg(unix)] @@ -81,12 +91,21 @@ const IDENTITY: &str = "throttleuser@throttlehost:/data"; /// caps to hours. const MIGRATE_CAP_BYTES_PER_SEC: i64 = 2_000_000; -/// The wall-clock floor a throttled migrate must clear, well under the ~14 s -/// measured at [`MIGRATE_CAP_BYTES_PER_SEC`] so a loaded CI box cannot make it -/// flake. The failure it guards is "the throttle was ignored entirely", which -/// lands at ~0.2 s — two orders of magnitude below this floor, so the verdict is -/// never ambiguous. Throttling can only ever make a run SLOWER, so a faster -/// machine cannot push a genuinely throttled run below the bound. +/// How many times slower than the UNTHROTTLED baseline the capped migrate must +/// be. The bound is relative on purpose: the test measures its own baseline on +/// the machine it is running on (an identical migrate of the identical fixture, +/// cold cache, no cap) instead of hard-coding a wall-clock figure that a slow CI +/// box could breach for the wrong reason. The measured ratio here is ~68x +/// (14.2 s vs 0.21 s), so 10x leaves ~7x of headroom, while a completely +/// ignored throttle lands at ~1x and cannot squeak past. +const MIGRATE_MIN_SLOWDOWN_FACTOR: u32 = 10; + +/// A wall-clock floor kept ALONGSIDE the relative bound, as a backstop for the +/// degenerate case where the baseline itself is pathologically slow (a stalled +/// CI runner) and would make the ratio trivially satisfiable. Well under the +/// ~14 s measured at [`MIGRATE_CAP_BYTES_PER_SEC`]; throttling can only ever +/// make a run slower, so a faster machine cannot push a genuinely throttled run +/// below it. const MIGRATE_MIN_ELAPSED_SECS: u64 = 3; fn config_path(config_dir: &std::path::Path) -> String { @@ -159,6 +178,76 @@ async fn try_set_epoch_advance_on_count( .await } +/// Assert the connection is read-only, BOTH ways: the recorded bit in the config +/// JSON, and behaviorally — a write-requiring verb still gets rejected. Asserting +/// only the first would let a kopia that records the bit but stops honoring it +/// pass; asserting only the second would miss the config regressing under a later +/// rewrite. `probe_count` must differ from the last value successfully written, +/// so a rejection can never be kopia short-circuiting a no-change write. +async fn assert_read_only(client: &KopiaClient, config_dir: &std::path::Path, probe_count: i64) { + assert_eq!( + read_config(config_dir).get("readonly"), + Some(&serde_json::Value::Bool(true)), + "the config must record `readonly: true`" + ); + let err = try_set_epoch_advance_on_count(client, probe_count) + .await + .expect_err("a write-requiring verb must be rejected while read-only"); + assert!( + err.to_string().contains("read-only") || err.to_string().contains("readonly"), + "expected a read-only storage rejection, got: {err}" + ); +} + +/// Every file in the repository backend, by path and size. Used to pin the claim +/// that `set-client` touches nothing on the backend — a claim that would +/// otherwise be inference from "it works on read-only storage". +fn backend_fingerprint(repo_dir: &std::path::Path) -> BTreeMap { + fn walk(dir: &std::path::Path, out: &mut BTreeMap) { + for entry in std::fs::read_dir(dir).expect("read repository directory") { + let entry = entry.expect("repository directory entry"); + let meta = entry.metadata().expect("repository entry metadata"); + if meta.is_dir() { + walk(&entry.path(), out); + } else { + out.insert(entry.path(), meta.len()); + } + } + } + let mut out = BTreeMap::new(); + walk(repo_dir, &mut out); + out +} + +/// Run one `snapshot migrate` of the whole source into `dest` and return how long +/// it took, with the source cache dropped first. +/// +/// Dropping the cache is load-bearing, not hygiene: kopia only throttles traffic +/// that actually reaches the backend, and the migrating process reads the source +/// through the cache directory recorded in the SOURCE config +/// (`caching.cacheDirectory`, relative to the config file → `/cache`). Left +/// warm, the migrate serves everything from local disk and finishes unthrottled +/// in ~200 ms whatever the cap says. A fresh mover pod's cache is cold by +/// construction, which is what this models — and it is also the caveat the +/// dependent milestones have to design around. +async fn timed_cold_migrate( + dest: &KopiaClient, + src_ro_cfg: &std::path::Path, +) -> std::time::Duration { + std::fs::remove_dir_all(src_ro_cfg.join("cache")).expect("drop the source cache"); + let started = Instant::now(); + dest.snapshot_migrate(&SnapshotMigrateOptions { + source_config_path: config_path(src_ro_cfg), + sources: MigrateSources::All, + latest_only: false, + parallel: None, + policies: MigratePolicies::None, + }) + .await + .expect("snapshot migrate"); + started.elapsed() +} + #[tokio::test] #[cfg_attr(not(feature = "integration"), ignore)] async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() { @@ -168,6 +257,8 @@ async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() let data_dir = tempfile::tempdir().unwrap(); let dest_repo = tempfile::tempdir().unwrap(); let dest_cfg = tempfile::tempdir().unwrap(); + let base_repo = tempfile::tempdir().unwrap(); + let base_cfg = tempfile::tempdir().unwrap(); // ---- fixture: a source repository holding one snapshot of 8 KiB of // deliberately incompressible-but-DETERMINISTIC bytes (a Knuth-multiplicative @@ -199,6 +290,10 @@ async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() .await .expect("source snapshot create"); + // Two destinations: one for the unthrottled BASELINE migrate that calibrates + // the timing bound, one for the throttled measurement. They must be separate + // repositories — migrate is idempotent by (source, startTime), so a second + // run into the same destination would copy nothing and time nothing. let dest = dest_client(dest_cfg.path(), DEST_PASSWORD); dest.repository_create( &ConnectSpec::Filesystem { @@ -209,6 +304,17 @@ async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() ) .await .expect("destination repository create"); + let baseline_dest = dest_client(base_cfg.path(), DEST_PASSWORD); + baseline_dest + .repository_create( + &ConnectSpec::Filesystem { + path: base_repo.path().to_path_buf(), + }, + Default::default(), + &Default::default(), + ) + .await + .expect("baseline destination repository create"); // ---- the connect the replication/seed movers perform for their SOURCE: // a separate config, read-only, credentials persisted beside it. @@ -235,37 +341,54 @@ async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() // (1) The read-only bit is REALLY in force: a verb that must write a blob // fails. This is the control for everything below — without it, "throttle // set works on read-only" would prove nothing. - let err = try_set_epoch_advance_on_count(&src_ro, 10) - .await - .expect_err("set-parameters must fail on a read-only connection"); - assert!( - err.to_string().contains("read-only") || err.to_string().contains("readonly"), - "expected a read-only storage rejection, got: {err}" - ); + assert_read_only(&src_ro, src_ro_cfg.path(), 10).await; + + // (1a) BASELINE: an identical migrate with NO cap, on this machine, to + // calibrate the timing bound in step (6). Taken before any throttle exists so + // it measures the genuinely unthrottled cost. + let baseline = timed_cold_migrate(&baseline_dest, src_ro_cfg.path()).await; - // (1b) THE PREMISE THIS MILESTONE WAS WRITTEN TO CHECK — and it does not - // hold. `repository throttle set` SUCCEEDS on the read-only connection: it - // only rewrites the local config's `throttlingLimits`, it never puts a blob. - // The flip window is therefore DEFENSIVE for throttling, not required. - // If a future kopia turns this into a real write action, this line fails - // loudly and the dependent milestones must adopt the flip unconditionally. - let benign = ThrottleArgs { - upload_bytes_per_second: Some(1_000_000), - download_bytes_per_second: Some(1_000_000), - ..Default::default() - }; + // (1b) THE PRODUCTION SEQUENCE, and the premise this milestone was written to + // check — which does NOT hold as the plan stated it. `repository throttle set` + // SUCCEEDS on the read-only connection: it only rewrites the local config's + // `throttlingLimits`, it never puts a blob. So what M3c/M3d will actually ship + // is `connect --readonly` → `throttle set` → migrate, with NO flip, and that + // is the sequence measured in step (6). + // + // If a future kopia turns this into a real write action, this fails loudly and + // the flip below stops being defensive and becomes mandatory. src_ro - .repository_throttle_set(&benign) + .repository_throttle_set(&ThrottleArgs { + upload_bytes_per_second: Some(MIGRATE_CAP_BYTES_PER_SEC), + download_bytes_per_second: Some(MIGRATE_CAP_BYTES_PER_SEC), + ..Default::default() + }) .await .expect("throttle set succeeds on a READ-ONLY connection (kopia 0.23.1)"); + let limits = read_config(src_ro_cfg.path())["throttlingLimits"].clone(); assert_eq!( - read_config(src_ro_cfg.path())["throttlingLimits"]["maxUploadSpeedBytesPerSecond"], - serde_json::json!(1_000_000), - "throttle set must persist into the local config even when read-only" + limits["maxUploadSpeedBytesPerSecond"], + serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), + "throttle set must persist into the local config even when read-only: {limits}" + ); + assert_eq!( + limits["maxDownloadSpeedBytesPerSecond"], + serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), + "throttle set must persist into the local config even when read-only: {limits}" ); - // (2) Open the flip window and prove it is real: the same write-requiring - // verb that failed above now SUCCEEDS. + // (1c) …AND THE SOURCE IS STILL READ-ONLY. This is the data-safety invariant + // of the whole no-flip sequence: `throttle set` rewrites the config file, and + // a rewrite that dropped the `readonly` key would silently leave a replication + // SOURCE writable. Asserted both ways — the recorded bit and the behavioral + // probe — because nothing downstream would notice otherwise: step (2) flips + // read-write on purpose, and step (3) re-establishes the bit explicitly. + assert_read_only(&src_ro, src_ro_cfg.path(), 11).await; + + // (2) The flip window itself — kept because the client method has to be + // proven for the verbs that genuinely DO need it. Opening it makes the same + // write-requiring verb that failed twice above SUCCEED. + let backend_before = backend_fingerprint(src_repo.path()); src_ro .repository_set_client_read_only(false) .await @@ -274,52 +397,37 @@ async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() read_config(src_ro_cfg.path()).get("readonly").is_none(), "read-write must clear the config's `readonly` key" ); + assert_eq!( + backend_fingerprint(src_repo.path()), + backend_before, + "set-client must not touch the backend — the flip is a LOCAL config edit" + ); try_set_epoch_advance_on_count(&src_ro, 10) .await .expect("set-parameters must succeed inside the read-write flip window"); - // Apply the cap the timing probe measures, inside the window — this is the - // exact flip → throttle set → flip-back sequence the dependent milestones use. - src_ro - .repository_throttle_set(&ThrottleArgs { - upload_bytes_per_second: Some(MIGRATE_CAP_BYTES_PER_SEC), - download_bytes_per_second: Some(MIGRATE_CAP_BYTES_PER_SEC), - ..Default::default() - }) - .await - .expect("throttle set inside the flip window"); - - // (3) Flip back, and (4) assert read-only is genuinely restored — config bit - // AND the behavioral probe, with a DIFFERENT epoch count (12, not the 10 just - // committed) so the rejection cannot be a no-change short-circuit. + // (3) Flip back, and (4) assert read-only is genuinely restored — with a + // DIFFERENT epoch count (12, not the 10 just committed) so the rejection + // cannot be a no-change short-circuit. src_ro .repository_set_client_read_only(true) .await .expect("set-client --read-only"); - assert_eq!( - read_config(src_ro_cfg.path()).get("readonly"), - Some(&serde_json::Value::Bool(true)), - "the flip back must restore `readonly: true`" - ); - let err = try_set_epoch_advance_on_count(&src_ro, 12) - .await - .expect_err("set-parameters must fail again after the flip back"); - assert!( - err.to_string().contains("read-only") || err.to_string().contains("readonly"), - "expected a read-only storage rejection after the flip back, got: {err}" - ); + assert_read_only(&src_ro, src_ro_cfg.path(), 12).await; - // (5) The limits survived the whole flip sequence, in the config on disk. + // (5) The cap set back at (1b), on the read-only connection, survived the + // whole flip sequence untouched — so the config step (6) measures is the one + // the no-flip production sequence wrote. let limits = read_config(src_ro_cfg.path())["throttlingLimits"].clone(); assert_eq!( limits["maxUploadSpeedBytesPerSecond"], serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), - "upload cap must survive the flip back: {limits}" + "upload cap must survive the flip sequence: {limits}" ); assert_eq!( limits["maxDownloadSpeedBytesPerSecond"], serde_json::json!(MIGRATE_CAP_BYTES_PER_SEC), - "download cap must survive the flip back: {limits}" + "download cap must survive the flip sequence: {limits}" ); // (6) THE LOAD-BEARING CLAIM: `snapshot migrate` reopens the source config @@ -327,28 +435,10 @@ async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() // flags, so if this were false there would be no way to throttle a // seed-migrate at all. // - // The source cache must be COLD first. kopia only throttles traffic that - // actually reaches the backend, and the migrating process reads the source - // through the cache directory recorded in the SOURCE config - // (`caching.cacheDirectory`, relative to the config file → `/cache`). - // The earlier steps warmed it; leaving it warm makes the migrate serve - // everything from local disk and finish unthrottled in ~200ms — which is - // also the honest caveat for the dependent milestones: throttling only - // bites on real backend traffic, and a mover pod's cache is cold by - // construction, which is what this models. - std::fs::remove_dir_all(src_ro_cfg.path().join("cache")).expect("drop the source cache"); - - let started = Instant::now(); - dest.snapshot_migrate(&SnapshotMigrateOptions { - source_config_path: config_path(src_ro_cfg.path()), - sources: MigrateSources::All, - latest_only: false, - parallel: None, - policies: MigratePolicies::None, - }) - .await - .expect("throttled snapshot migrate"); - let elapsed = started.elapsed(); + // The config this reads is the one the NO-FLIP sequence wrote at (1b) and + // (5) proved untouched since — i.e. this measures exactly what production + // will do, not a flip-bracketed variant of it. + let elapsed = timed_cold_migrate(&dest, src_ro_cfg.path()).await; let migrated = dest.snapshot_list_all().await.expect("dest list --all"); assert_eq!( @@ -357,11 +447,23 @@ async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() "the snapshot must actually arrive: {migrated:?}" ); assert_eq!(migrated[0].source.identity(), IDENTITY); + + // Both bounds, and the failure message carries the baseline so a breach is + // diagnosable rather than just "too fast". The mode this guards is "the + // throttle was ignored entirely", which lands at ~1x the baseline. + assert!( + elapsed >= baseline * MIGRATE_MIN_SLOWDOWN_FACTOR, + "migrate finished in {elapsed:?} with the source config capped at \ + {MIGRATE_CAP_BYTES_PER_SEC} B/s, against an UNTHROTTLED baseline of \ + {baseline:?} for the identical migrate — under {MIGRATE_MIN_SLOWDOWN_FACTOR}x \ + it must have IGNORED the persisted throttle. The entire source-side \ + throttle design rests on migrate honoring it." + ); assert!( elapsed.as_secs() >= MIGRATE_MIN_ELAPSED_SECS, "migrate finished in {elapsed:?} with the source config capped at \ - {MIGRATE_CAP_BYTES_PER_SEC} B/s — it must have IGNORED the persisted \ - throttle (an unthrottled migrate of this fixture is sub-second). The \ - entire source-side throttle design rests on migrate honoring it." + {MIGRATE_CAP_BYTES_PER_SEC} B/s (baseline {baseline:?}) — below the \ + {MIGRATE_MIN_ELAPSED_SECS}s absolute floor, so the relative bound was \ + satisfied only because the baseline itself was pathologically slow." ); } From 869cf8f1908f386e7b041e1c503f3bd1d58a5ea2 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 11:51:07 -0700 Subject: [PATCH 07/15] fix(mover): honor moverDefaults.throttle in every own-connect flow (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `repository throttle set` ran only in `run()`'s generic connect arm, so every flow that opens its OWN connection silently dropped the `spec.throttle` its controller had faithfully put on the wire: bootstrap (all three connects, including the `mode: ReadOnly` probe), seed-migrate's local repository, maintenance, verify and replicate. kopia's limits are per-CONNECTION — they persist into that connection's client config — so a second connect under a second config needs its own call; there is nothing repository-wide to inherit. The invariant this ships: EVERY mover flow that connects a repository applies `MoverWorkSpec.throttle` immediately after its connect, no exceptions. One shared `apply_repository_throttle` helper does it, skipping when no limit is set (no kopia process spawned for the common case), treating a refusal as terminal (running uncapped over the link the user capped is worse than not running), and emitting the one `applied repository throttle` line that is both the e2e-assertable proof and the old-mover/new-controller skew signal. Read-only connections are throttled too. M3a measured `repository throttle set` succeeding on a `--readonly` connect against kopia 0.23.1 (with `set-parameters` as the control probe that does hard-error there), so the read-write flip is not needed and `repository_set_client_read_only` stays an unwired contingency. The dispatch `_ =>` catch-all in `run()` is replaced by an exhaustive `Operation` match — that catch-all is how this bug shipped, and a new operation must now decide its throttle story to compile. Bootstrap's three connects go through one funnel that applies the throttle by construction; a throttle failure there is kept structurally distinct from a CONNECT failure (`BootstrapConnectFailure`), since the probe reads a connect failure to decide whether to CREATE or SEED a repository over the backend and a throttle failure says nothing about it. Deliberately out of scope, both stated rather than forgotten: - snapshot delete-batch and pin work specs carry `Default` (empty) throttle from their controllers, so they ride the generic arm as a no-op — a controller-side gap, not a mover one; - browse builds its work spec in the CLI with `throttle: Default::default()`, so its `spec.throttle` is structurally always empty (comment only); - SnapshotReplicate's dual connect and the seed SOURCE (replica) connect land with their own per-CR override surfaces in the following commits. E2e, no silent green: three fixture repositories gain a non-binding 100 MiB/s `moverDefaults.throttle` and the scenarios assert the mover's own log line — timing proves nothing, since kopia's cap only bites cold-cache backend traffic. Covers bootstrap + maintenance (S3 maintenance scenario), verify, and replicate; each pre-created repository is read back so a dropped overlay cannot make the assertion vacuous. --- crates/e2e/src/consts.rs | 14 ++ crates/e2e/src/wait.rs | 62 +++++ crates/e2e/tests/object_store.rs | 37 +++ crates/e2e/tests/replication.rs | 55 +++++ crates/e2e/tests/verification.rs | 56 +++++ crates/mover/src/main.rs | 402 ++++++++++++++++++++++++++----- crates/mover/src/status.rs | 63 ++++- 7 files changed, 626 insertions(+), 63 deletions(-) diff --git a/crates/e2e/src/consts.rs b/crates/e2e/src/consts.rs index aa3d53ed..327bd79b 100644 --- a/crates/e2e/src/consts.rs +++ b/crates/e2e/src/consts.rs @@ -246,6 +246,20 @@ pub const PV_SRC_EH: &str = "kopiur-e2e-src-eh"; /// PVC (operator namespace) binding [`PV_SRC_EH`]. pub const PVC_SRC_EH: &str = "e2e-src-eh"; +// --- Repository throttle (#374) ------------------------------------------------ +/// The mover's log line proving `moverDefaults.throttle` reached kopia on a +/// connection (`crates/mover/src/main.rs::apply_repository_throttle`). Scenarios +/// assert on THIS, not on timing: kopia's limits only bite cold-cache backend +/// traffic, so a generous cap changes no runtime and a timing-based assertion +/// would be a silent green. Keep in lockstep with the mover's `info!` message. +pub const THROTTLE_APPLIED_LOG: &str = "applied repository throttle"; + +/// A deliberately NON-BINDING throttle (100 MiB/s) for the scenarios that assert +/// the throttle is applied at all. Generous on purpose: it must not move any +/// scenario's runtime, and a low byte cap is punishing out of all proportion on +/// the small-object workloads these fixtures produce. +pub const THROTTLE_BYTES_PER_SECOND: i64 = 104_857_600; + // --- Secrets ------------------------------------------------------------------- /// Filesystem-backend credentials (just `KOPIA_PASSWORD`). pub const SECRET_FS_CREDS: &str = "kopia-creds"; diff --git a/crates/e2e/src/wait.rs b/crates/e2e/src/wait.rs index fd6c57c6..4a082650 100644 --- a/crates/e2e/src/wait.rs +++ b/crates/e2e/src/wait.rs @@ -71,6 +71,68 @@ pub async fn pod_succeeded(client: &Client, ns: &str, name: &str) -> Result<()> Err(anyhow!("pod {ns}/{name} ended {phase}; logs:\n{logs}")) } +/// Concatenated stdout of every non-terminating Pod matching `selector` in `ns`. +/// +/// Mover Jobs carry their component/instance labels on the POD template as well +/// as the Job, so the same selector a scenario uses to find a mover Job finds +/// the pod that ran it. Pods that have not produced logs yet (still `Pending`, +/// or already reaped) are skipped rather than failing the call, so this composes +/// with [`crate::wait_until`]: poll until the line you are looking for shows up. +/// +/// Every matching pod is concatenated on purpose — a retried Job has several, +/// and the assertion is about what the mover DID, not about which attempt did it. +pub async fn pod_logs_for_selector( + client: &Client, + ns: &str, + selector: &str, +) -> std::result::Result { + let api: Api = Api::namespaced(client.clone(), ns); + let list = api + .list(&kube::api::ListParams::default().labels(selector)) + .await?; + let mut out = String::new(); + for pod in list + .items + .into_iter() + .filter(|p| p.metadata.deletion_timestamp.is_none()) + { + let Some(name) = pod.metadata.name else { + continue; + }; + if let Ok(logs) = api.logs(&name, &LogParams::default()).await { + out.push_str(&logs); + } + } + Ok(out) +} + +/// Poll until some pod matching `selector` has logged `needle`. +/// +/// The assertion form for "the mover actually did X" when X leaves no trace in +/// any CR status — e.g. the repository throttle (#374), whose only observable is +/// [`crate::consts::THROTTLE_APPLIED_LOG`]. Polls rather than reading once, +/// because the pod may not exist (or may still be starting) when the caller +/// first looks. +pub async fn wait_for_pod_log( + client: &Client, + ns: &str, + selector: &str, + needle: &str, +) -> Result<()> { + crate::wait_until( + &format!("a pod matching `{selector}` logs `{needle}`"), + default_timeout(), + crate::poll_interval(), + || async { + Ok(pod_logs_for_selector(client, ns, selector) + .await? + .contains(needle) + .then_some(())) + }, + ) + .await +} + /// Fetch a Pod's logs (best-effort context for failure messages). pub async fn pod_logs(client: &Client, ns: &str, name: &str) -> Result { let api: Api = Api::namespaced(client.clone(), ns); diff --git a/crates/e2e/tests/object_store.rs b/crates/e2e/tests/object_store.rs index 4678f984..52980110 100644 --- a/crates/e2e/tests/object_store.rs +++ b/crates/e2e/tests/object_store.rs @@ -496,6 +496,17 @@ async fn s3_maintenance_runs_in_a_mover_job() { let repo_json = { let mut v = s3_repository_json("e2e-s3-maint", "kopiur-maint", "kopia-s3-creds", true); v["spec"]["maintenance"] = serde_json::json!({ "enabled": false }); + // #374: a repository-wide throttle every mover flow must honor. This repo + // drives TWO of the flows that used to ignore it — the BOOTSTRAP Job below + // and the MAINTENANCE Job after it — and both are asserted at the mover's + // own log line further down. Deliberately non-binding (100 MiB/s against + // an in-cluster MinIO), so it proves the wiring without moving runtimes. + v["spec"]["moverDefaults"] = serde_json::json!({ + "throttle": { + "uploadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND, + "downloadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND + } + }); v }; repos @@ -506,6 +517,19 @@ async fn s3_maintenance_runs_in_a_mover_job() { .await .expect("S3 Repository should reach Ready via the bootstrap Job"); + // #374 regression guard, flow 1 of 2: the BOOTSTRAP mover applied the + // repository throttle. Before the fix, bootstrap connected on its own and + // silently dropped `spec.throttle` — the run still succeeded, which is + // exactly why only the mover's own log line can prove it. + kopiur_e2e::wait::wait_for_pod_log( + &client, + E2E_NAMESPACE, + "kopiur.home-operations.com/repository=e2e-s3-maint", + consts::THROTTLE_APPLIED_LOG, + ) + .await + .expect("the bootstrap mover must apply moverDefaults.throttle on its own connect (#374)"); + // Create the explicit Maintenance; the controller spawns a per-slot mover Job. // (The operator honors an externally-authored Maintenance even when the repo // opted out of managed maintenance — ADR §3.7.) @@ -537,6 +561,19 @@ async fn s3_maintenance_runs_in_a_mover_job() { .await .expect("a maintenance mover Job should run to completion"); + // #374 regression guard, flow 2 of 2: the MAINTENANCE mover applied the same + // repository throttle on its own connect. A full maintenance rewrites and + // drops blobs, so this is real backend traffic that used to run uncapped. + kopiur_e2e::wait::wait_for_pod_log( + &client, + E2E_NAMESPACE, + "app.kubernetes.io/component=maintenance,\ + kopiur.home-operations.com/maintenance=e2e-s3-maint", + consts::THROTTLE_APPLIED_LOG, + ) + .await + .expect("the maintenance mover must apply moverDefaults.throttle on its own connect (#374)"); + // The mover ran `kopia maintenance` against S3 and PATCHed the status: the // first run is full (which also stamps quick), and the lease is owned. This // can only be true if maintenance actually executed. diff --git a/crates/e2e/tests/replication.rs b/crates/e2e/tests/replication.rs index 82b87fad..5f22a41c 100644 --- a/crates/e2e/tests/replication.rs +++ b/crates/e2e/tests/replication.rs @@ -31,6 +31,49 @@ async fn repository_replication_mirrors_to_second_filesystem_repo() { }; world.ensure(&[Need::Filesystem]).await.expect("fixtures"); let client = world.client().clone(); + + // #374: the SOURCE repository carries a repository-wide throttle, so the + // replication mover — which opens its own connect to the source and reads the + // whole repository through it — must apply it. Created HERE rather than + // through `ensure_seed` (which hardcodes an empty overlay and is + // create-if-absent); no other scenario uses this repo, so pre-creating it is + // race-free. Non-binding at 100 MiB/s over a local hostPath. + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + ensure_repo(&client, "repl-src").await; + create_idempotent( + &repos, + &cr(repository_json( + "e2e-repl-src", + "repl-src", + serde_json::json!({ + "moverDefaults": { + "throttle": { + "uploadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND, + "downloadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND + } + } + }), + )), + "create the throttled replication source Repository", + ) + .await; + // Bare spec fields, not a `{"spec": ...}` wrapper — the wrapper is dropped + // silently and would leave the throttle assertion below vacuous. + let landed = repos + .get("e2e-repl-src") + .await + .expect("read back the throttled replication source Repository"); + assert_eq!( + landed + .spec + .mover_defaults + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.upload_bytes_per_second), + Some(consts::THROTTLE_BYTES_PER_SECOND), + "moverDefaults.throttle must survive onto the source Repository" + ); + // A source repo with a real snapshot to mirror. ensure_seed( &client, @@ -139,6 +182,18 @@ async fn repository_replication_mirrors_to_second_filesystem_repo() { .await .expect("the replication should run and stamp status.lastReplicated"); + // #374 regression guard: the replication mover applied the SOURCE + // repository's throttle on its own connect. `sync-to` succeeds either way, + // so the mover's log line is the only proof the cap was in force. + kopiur_e2e::wait::wait_for_pod_log( + &client, + E2E_NAMESPACE, + &selector, + consts::THROTTLE_APPLIED_LOG, + ) + .await + .expect("the replication mover must apply moverDefaults.throttle on its own connect (#374)"); + // kstatus consistency guard (regression: the same two-pass heal bug as // restore/snapshot). The mover stamps `phase: Succeeded` + `lastReplicated`; // the controller heals `Ready=True` in a FOLLOW-UP reconcile. Gate on the diff --git a/crates/e2e/tests/verification.rs b/crates/e2e/tests/verification.rs index a0519480..af7f3aa6 100644 --- a/crates/e2e/tests/verification.rs +++ b/crates/e2e/tests/verification.rs @@ -74,6 +74,50 @@ async fn verification_quick_with_success_expr_stamps_last_verified() { }; world.ensure(&[Need::Filesystem]).await.expect("fixtures"); let client = world.client().clone(); + + // #374: this repository carries a repository-wide throttle, so the verify + // mover — which owns its OWN connect and used to drop `spec.throttle` on the + // floor — must apply it. Created HERE rather than through `ensure_seed`, + // which hardcodes an empty overlay and is create-if-absent; this repo is + // used by no other scenario, so pre-creating it is race-free. The cap is + // deliberately non-binding (100 MiB/s over a local hostPath). + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + ensure_repo(&client, "verify").await; + create_idempotent( + &repos, + &cr(repository_json( + "e2e-verify-repo", + "verify", + serde_json::json!({ + "moverDefaults": { + "throttle": { + "uploadBytesPerSecond": kopiur_e2e::consts::THROTTLE_BYTES_PER_SECOND, + "downloadBytesPerSecond": kopiur_e2e::consts::THROTTLE_BYTES_PER_SECOND + } + } + }), + )), + "create the throttled verify Repository", + ) + .await; + // The overlay is BARE spec fields; a `{"spec": ...}` wrapper would be dropped + // by serde AND pruned by the apiserver, leaving a repo with no throttle and a + // test that asserts nothing. Read it back and prove the field landed. + let landed = repos + .get("e2e-verify-repo") + .await + .expect("read back the throttled verify Repository"); + assert_eq!( + landed + .spec + .mover_defaults + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.upload_bytes_per_second), + Some(kopiur_e2e::consts::THROTTLE_BYTES_PER_SECOND), + "moverDefaults.throttle must survive onto the Repository, or the assertion below is vacuous" + ); + // Seed a real snapshot so quick-verify has something to verify. ensure_seed( &client, @@ -135,6 +179,18 @@ async fn verification_quick_with_success_expr_stamps_last_verified() { // mover contract (`operation.verify.tier.quick.*`). let selector = "app.kubernetes.io/component=verify,\ kopiur.home-operations.com/verify=e2e-verify-policy"; + + // #374 regression guard: the verify mover applied the repository throttle on + // its own connect. A capped repo whose verify Job ignores the cap still + // stamps `lastVerified`, so the mover's log line is the only proof. + kopiur_e2e::wait::wait_for_pod_log( + &client, + E2E_NAMESPACE, + selector, + kopiur_e2e::consts::THROTTLE_APPLIED_LOG, + ) + .await + .expect("the verify mover must apply moverDefaults.throttle on its own connect (#374)"); let spec = verify_work_spec_json(&client, selector).await; let quick = spec .pointer("/operation/verify/tier/quick") diff --git a/crates/mover/src/main.rs b/crates/mover/src/main.rs index f7b4df65..0b87c606 100644 --- a/crates/mover/src/main.rs +++ b/crates/mover/src/main.rs @@ -41,15 +41,15 @@ use kopiur_mover::resolve::{match_current_manifest, matches_source}; use kopiur_mover::serve::ServerWorkSpec; use kopiur_mover::status::{ SnapshotReplicationRunStats, StatusReporter, StatusUpdate, lease_blocked_body, - maintenance_failed_body, maintenance_ran_body, replicate_failed_body, replicate_ok_body, - snapshot_replicate_failed_body, snapshot_replicate_ok_body, split_api_version, - verify_failed_body, verify_ok_body, + maintenance_failed_body, maintenance_failed_body_from_mover, maintenance_ran_body, + replicate_failed_body, replicate_ok_body, snapshot_replicate_failed_body, + snapshot_replicate_ok_body, split_api_version, verify_failed_body, verify_ok_body, }; use kopiur_mover::workspec::{ self, BootstrapRepositoryOp, BrowseSessionOp, KOPIA_KEEP_MAX, KOPIUR_PIN_NAME, MaintenanceOp, MoverWorkSpec, Operation, ReplicateOp, RestoreOp, RestoreSelection, RestoreSelector, - SnapshotAnchor, SnapshotDeleteBatchOp, SnapshotPinOp, SnapshotReplicateOp, VerifyOp, - VerifyTier, maintenance_restamp_target, + SnapshotAnchor, SnapshotDeleteBatchOp, SnapshotPinOp, SnapshotReplicateOp, ThrottleSpec, + VerifyOp, VerifyTier, maintenance_restamp_target, }; #[cfg(test)] use kopiur_mover::workspec::{SnapshotDeleteItem, SnapshotDeleteOp}; @@ -264,7 +264,25 @@ async fn run(cli: &MoverCli) -> Result<()> { Operation::BrowseSession(op) => { run_browse_session_flow(&client, &spec, op, &connect).await } - _ => { + // Everything else shares ONE lifecycle: connect, apply the + // repository throttle, then run with periodic progress PATCHes. + // + // Listed variant by variant, NOT `_ =>`. The catch-all that used to + // stand here is exactly how #374 shipped: every flow above owns its + // own connect, and because the throttle lived only in this arm, a + // `moverDefaults.throttle` the controller faithfully put on the wire + // was silently inert for six of them. A new operation must now + // decide its throttle story to compile. + // + // `SnapshotDeleteBatch` and `SnapshotPin` reach here with a + // `Default` (empty) throttle from their controllers, so + // `apply_repository_throttle` is a no-op for them today — that is a + // controller-side gap, deliberately out of scope for this change. + Operation::Snapshot(_) + | Operation::Restore(_) + | Operation::SnapshotDelete(_) + | Operation::SnapshotDeleteBatch(_) + | Operation::SnapshotPin(_) => { // A best-effort status reporter. If we cannot build a kube client // (e.g. running outside a cluster), we log instead of failing. // SnapshotDeleteBatch's targetRef names the Job itself — nothing @@ -290,23 +308,9 @@ async fn run(cli: &MoverCli) -> Result<()> { .await } Ok(()) => { - // Apply repository throttle (moverDefaults.throttle, ADR-0005 - // §13(e)) after connecting, before the data op. A throttle - // failure is terminal: an un-throttled run could saturate the - // link the user explicitly capped. - if !spec.throttle.is_empty() - && let Err(e) = client - .repository_throttle_set(&spec.throttle.to_kopia()) - .await - { - return terminal_failure( - &reporter, - MoverError::Kopia { - op: KopiaOp::ThrottleSet, - source: e, - }, - ) - .await; + // Throttle right after the connect, before the data op. + if let Err(e) = apply_repository_throttle(&client, &spec.throttle).await { + return terminal_failure(&reporter, e).await; } match execute(&client, &spec, &reporter).await { Ok(update) => { @@ -337,6 +341,56 @@ async fn run(cli: &MoverCli) -> Result<()> { result } +/// Apply the work spec's [`ThrottleSpec`] (`moverDefaults.throttle`, ADR-0005 +/// §13(e)) to the repository connection the caller just opened. +/// +/// **The invariant: every mover flow that connects a repository calls this +/// immediately after its connect, no exceptions.** #374 shipped because the +/// `throttle set` lived in ONE connect arm of `run()`, while bootstrap, seed, +/// maintenance, verify and replicate each open their own connection — so a cap +/// the user configured and the controller faithfully put on the wire was inert +/// for all of them. kopia's limits are per-CONNECTION (persisted into that +/// connection's client config), so a second connect under a second config needs +/// its own call; there is no repository-wide setting to inherit. +/// +/// No limits set → no kopia process is spawned (the common case stays free). +/// +/// A failure is TERMINAL for the run, never best-effort: continuing would +/// saturate exactly the link the user capped, which is worse than not running. +/// +/// The `info!` line is the observable proof the limits reached kopia — the e2e +/// scenarios assert on it, and its ABSENCE is the version-skew signal for a new +/// controller driving an old mover image. Emitted only after kopia accepted +/// them, so it can never claim a throttle that is not in force. +/// +/// Applied on read-only connections too. `repository throttle set` registers a +/// kopia *write* action, but is empirically accepted on a `--readonly` connect +/// against kopia 0.23.1 — pinned by +/// `crates/kopia/tests/integration_set_client_throttle.rs`, with `set-parameters` +/// as the control probe that does hard-error there. So no read-write flip is +/// needed anywhere, and `mode: ReadOnly` repositories are throttled like any +/// other. +async fn apply_repository_throttle(client: &KopiaClient, throttle: &ThrottleSpec) -> Result<()> { + if throttle.is_empty() { + return Ok(()); + } + client + .repository_throttle_set(&throttle.to_kopia()) + .await + .map_err(|source| MoverError::Kopia { + op: KopiaOp::ThrottleSet, + source, + })?; + info!( + upload = ?throttle.upload_bytes_per_second, + download = ?throttle.download_bytes_per_second, + read_ops = ?throttle.read_ops_per_second, + write_ops = ?throttle.write_ops_per_second, + "applied repository throttle" + ); + Ok(()) +} + /// Whether `run()`'s generic connect+execute path must hand-select the /// log-only status reporter instead of `StatusReporter::try_new`'s /// best-effort kube-client attempt. There is no compiler-enforced link @@ -1077,6 +1131,32 @@ fn identity_retention_policy( } } +/// Why a bootstrap connect did not leave a usable, throttled connection. +/// +/// The two are kept apart because only ONE of them is a verdict about the +/// backend: [`bootstrap_connect_probe`] reads a `Connect` failure to decide +/// whether this backend holds a repository, is uninitialized (create/seed it), +/// or is simply unreachable. A throttle failure says nothing about any of that +/// — folding it into the connect error would let a `NotFound`-classified +/// throttle failure be read as "empty backend" and initialize a repository. +enum BootstrapConnectFailure { + /// `repository connect` itself failed — the shape the probe classifies. + Connect(KopiaError), + /// The connection opened, but the repository throttle could not be applied. + /// Terminal for the bootstrap, and never a statement about the backend. + Throttle(MoverError), +} + +impl BootstrapConnectFailure { + /// The terminal [`BootstrapResult`] this failure reports to the controller. + fn into_result(self) -> BootstrapResult { + match self { + BootstrapConnectFailure::Connect(e) => BootstrapResult::failed(&e), + BootstrapConnectFailure::Throttle(e) => BootstrapResult::from_mover_error(&e), + } + } +} + /// Connect for bootstrap, honoring [`BootstrapRepositoryOp::read_only`] /// (M6): a `mode: ReadOnly` repository's bootstrap connects with `--readonly` /// (`repository_connect_readonly`) instead of the normal read-write connect — @@ -1084,17 +1164,30 @@ fn identity_retention_policy( /// ReadOnly consumer repo was exactly what let it clobber the primary's /// maintenance owner. Every other mover flow (restore, delete, snapshot) /// stays on the plain read-write connect regardless. +/// +/// The ONE funnel for every bootstrap connect (probe, post-create reconnect, +/// post-seed reconnect), so [`apply_repository_throttle`]'s no-exceptions +/// invariant holds for all of them by construction rather than by three +/// remembered call sites. The `--readonly` probe is throttled too: `throttle +/// set` is accepted on a read-only connect (see `apply_repository_throttle`), +/// and a bootstrap's catalog scan reads real backend traffic that a capped link +/// must not carry unthrottled. async fn bootstrap_connect( client: &KopiaClient, spec: &ConnectSpec, cache: kopiur_kopia::CacheTuning, read_only: bool, -) -> std::result::Result<(), KopiaError> { - if read_only { + throttle: &ThrottleSpec, +) -> std::result::Result<(), BootstrapConnectFailure> { + let connected = if read_only { client.repository_connect_readonly(spec, cache).await } else { client.repository_connect(spec, cache).await - } + }; + connected.map_err(BootstrapConnectFailure::Connect)?; + apply_repository_throttle(client, throttle) + .await + .map_err(BootstrapConnectFailure::Throttle) } /// How a seed step reports failure: `Ok` on success, or the terminal @@ -1473,16 +1566,23 @@ async fn seed_create_and_connect_local( .env_remove(kopiur_kopia::env::CACHE_DIRECTORY_ENV) .env("XDG_CACHE_HOME", &paths.local_xdg) .build(); - match local_client + if let Err(e) = local_client .repository_connect(local_connect, spec.cache) .await { - Ok(()) => Ok(local_client), - Err(e) => { - error!(class = %e.class(), "could not connect the repository a migrate seed writes into"); - Err(Box::new(BootstrapResult::failed(&e))) - } + error!(class = %e.class(), "could not connect the repository a migrate seed writes into"); + return Err(Box::new(BootstrapResult::failed(&e))); } + // THIS repository's own `moverDefaults.throttle`, on its own connection: a + // seed's local client runs under a separate kopia config from the bootstrap + // client the probe threw the throttle at, and kopia's limits are per-config. + // `snapshot migrate` has no speed flags of its own, so the limits persisted + // into this config are the ONLY cap on the heaviest transfer kopiur performs. + // (The seed SOURCE's own cap is a separate knob, wired in a later change.) + apply_repository_throttle(&local_client, &spec.throttle) + .await + .map_err(|e| Box::new(BootstrapResult::from_mover_error(&e)))?; + Ok(local_client) } /// The MANDATORY post-verify for a migrate-mode seed: how many snapshots @@ -1763,15 +1863,19 @@ async fn bootstrap_create_arm( op: &BootstrapRepositoryOp, connect_spec: &ConnectSpec, cache: kopiur_kopia::CacheTuning, + throttle: &ThrottleSpec, ) -> SeedStep<()> { + // `repository create` runs before any connection exists to throttle; it + // writes one format blob, so there is nothing here for a bandwidth cap to + // bound. The reconnect below is throttled like every other connect. if let Err(ce) = client .repository_create(connect_spec, cache, &op.create_options()) .await { return Err(Box::new(BootstrapResult::failed(&ce))); } - if let Err(ce) = bootstrap_connect(client, connect_spec, cache, op.read_only).await { - return Err(Box::new(BootstrapResult::failed(&ce))); + if let Err(ce) = bootstrap_connect(client, connect_spec, cache, op.read_only, throttle).await { + return Err(Box::new(ce.into_result())); } stamp_owner_on_new_repository(client, op.maintenance_owner.as_ref()).await; Ok(()) @@ -1828,8 +1932,10 @@ async fn bootstrap_seed_arm( workspec::SeedModeSpec::Blob => false, workspec::SeedModeSpec::Migrate => !local_initialized, }; - if let Err(ce) = bootstrap_connect(client, connect_spec, cache, op.read_only).await { - return Err(Box::new(BootstrapResult::failed(&ce))); + if let Err(ce) = + bootstrap_connect(client, connect_spec, cache, op.read_only, &spec.throttle).await + { + return Err(Box::new(ce.into_result())); } if created { stamp_owner_on_new_repository(client, op.maintenance_owner.as_ref()).await; @@ -1866,10 +1972,18 @@ async fn bootstrap_connect_probe( connect_spec: &ConnectSpec, cache: kopiur_kopia::CacheTuning, op: &BootstrapRepositoryOp, -) -> BootstrapProbe { - let err = bootstrap_connect(client, connect_spec, cache, op.read_only) - .await - .err(); + throttle: &ThrottleSpec, +) -> SeedStep { + // Exhaustive over the failure shapes: a CONNECT failure is the probe's raw + // material (it decides initialized/uninitialized/create/seed), while a + // THROTTLE failure ends the bootstrap here — the connection is open but + // uncapped, and nothing about the backend has been learned that would + // justify creating or seeding a repository over it. + let err = match bootstrap_connect(client, connect_spec, cache, op.read_only, throttle).await { + Ok(()) => None, + Err(BootstrapConnectFailure::Connect(e)) => Some(e), + Err(e @ BootstrapConnectFailure::Throttle(_)) => return Err(Box::new(e.into_result())), + }; let local_initialized = err.is_none(); let uninitialized = err .as_ref() @@ -1895,13 +2009,13 @@ async fn bootstrap_connect_probe( err.as_ref().map(KopiaError::class), uninitialized, ); - BootstrapProbe { + Ok(BootstrapProbe { err, local_initialized, uninitialized, already_initialized, action, - } + }) } /// The bootstrap routine: connect-first (adopt an existing repo), create only @@ -1930,7 +2044,14 @@ async fn run_bootstrap( // maintenance forever. let mut seeded = false; - let probe = bootstrap_connect_probe(client, &connect_spec, cache, op).await; + // A throttle the connection could not accept ends the bootstrap here — the + // probe learned nothing about the backend, so there is no verdict to carry on + // with. + let probe = + match bootstrap_connect_probe(client, &connect_spec, cache, op, &spec.throttle).await { + Ok(p) => p, + Err(result) => return *result, + }; let (connect_err, local_initialized, uninitialized) = (probe.err, probe.local_initialized, probe.uninitialized); // A successful connect over a NON-resuming armed seed is the documented @@ -1960,7 +2081,9 @@ async fn run_bootstrap( class = ?connect_err.as_ref().map(KopiaError::class), "connect failed; attempting repository create" ); - if let Err(result) = bootstrap_create_arm(client, op, &connect_spec, cache).await { + if let Err(result) = + bootstrap_create_arm(client, op, &connect_spec, cache, &spec.throttle).await + { return *result; } created = true; @@ -2347,6 +2470,14 @@ async fn run_maintenance_flow( source: e, }); } + // Cap this connection before any maintenance IO: a full maintenance rewrites + // and drops blobs, so it is exactly the kind of backend traffic + // `moverDefaults.throttle` exists to bound. + if let Err(e) = apply_repository_throttle(client, &spec.throttle).await { + patch_maintenance_status(&spec.target_ref, &maintenance_failed_body_from_mover(&e)).await; + error!(class = %e.kopia_class(), "maintenance repository throttle set failed"); + return Err(e); + } // Assume the STABLE lease-derived client identity before anything else: // this pod's own user@hostname is ephemeral (a fresh pod every run), so @@ -2502,6 +2633,13 @@ async fn run_verify_flow( source: e, }); } + // Verification READS the repository — a deep verify scratch-restores a whole + // snapshot — so it is throttled like any other data flow. + if let Err(e) = apply_repository_throttle(client, &spec.throttle).await { + patch_verify_status(&spec.target_ref, &verify_failed_body(&e.to_string())).await; + error!(class = %e.kopia_class(), "verify repository throttle set failed"); + return Err(e); + } // Run the tier and collect the result environment for successExpr. A kopia // failure is terminal; a clean run yields the stats the predicate inspects. @@ -2746,6 +2884,14 @@ async fn run_replicate_flow( source: e, }); } + // The source repository's own cap, on the connection `repository sync-to` + // reads through — a whole-repository blob mirror is the single heaviest read + // this flow performs, and the source is often the link the user capped. + if let Err(e) = apply_repository_throttle(client, &spec.throttle).await { + patch_replicate_status(&spec.target_ref, &replicate_failed_body(&e.to_string())).await; + error!(class = %e.kopia_class(), "replication repository throttle set failed"); + return Err(e); + } // The destination's credentials arrive under the KOPIUR_DEST_ env prefix so they // never collide with the source's identically named ones (issue #200). Read them @@ -3351,6 +3497,12 @@ async fn run_browse_session_flow( source: e, }); } + // NO throttle here, and not by oversight: a browse session's work spec is + // built by the CLI (`crates/cli/src/cmd/browse/session.rs`) with + // `throttle: Default::default()`, so `spec.throttle` is structurally always + // empty and the call would be a guaranteed no-op. The session's reads run + // through `kopia` invocations the CLI execs into this pod; giving them a cap + // means teaching the CLI to resolve the repository's `moverDefaults` first. // Signal readiness: the marker flips the pod Ready so the CLI knows the // session is exec-able. A marker that cannot be written would leave the @@ -3640,37 +3792,163 @@ mod tests { assert!(!wants_log_only_reporter(&delete)); } - // --- delete_one / delete_batch against a fake kopia binary --- + // --- a fake kopia binary, shared by the shim-driven test modules below --- // // Mirrors `crates/kopia/tests/fake_shim.rs`: a tiny shell script stands in - // for kopia so the self-heal gate and the attempt-all-then-fail loop are - // exercised through the real `KopiaClient` subprocess path with no real - // kopia binary. + // for kopia so a code path is exercised through the real `KopiaClient` + // subprocess path with no real kopia binary. #[cfg(unix)] - mod delete_shim_tests { + struct Shim { + _dir: tempfile::TempDir, + path: PathBuf, + } + + #[cfg(unix)] + fn shim(script: &str) -> Shim { use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("kopia-shim.sh"); + std::fs::write(&path, script).unwrap(); + let mut perms = std::fs::metadata(&path).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&path, perms).unwrap(); + Shim { _dir: dir, path } + } + + #[cfg(unix)] + fn client_for(shim: &Shim) -> KopiaClient { + KopiaClient::builder().binary(shim.path.clone()).build() + } + + // --- #374: the repository throttle every own-connect flow now applies --- + #[cfg(unix)] + mod throttle_tests { use super::*; - struct Shim { - _dir: tempfile::TempDir, - path: PathBuf, + fn full_throttle() -> ThrottleSpec { + ThrottleSpec { + upload_bytes_per_second: Some(1_000), + download_bytes_per_second: Some(2_000), + read_ops_per_second: Some(3), + write_ops_per_second: Some(4), + } } - fn shim(script: &str) -> Shim { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("kopia-shim.sh"); - std::fs::write(&path, script).unwrap(); - let mut perms = std::fs::metadata(&path).unwrap().permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&path, perms).unwrap(); - Shim { _dir: dir, path } + /// An empty throttle must not spawn kopia at all: the common case (no + /// `moverDefaults.throttle`) now runs on EVERY connect, so a wasted + /// subprocess per connect would be the cost of the fix. + #[tokio::test] + async fn empty_throttle_never_invokes_kopia() { + let marker_dir = tempfile::tempdir().unwrap(); + let marker = marker_dir.path().join("invoked"); + let s = shim(&format!( + "#!/bin/sh\ntouch \"{}\"\nexit 0\n", + marker.display() + )); + apply_repository_throttle(&client_for(&s), &ThrottleSpec::default()) + .await + .expect("an empty throttle is a no-op"); + assert!( + !marker.exists(), + "an empty throttle must not spawn a kopia process" + ); } - fn client_for(shim: &Shim) -> KopiaClient { - KopiaClient::builder().binary(shim.path.clone()).build() + /// Every configured limit reaches kopia's argv. The controller resolves + /// all four knobs; dropping one on the floor here would be #374 again, + /// one field at a time. + #[tokio::test] + async fn every_limit_reaches_kopia_argv() { + let argv_dir = tempfile::tempdir().unwrap(); + let argv = argv_dir.path().join("argv"); + let s = shim(&format!( + "#!/bin/sh\necho \"$*\" > \"{}\"\nexit 0\n", + argv.display() + )); + apply_repository_throttle(&client_for(&s), &full_throttle()) + .await + .expect("throttle set succeeds"); + let recorded = std::fs::read_to_string(&argv).expect("the shim recorded its argv"); + for expected in [ + "repository throttle set", + "--upload-bytes-per-second 1000", + "--download-bytes-per-second 2000", + "--read-requests-per-second 3", + "--write-requests-per-second 4", + ] { + assert!( + recorded.contains(expected), + "kopia argv must carry `{expected}`; got: {recorded}" + ); + } + } + + /// A throttle kopia refused is TERMINAL and labeled `ThrottleSet` — the + /// run must not continue uncapped over the link the user capped, and the + /// persisted failure must name the invocation that failed rather than + /// the connect that preceded it. + #[tokio::test] + async fn a_refused_throttle_is_terminal_and_labeled() { + let s = shim("#!/bin/sh\necho 'nope' >&2\nexit 1\n"); + let err = apply_repository_throttle(&client_for(&s), &full_throttle()) + .await + .expect_err("a failing throttle set must surface"); + assert!( + matches!( + err, + MoverError::Kopia { + op: KopiaOp::ThrottleSet, + .. + } + ), + "expected a ThrottleSet kopia failure, got {err:?}" + ); } + /// A throttle failure during bootstrap must never be reported as a + /// CONNECT verdict. `bootstrap_connect_probe` reads a connect failure to + /// decide whether to CREATE or SEED a repository over the backend; a + /// throttle failure says nothing about the backend, and conflating them + /// could initialize a repository on the strength of an unrelated error. + #[test] + fn connect_and_throttle_failures_report_differently() { + let kopia_err = KopiaError::NonZeroExit { + args: "repository connect filesystem".into(), + code: Some(1), + class: KopiaErrorClass::NotFound, + stderr_tail: "repository not initialized".into(), + }; + let connect = BootstrapConnectFailure::Connect(kopia_err).into_result(); + let cf = connect.failure.expect("a connect failure is reported"); + assert_eq!(cf.kopia_error_class, "NotFound"); + assert_eq!(cf.op, None, "a bare connect error carries no op label"); + + let throttled = BootstrapConnectFailure::Throttle(MoverError::Kopia { + op: KopiaOp::ThrottleSet, + source: KopiaError::NonZeroExit { + args: "repository throttle set".into(), + code: Some(1), + class: KopiaErrorClass::RepositoryUnavailable, + stderr_tail: "backend went away".into(), + }, + }) + .into_result(); + let tf = throttled.failure.expect("a throttle failure is reported"); + assert_eq!(tf.kopia_error_class, "RepositoryUnavailable"); + assert_eq!( + tf.op.as_deref(), + Some(KopiaOp::ThrottleSet.as_str()), + "a throttle failure must name the throttle invocation, not the connect" + ); + } + } + + // --- delete_one / delete_batch against a fake kopia binary --- + #[cfg(unix)] + mod delete_shim_tests { + use super::*; + #[tokio::test] async fn delete_one_skips_self_heal_when_anchor_has_no_start_time() { // Regression (data-loss fix, adversarial review): an anchor with a diff --git a/crates/mover/src/status.rs b/crates/mover/src/status.rs index dd3a701b..3d5304ce 100644 --- a/crates/mover/src/status.rs +++ b/crates/mover/src/status.rs @@ -766,12 +766,31 @@ pub fn lease_blocked_body(owner: &str, reason: &str, message: &str) -> serde_jso /// `{ "status": ... }` body for a failed kopia maintenance call. pub fn maintenance_failed_body(e: &KopiaError) -> serde_json::Value { + maintenance_failure_body(e.class(), &e.to_string()) +} + +/// [`maintenance_failed_body`] for a maintenance step whose typed cause is a +/// [`MoverError`] rather than a bare [`KopiaError`] — the repository throttle +/// applied right after the maintenance connect. Same condition shape; the class +/// comes from the error's own classification +/// ([`MoverError::kopia_class`](crate::error::MoverError::kopia_class)) so the +/// controller's retry hint cannot drift from the message. +pub fn maintenance_failed_body_from_mover(e: &MoverError) -> serde_json::Value { + maintenance_failure_body(e.kopia_class(), &e.to_string()) +} + +/// The one `MaintenanceFailed` condition body, so the two constructors above +/// cannot drift in reason, wording or timestamp shape. +fn maintenance_failure_body( + class: kopiur_kopia::KopiaErrorClass, + message: &str, +) -> serde_json::Value { serde_json::json!({ "status": { "conditions": [lease_condition_body( "False", "MaintenanceFailed", - &format!("maintenance failed (class {}): {e}", e.class()), + &format!("maintenance failed (class {class}): {message}"), &chrono::Utc::now(), )], } @@ -1008,6 +1027,48 @@ mod tests { "2026-06-01T12:00:00Z".parse().unwrap() } + /// #374: a maintenance run can now fail on the repository THROTTLE, whose + /// typed cause is a `MoverError`. Both constructors must produce the same + /// `MaintenanceFailed` condition shape and carry the real class, so the + /// controller's retry hint is identical whichever step failed. + #[test] + fn both_maintenance_failure_bodies_share_one_condition_shape() { + let kopia = KopiaError::NonZeroExit { + args: "maintenance run".into(), + code: Some(1), + class: KopiaErrorClass::Locked, + stderr_tail: "repository is locked".into(), + }; + let from_kopia = maintenance_failed_body(&kopia); + let from_mover = maintenance_failed_body_from_mover(&crate::error::MoverError::Kopia { + op: crate::error::KopiaOp::ThrottleSet, + source: KopiaError::NonZeroExit { + args: "repository throttle set".into(), + code: Some(1), + class: KopiaErrorClass::Locked, + stderr_tail: "repository is locked".into(), + }, + }); + for body in [&from_kopia, &from_mover] { + let cond = &body["status"]["conditions"][0]; + assert_eq!(cond["type"], kopiur_api::maintenance::LEASE_OWNED_CONDITION); + assert_eq!(cond["status"], "False"); + assert_eq!(cond["reason"], "MaintenanceFailed"); + assert!( + cond["message"] + .as_str() + .is_some_and(|m| m.contains("class Locked")), + "the condition message must carry the real kopia class: {cond}" + ); + } + assert!( + from_mover["status"]["conditions"][0]["message"] + .as_str() + .is_some_and(|m| m.contains("repository throttle set")), + "a throttle failure must name the invocation that failed" + ); + } + #[test] fn stats_from_create_result_uses_crd_field_names() { let json = r#"{ From 393859c781e7807d3316306fd949b02ea6743b87 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 12:04:50 -0700 Subject: [PATCH 08/15] refactor(mover): fuse repository connect and throttle into one funnel (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up to 869cf8f. Two findings, both about the fix being remembered rather than enforced. **The six own-connect flows were still remembered call sites.** Deleting `apply_repository_throttle` from `run_verify_flow` compiled and passed every hermetic gate — the run just went uncapped — which is the structure that caused #374 in the first place, one layer up. Replaced with a `connect_and_throttle` funnel (mirroring the `bootstrap_connect` one this PR already added): the generic `run()` arm, maintenance, verify, replicate and a seed's local repository now obtain a connection and its cap as ONE operation, so a future flow cannot get the first without the second. Its `ConnectFailure` keeps the connect and throttle halves distinct and labels the connect half with the calling flow's own `KopiaOp`, so a status block still names the invocation that actually failed. Nothing in the type system stops a new flow from calling `repository_connect` directly, so `connect_funnel_is_the_only_way_flows_connect` ratchets the count of direct connects left in the file against a documented allowlist (the two funnels plus the four connects that genuinely carry no `MoverWorkSpec.throttle`: serve, browse, snapshot-replication's dual connect, a seed's source). Verified to fire: adding one bypassing connect fails it 10 vs 9. **Seed-migrate's local connect had no coverage of any kind** — no hermetic test reaches `seed_create_and_connect_local`, and the e2e table covered bootstrap/maintenance/verify/replicate only, so the highest-value call site of the change ("the heaviest transfer kopiur performs") could have been deleted and stayed green forever. The migrate-seed scenario now sets `moverDefaults.throttle` (read back, so a dropped overlay cannot make it vacuous) and asserts the mover's log line appears at least TWICE in a SINGLE pod: a seeding bootstrap caps two connections (the seed's own local kopia config — the only cap `snapshot migrate` can honor — and the post-seed reconnect). Per-pod counting is load-bearing: the same selector also matches later catalog-rescan pods that log it once each, so a bare `contains`, or a count across pods, would pass with the seed-local call removed. `wait_for_pod_log` is now the `times = 1` case of `wait_for_pod_log_times`. Also, from the review's minor ledger: the helper's doc no longer claims a no-exceptions invariant it cannot hold — it carries the exception list with each reason and owning milestone; the generic arm's throttle failure now flows into `result` instead of returning early past `metrics.record`/`shutdown`; and the pod-log helper retains the last per-pod log error so a poll that times out reports why rather than just "not found". --- crates/e2e/src/wait.rs | 81 ++++++++--- crates/e2e/tests/seed.rs | 47 ++++++ crates/mover/src/main.rs | 300 ++++++++++++++++++++++++++++----------- 3 files changed, 323 insertions(+), 105 deletions(-) diff --git a/crates/e2e/src/wait.rs b/crates/e2e/src/wait.rs index 4a082650..585dc343 100644 --- a/crates/e2e/src/wait.rs +++ b/crates/e2e/src/wait.rs @@ -71,26 +71,34 @@ pub async fn pod_succeeded(client: &Client, ns: &str, name: &str) -> Result<()> Err(anyhow!("pod {ns}/{name} ended {phase}; logs:\n{logs}")) } -/// Concatenated stdout of every non-terminating Pod matching `selector` in `ns`. +/// Per-pod stdout for every non-terminating Pod matching `selector` in `ns`, +/// plus the LAST per-pod log error, if any. /// /// Mover Jobs carry their component/instance labels on the POD template as well /// as the Job, so the same selector a scenario uses to find a mover Job finds -/// the pod that ran it. Pods that have not produced logs yet (still `Pending`, -/// or already reaped) are skipped rather than failing the call, so this composes -/// with [`crate::wait_until`]: poll until the line you are looking for shows up. +/// the pod that ran it. /// -/// Every matching pod is concatenated on purpose — a retried Job has several, -/// and the assertion is about what the mover DID, not about which attempt did it. -pub async fn pod_logs_for_selector( +/// Kept per-pod rather than concatenated because "how many times did ONE mover +/// log this" is a real question: a seeding bootstrap applies the repository +/// throttle to two different connections, and only a per-pod count can tell that +/// apart from two pods that each applied it once. +/// +/// A pod whose logs cannot be read yet (still `Pending`/`ContainerCreating`) is +/// skipped rather than failing the whole call — one such pod left over from an +/// earlier attempt must not hide a good pod's logs — but its error is RETURNED +/// alongside, so a caller polling to a timeout can report why it never saw what +/// it wanted instead of just "not found". +pub async fn pod_logs_by_pod( client: &Client, ns: &str, selector: &str, -) -> std::result::Result { +) -> std::result::Result<(Vec<(String, String)>, Option), kube::Error> { let api: Api = Api::namespaced(client.clone(), ns); let list = api .list(&kube::api::ListParams::default().labels(selector)) .await?; - let mut out = String::new(); + let mut out = Vec::new(); + let mut last_err = None; for pod in list .items .into_iter() @@ -99,40 +107,75 @@ pub async fn pod_logs_for_selector( let Some(name) = pod.metadata.name else { continue; }; - if let Ok(logs) = api.logs(&name, &LogParams::default()).await { - out.push_str(&logs); + match api.logs(&name, &LogParams::default()).await { + Ok(logs) => out.push((name, logs)), + Err(e) => last_err = Some(e), } } - Ok(out) + Ok((out, last_err)) +} + +/// Concatenated stdout of every non-terminating Pod matching `selector` in `ns`. +/// The whole-population view over [`pod_logs_by_pod`], for assertions about what +/// SOME mover did rather than which attempt did it. +pub async fn pod_logs_for_selector( + client: &Client, + ns: &str, + selector: &str, +) -> std::result::Result { + let (logs, _) = pod_logs_by_pod(client, ns, selector).await?; + Ok(logs.into_iter().map(|(_, l)| l).collect()) } -/// Poll until some pod matching `selector` has logged `needle`. +/// Poll until some single pod matching `selector` has logged `needle` at least +/// `times`. /// /// The assertion form for "the mover actually did X" when X leaves no trace in /// any CR status — e.g. the repository throttle (#374), whose only observable is /// [`crate::consts::THROTTLE_APPLIED_LOG`]. Polls rather than reading once, /// because the pod may not exist (or may still be starting) when the caller /// first looks. -pub async fn wait_for_pod_log( +/// +/// `times > 1` is what makes a multi-connection flow's assertion non-vacuous: +/// counting PER POD means a second pod that logged the line once cannot stand in +/// for a second connection the one pod under test never capped. +pub async fn wait_for_pod_log_times( client: &Client, ns: &str, selector: &str, needle: &str, + times: usize, ) -> Result<()> { crate::wait_until( - &format!("a pod matching `{selector}` logs `{needle}`"), + &format!("a pod matching `{selector}` logs `{needle}` at least {times}x"), default_timeout(), crate::poll_interval(), || async { - Ok(pod_logs_for_selector(client, ns, selector) - .await? - .contains(needle) - .then_some(())) + let (logs, last_err) = pod_logs_by_pod(client, ns, selector).await?; + if logs.iter().any(|(_, l)| l.matches(needle).count() >= times) { + return Ok(Some(())); + } + // Never let a log-fetch blip mask a match, but do carry it to the + // timeout message when there was no match to report. + match last_err { + Some(e) => Err(e), + None => Ok(None), + } }, ) .await } +/// [`wait_for_pod_log_times`] for the ordinary "logged it at all" case. +pub async fn wait_for_pod_log( + client: &Client, + ns: &str, + selector: &str, + needle: &str, +) -> Result<()> { + wait_for_pod_log_times(client, ns, selector, needle, 1).await +} + /// Fetch a Pod's logs (best-effort context for failure messages). pub async fn pod_logs(client: &Client, ns: &str, name: &str) -> Result { let api: Api = Api::namespaced(client.clone(), ns); diff --git a/crates/e2e/tests/seed.rs b/crates/e2e/tests/seed.rs index d49ace5a..3a347fa2 100644 --- a/crates/e2e/tests/seed.rs +++ b/crates/e2e/tests/seed.rs @@ -553,6 +553,20 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident // itself, and a seed-armed bootstrap never takes the create // fallback — so reaching Ready here proves both at once. "create": { "enabled": false }, + // #374: a migrate seed opens TWO connections to this repository + // under two kopia configs — the seed's own local client (which + // `snapshot migrate` reopens, and which has no speed flags of its + // own, making this the ONLY cap on the heaviest transfer kopiur + // performs) and the post-seed bootstrap reconnect. Both must be + // capped; the assertion below counts the mover's log line per pod + // to prove the seed-local one specifically fired. Non-binding at + // 100 MiB/s over a local hostPath. + "moverDefaults": { + "throttle": { + "uploadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND, + "downloadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND + } + }, "seed": { "from": { "repository": { "kind": "Repository", "name": SOURCE } }, "migrate": { "parallel": 2 } @@ -571,6 +585,18 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident Some(SOURCE), "spec.seed.from.repository must have survived admission" ); + // Bare spec fields, not a `{"spec": ...}` wrapper — read the throttle back + // too, or the log-count assertion below would pass vacuously against a repo + // that never carried a cap. + assert_eq!( + live.spec + .mover_defaults + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.upload_bytes_per_second), + Some(consts::THROTTLE_BYTES_PER_SECOND), + "moverDefaults.throttle must survive onto the seeded Repository" + ); wait_seeded(&repos, SEEDED, "True", "Seeded").await; wait_phase(&repos, SEEDED, "Ready") @@ -593,6 +619,27 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident "a migrate seed must report the manifests it copied; status.seed: {seed_status}" ); + // #374 regression guard for the highest-value call site of the fix: the + // seed's LOCAL repository connect. This one bootstrap pod opens two capped + // connections — the seed's local client (under its own kopia config, the + // only cap `snapshot migrate` can honor) and the post-seed reconnect — so it + // must log the line TWICE. Counting per pod is load-bearing: the same + // selector also matches later catalog-rescan/probe pods that log it once + // each, and a bare `contains` (or a count across pods) would stay green with + // the seed-local call deleted. + kopiur_e2e::wait::wait_for_pod_log_times( + &client, + E2E_NAMESPACE, + &format!("kopiur.home-operations.com/repository={SEEDED}"), + consts::THROTTLE_APPLIED_LOG, + 2, + ) + .await + .expect( + "the seeding mover must cap BOTH the seed's local connect and the post-seed \ + reconnect (#374); one occurrence means the seed-local throttle is gone", + ); + // Identity survives the migrate, so a restore keyed on the SOURCE's // identity finds the history in the SEEDED repository. let restores: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); diff --git a/crates/mover/src/main.rs b/crates/mover/src/main.rs index 0b87c606..25b5b813 100644 --- a/crates/mover/src/main.rs +++ b/crates/mover/src/main.rs @@ -296,34 +296,23 @@ async fn run(cli: &MoverCli) -> Result<()> { } else { StatusReporter::try_new(&spec).await }; - match client.repository_connect(&connect, spec.cache).await { + // Connect AND throttle as one operation — see `connect_and_throttle`. + match connect_and_throttle(&client, &connect, spec.cache, &spec.throttle).await { Err(e) => { - terminal_failure( - &reporter, - MoverError::Kopia { - op: KopiaOp::RepositoryConnect, - source: e, - }, - ) - .await + terminal_failure(&reporter, e.into_mover_error(KopiaOp::RepositoryConnect)) + .await } - Ok(()) => { - // Throttle right after the connect, before the data op. - if let Err(e) = apply_repository_throttle(&client, &spec.throttle).await { - return terminal_failure(&reporter, e).await; - } - match execute(&client, &spec, &reporter).await { - Ok(update) => { - reporter.report(&update).await; - info!( - phase = update.phase.as_deref().unwrap_or("done"), - "operation succeeded" - ); - Ok(()) - } - Err(e) => terminal_failure(&reporter, e).await, + Ok(()) => match execute(&client, &spec, &reporter).await { + Ok(update) => { + reporter.report(&update).await; + info!( + phase = update.phase.as_deref().unwrap_or("done"), + "operation succeeded" + ); + Ok(()) } - } + Err(e) => terminal_failure(&reporter, e).await, + }, } } }, @@ -341,17 +330,98 @@ async fn run(cli: &MoverCli) -> Result<()> { result } +/// Why a repository connection is not usable for work: it never opened, or it +/// opened UNCAPPED. Mirrors [`BootstrapConnectFailure`] and for the same reason +/// — the two are different facts and some callers must tell them apart — but +/// unlike bootstrap's, this one is generic over the calling flow's connect op +/// label. +enum ConnectFailure { + /// `repository connect` itself failed. + Connect(KopiaError), + /// The connection opened, but `repository throttle set` failed. Already + /// carries [`KopiaOp::ThrottleSet`]. + Throttle(MoverError), +} + +impl ConnectFailure { + /// The typed mover failure, labeled with the CALLING flow's connect op + /// (`MaintenanceConnect`, `VerifyConnect`, `ReplicateConnect`, …). A + /// throttle failure passes through with its own label intact, so a status + /// block always names the invocation that actually failed. + fn into_mover_error(self, connect_op: KopiaOp) -> MoverError { + match self { + ConnectFailure::Connect(source) => MoverError::Kopia { + op: connect_op, + source, + }, + ConnectFailure::Throttle(e) => e, + } + } +} + +/// Open a repository connection **and** apply its throttle, as ONE operation. +/// +/// This is the funnel every flow that owns its own read-write connect goes +/// through (the generic `run()` arm, maintenance, verify, replicate, and a +/// seed's local repository); bootstrap has its own funnel, +/// [`bootstrap_connect`], because its connect failure additionally feeds a +/// create-vs-seed verdict. +/// +/// **Why a funnel rather than two calls.** #374 was not a missing feature, it +/// was six remembered call sites: `repository_connect` and `throttle set` were +/// separate steps, so a flow could — and did — do the first and forget the +/// second, and every test still passed because the run succeeded uncapped. +/// Fusing them means a future flow cannot obtain a connection without also +/// obtaining its cap. `connect_funnel_is_the_only_way_flows_connect` (below) +/// makes an attempt to bypass it loud. +async fn connect_and_throttle( + client: &KopiaClient, + connect: &ConnectSpec, + cache: kopiur_kopia::CacheTuning, + throttle: &ThrottleSpec, +) -> std::result::Result<(), ConnectFailure> { + client + .repository_connect(connect, cache) + .await + .map_err(ConnectFailure::Connect)?; + apply_repository_throttle(client, throttle) + .await + .map_err(ConnectFailure::Throttle) +} + /// Apply the work spec's [`ThrottleSpec`] (`moverDefaults.throttle`, ADR-0005 /// §13(e)) to the repository connection the caller just opened. /// -/// **The invariant: every mover flow that connects a repository calls this -/// immediately after its connect, no exceptions.** #374 shipped because the -/// `throttle set` lived in ONE connect arm of `run()`, while bootstrap, seed, -/// maintenance, verify and replicate each open their own connection — so a cap -/// the user configured and the controller faithfully put on the wire was inert -/// for all of them. kopia's limits are per-CONNECTION (persisted into that -/// connection's client config), so a second connect under a second config needs -/// its own call; there is no repository-wide setting to inherit. +/// **The invariant: every mover flow that connects a repository applies the +/// throttle to that connection.** Reached through one of the two funnels — +/// [`connect_and_throttle`] or [`bootstrap_connect`] — never by a flow +/// remembering to call this itself. #374 shipped because the `throttle set` +/// lived in ONE connect arm of `run()`, while bootstrap, seed, maintenance, +/// verify and replicate each open their own connection — so a cap the user +/// configured and the controller faithfully put on the wire was inert for all +/// of them. kopia's limits are per-CONNECTION (persisted into that connection's +/// client config), so a second connect under a second config needs its own +/// call; there is no repository-wide setting to inherit. +/// +/// The connects that do NOT go through a funnel, each for a stated reason — +/// keep this list in step with `connect_funnel_is_the_only_way_flows_connect`: +/// +/// * **`kopiur-mover serve`** (the browse/UI kopia server) runs off a +/// [`ServerWorkSpec`], which carries no throttle field at all. Nothing to +/// apply until that wire type grows one. +/// * **browse session** — its work spec is built by the CLI with +/// `throttle: Default::default()`, so `spec.throttle` is structurally always +/// empty; capping it means teaching the CLI to resolve `moverDefaults` first. +/// * **snapshot-replication source and destination** — a dual connect under two +/// kopia configs, with per-CR per-side overrides; wired by the next milestone +/// (M3c), which adds the destination's own `destinationThrottle`. +/// * **a seed's SOURCE (replica) connect** — same shape, wired by M3d together +/// with its `replica_throttle`. +/// +/// No limits set → no kopia process is spawned (the common case stays free). +/// +/// A failure is TERMINAL for the run, never best-effort: continuing would +/// saturate exactly the link the user capped, which is worse than not running. /// /// No limits set → no kopia process is spawned (the common case stays free). /// @@ -361,7 +431,10 @@ async fn run(cli: &MoverCli) -> Result<()> { /// The `info!` line is the observable proof the limits reached kopia — the e2e /// scenarios assert on it, and its ABSENCE is the version-skew signal for a new /// controller driving an old mover image. Emitted only after kopia accepted -/// them, so it can never claim a throttle that is not in force. +/// them, so it can never claim a throttle that is not in force. One line per +/// APPLICATION, not per run: a flow that opens two connections (a seed: the +/// bootstrap's and the local repository's) logs it once per connection, which is +/// what lets an e2e assert that both were capped. /// /// Applied on read-only connections too. `repository throttle set` registers a /// kopia *write* action, but is empirically accepted on a `--readonly` connect @@ -1566,22 +1639,20 @@ async fn seed_create_and_connect_local( .env_remove(kopiur_kopia::env::CACHE_DIRECTORY_ENV) .env("XDG_CACHE_HOME", &paths.local_xdg) .build(); - if let Err(e) = local_client - .repository_connect(local_connect, spec.cache) - .await + // Connect AND cap, as one operation. THIS repository's own + // `moverDefaults.throttle` goes on its own connection: a seed's local client + // runs under a separate kopia config from the bootstrap client the probe + // capped, and kopia's limits are per-config. `snapshot migrate` has no speed + // flags of its own, so the limits persisted into this config are the ONLY cap + // on the heaviest transfer kopiur performs. (The seed SOURCE's own cap is a + // separate knob, wired in a later change.) + if let Err(e) = + connect_and_throttle(&local_client, local_connect, spec.cache, &spec.throttle).await { - error!(class = %e.class(), "could not connect the repository a migrate seed writes into"); - return Err(Box::new(BootstrapResult::failed(&e))); + let e = e.into_mover_error(KopiaOp::SeedLocalConnect); + error!(class = %e.kopia_class(), "could not open a capped connection to the repository a migrate seed writes into"); + return Err(Box::new(BootstrapResult::from_mover_error(&e))); } - // THIS repository's own `moverDefaults.throttle`, on its own connection: a - // seed's local client runs under a separate kopia config from the bootstrap - // client the probe threw the throttle at, and kopia's limits are per-config. - // `snapshot migrate` has no speed flags of its own, so the limits persisted - // into this config are the ONLY cap on the heaviest transfer kopiur performs. - // (The seed SOURCE's own cap is a separate knob, wired in a later change.) - apply_repository_throttle(&local_client, &spec.throttle) - .await - .map_err(|e| Box::new(BootstrapResult::from_mover_error(&e)))?; Ok(local_client) } @@ -2462,20 +2533,13 @@ async fn run_maintenance_flow( ); // Connect first: for object stores this pod is the only place with repo // access, which is exactly why the lease decision is made here. - if let Err(e) = client.repository_connect(connect, spec.cache).await { - patch_maintenance_status(&spec.target_ref, &maintenance_failed_body(&e)).await; - error!(class = %e.class(), "maintenance connect failed"); - return Err(MoverError::Kopia { - op: KopiaOp::MaintenanceConnect, - source: e, - }); - } - // Cap this connection before any maintenance IO: a full maintenance rewrites - // and drops blobs, so it is exactly the kind of backend traffic - // `moverDefaults.throttle` exists to bound. - if let Err(e) = apply_repository_throttle(client, &spec.throttle).await { + // Connect AND cap, as one operation (`connect_and_throttle`): a full + // maintenance rewrites and drops blobs, so it is exactly the kind of backend + // traffic `moverDefaults.throttle` exists to bound. + if let Err(e) = connect_and_throttle(client, connect, spec.cache, &spec.throttle).await { + let e = e.into_mover_error(KopiaOp::MaintenanceConnect); patch_maintenance_status(&spec.target_ref, &maintenance_failed_body_from_mover(&e)).await; - error!(class = %e.kopia_class(), "maintenance repository throttle set failed"); + error!(class = %e.kopia_class(), "maintenance connect failed"); return Err(e); } @@ -2625,19 +2689,13 @@ async fn run_verify_flow( policy = %spec.target_ref.name, "running verification" ); - if let Err(e) = client.repository_connect(connect, spec.cache).await { + // Connect AND cap, as one operation: verification READS the repository — a + // deep verify scratch-restores a whole snapshot — so it is throttled like + // any other data flow. + if let Err(e) = connect_and_throttle(client, connect, spec.cache, &spec.throttle).await { + let e = e.into_mover_error(KopiaOp::VerifyConnect); patch_verify_status(&spec.target_ref, &verify_failed_body(&e.to_string())).await; - error!(class = %e.class(), "verify connect failed"); - return Err(MoverError::Kopia { - op: KopiaOp::VerifyConnect, - source: e, - }); - } - // Verification READS the repository — a deep verify scratch-restores a whole - // snapshot — so it is throttled like any other data flow. - if let Err(e) = apply_repository_throttle(client, &spec.throttle).await { - patch_verify_status(&spec.target_ref, &verify_failed_body(&e.to_string())).await; - error!(class = %e.kopia_class(), "verify repository throttle set failed"); + error!(class = %e.kopia_class(), "verify connect failed"); return Err(e); } @@ -2876,20 +2934,14 @@ async fn run_replicate_flow( ); // Connect to the source first (this pod is the only place with repo access for // object stores — the same rationale as maintenance/verify). - if let Err(e) = client.repository_connect(connect, spec.cache).await { + // Connect AND cap, as one operation: the source repository's own cap goes on + // the connection `repository sync-to` reads through — a whole-repository blob + // mirror is the single heaviest read this flow performs, and the source is + // often the link the user capped. + if let Err(e) = connect_and_throttle(client, connect, spec.cache, &spec.throttle).await { + let e = e.into_mover_error(KopiaOp::ReplicateConnect); patch_replicate_status(&spec.target_ref, &replicate_failed_body(&e.to_string())).await; - error!(class = %e.class(), "replication source connect failed"); - return Err(MoverError::Kopia { - op: KopiaOp::ReplicateConnect, - source: e, - }); - } - // The source repository's own cap, on the connection `repository sync-to` - // reads through — a whole-repository blob mirror is the single heaviest read - // this flow performs, and the source is often the link the user capped. - if let Err(e) = apply_repository_throttle(client, &spec.throttle).await { - patch_replicate_status(&spec.target_ref, &replicate_failed_body(&e.to_string())).await; - error!(class = %e.kopia_class(), "replication repository throttle set failed"); + error!(class = %e.kopia_class(), "replication source connect failed"); return Err(e); } @@ -3906,6 +3958,82 @@ mod tests { ); } + /// This file, for the source-level ratchet below. `include_str!` is + /// relative to this file, so it is exactly the module being compiled. + const MAIN_RS: &str = include_str!("main.rs"); + + /// Every DIRECT `repository_connect*` call this file is allowed to + /// contain. Two are the funnels themselves; the rest are the connects + /// that legitimately do not carry a `MoverWorkSpec.throttle` — keep in + /// step with the exception list on `apply_repository_throttle`. + const SANCTIONED_DIRECT_CONNECTS: &[(&str, &str)] = &[ + ("connect_and_throttle", "the funnel (1 call)"), + ( + "bootstrap_connect", + "the bootstrap funnel (2 calls: rw + readonly)", + ), + ( + "main_serve", + "kopia server start; ServerWorkSpec has no throttle field (2 calls)", + ), + ( + "seed_connect_source", + "seed SOURCE/replica; its own cap lands with M3d (1 call)", + ), + ( + "srepl_connect_source", + "snapshot-replication source; M3c (1 call)", + ), + ( + "srepl_connect_dest", + "snapshot-replication destination; M3c (1 call)", + ), + ( + "run_browse_session_flow", + "CLI-built work spec, throttle always empty (1 call)", + ), + ]; + + /// Total direct calls the sanctioned list above accounts for. + const SANCTIONED_DIRECT_CONNECT_CALLS: usize = 9; + + /// **The #374 ratchet.** A mover flow must obtain its connection from + /// [`connect_and_throttle`] or [`bootstrap_connect`], so that "connects + /// a repository" and "applies the repository throttle" are one + /// operation. Nothing in the type system stops a new flow from calling + /// `client.repository_connect` directly and quietly shipping #374 again + /// — every hermetic gate would stay green, because an uncapped run + /// still succeeds. So the ratchet is source-level and deliberately + /// blunt: add a direct connect and this test fails, forcing the author + /// to either use a funnel or state why their connect carries no cap. + #[test] + fn connect_funnel_is_the_only_way_flows_connect() { + // Assembled at runtime, never written whole: a literal here would + // match THIS test's own source lines and inflate the count. + let base = ".repository_connect"; + let patterns = [ + format!("{base}("), + format!("{base}_readonly("), + format!("{base}_with("), + ]; + let direct = MAIN_RS + .lines() + .filter(|l| !l.trim_start().starts_with("//")) + .filter(|l| patterns.iter().any(|p| l.contains(p.as_str()))) + .count(); + assert_eq!( + direct, SANCTIONED_DIRECT_CONNECT_CALLS, + "direct `repository_connect*` calls in this file changed ({direct} found, \ + {SANCTIONED_DIRECT_CONNECT_CALLS} sanctioned). A flow that connects a \ + repository must go through `connect_and_throttle` (or `bootstrap_connect`) \ + so it cannot get a connection without its throttle — that separation is \ + exactly how #374 shipped. If a new connect genuinely carries no \ + `MoverWorkSpec.throttle`, add it here AND to the exception list on \ + `apply_repository_throttle`, with the reason. Sanctioned today: \ + {SANCTIONED_DIRECT_CONNECTS:?}" + ); + } + /// A throttle failure during bootstrap must never be reported as a /// CONNECT verdict. `bootstrap_connect_probe` reads a connect failure to /// decide whether to CREATE or SEED a repository over the backend; a From c46325e93638d2718a87f864443bcf19ac2ea686 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 12:29:28 -0700 Subject: [PATCH 09/15] feat(snapshot-replication): destination + per-CR migrate throttles (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `kopia snapshot migrate` has no speed flags, and kopia's limits are per-CONNECTION — so a snapshot replication, which opens two repositories under two kopia configs, needs two independently resolved caps. Before this, only the source connection was even reachable by a throttle (via the work spec's `throttle`, itself only just wired), and the destination — the off-site link that is usually the scarce one — had no cap surface at all. - API: `MigrateOptions.throttle: Option` with `source`/`destination` sub-blocks reusing `Throttle`. `MigrateOptions` loses `Copy`. Admission rejects a non-positive rate on either side, naming the side and the knob (a shared `validate_throttle` helper). - Controller: `io::merged_throttle` resolves ONE side — that repository's `moverDefaults.throttle` overlaid by the CR's override FIELD BY FIELD, so pinning one knob never drops the repository's others. Called twice; the destination block rides the op as `destinationThrottle`. - Mover: both srepl connects apply their own block immediately after connecting, each terminal via `srepl_terminal` — proceeding uncapped would saturate exactly the link the user asked to protect. The source application lands on the read-only connect (no read-write flip: kopia accepts `throttle set` there, pinned by the kopia integration test). - Tests: the controller-glue guard now carries per-side overrides over two repositories that BOTH set defaults, asserting the merge and that neither side's knobs cross into the other; merge unit tests; wire-shape + old-wire-decode for `destinationThrottle`; camelCase roundtrip. - E2e: the full-history scenario throttles both fixture repos and both sides of the CR, reads the CR back to prove the new field survived admission, asserts both merged blocks on the wire, and requires the srepl pod to log `applied repository throttle` twice — one occurrence means one side's cap is gone. - Docs: per-side semantics and precedence, why migrate has no speed flags, the measured caveats (byte caps are far more punitive than nominal on small-object cold-cache work; warm reads bypass throttling entirely), and the helm-`crds/`-is-install-only trap that silently prunes new fields. --- crates/api/src/common/mover.rs | 30 ++++ crates/api/src/snapshot_replication.rs | 18 ++- crates/api/src/validate/mod.rs | 28 ++++ crates/api/src/validate/repository.rs | 26 +++- crates/api/src/validate/tests.rs | 133 +++++++++++++++++ crates/controller/src/io/repo.rs | 89 +++++++++++- crates/controller/src/snapshot_replication.rs | 85 +++++++++-- crates/e2e/tests/snapshot_replication.rs | 136 +++++++++++++++++- crates/mover/src/main.rs | 37 ++++- crates/mover/src/workspec/mod.rs | 10 ++ crates/mover/src/workspec/tests.rs | 34 +++++ ...pshots_test__snapshot_replication_crd.snap | 69 +++++++++ deploy/crds/all-crds.yaml | 69 +++++++++ deploy/crds/snapshotreplications.yaml | 69 +++++++++ deploy/examples/39-snapshot-replication.yaml | 21 +++ .../kopiur/crds/snapshotreplications.yaml | 69 +++++++++ docs/field-reference.md | 26 ++++ docs/install.md | 2 + docs/reference/crds/shared-types.md | 33 ++++- docs/reference/crds/snapshot-replication.md | 3 + docs/snapshot-replication.md | 87 ++++++++++- 21 files changed, 1044 insertions(+), 30 deletions(-) diff --git a/crates/api/src/common/mover.rs b/crates/api/src/common/mover.rs index 15d9f4c2..ee59b066 100644 --- a/crates/api/src/common/mover.rs +++ b/crates/api/src/common/mover.rs @@ -134,6 +134,36 @@ pub struct MoverDefaults { /// their pods self-GC instead of lingering (ADR-0005 §12). pub const DEFAULT_JOB_TTL_SECONDS: i64 = 3600; +/// Per-side throttle overrides for a `kopia snapshot migrate` run, which reads +/// from a SOURCE repository and writes into a DESTINATION repository under two +/// separate kopia connections. +/// +/// `snapshot migrate` has **no speed flags of its own** — the only lever is +/// `kopia repository throttle set` on each side's connection (kopia persists the +/// limits in that connection's client config, and the migrate honors them when it +/// reopens it). So a cap here is expressed per side, and each side overrides +/// **that side's repository's** `moverDefaults.throttle` field by field: a field +/// set here wins, a field left unset falls back to the repository default, and +/// all four [`Throttle`] knobs are available on each side independently. +/// +/// Not `Copy` (neither is [`Throttle`]). +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Default, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct MigrateThrottle { + /// Caps for the SOURCE (read) side, overriding the source repository's + /// `moverDefaults.throttle` field by field. Applied with `repository + /// throttle set` on the migrate's read-only source connection — accepted + /// there, so a read-only source is throttled like any other. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// Caps for the DESTINATION (write) side, overriding the destination + /// repository's `moverDefaults.throttle` field by field. Applied with + /// `repository throttle set` on the destination connection the migrate + /// writes through. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub destination: Option, +} + /// Repository-wide throttling for a mover's kopia connection; each `None` leaves kopia's current limit. #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Default, JsonSchema)] #[serde(rename_all = "camelCase")] diff --git a/crates/api/src/snapshot_replication.rs b/crates/api/src/snapshot_replication.rs index da4c31a4..4b59912b 100644 --- a/crates/api/src/snapshot_replication.rs +++ b/crates/api/src/snapshot_replication.rs @@ -20,7 +20,8 @@ //! `Snapshot` CRs themselves) removes replicated data. use crate::common::{ - CredentialProjection, CronSpec, MoverSpec, ReplicationManualRunStatus, RepositoryRef, Retention, + CredentialProjection, CronSpec, MigrateThrottle, MoverSpec, ReplicationManualRunStatus, + RepositoryRef, Retention, }; use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; use kube::CustomResource; @@ -146,8 +147,9 @@ pub struct IdentityMatcher { pub source_path: Option, } -/// Tuning for `kopia snapshot migrate`. Pure scalars, so `Eq`. -#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Default, JsonSchema)] +/// Tuning for `kopia snapshot migrate`. Pure scalars + an optional throttle +/// sub-object, so `Eq` but NOT `Copy` (`Throttle` isn't). +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Default, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MigrateOptions { /// `--parallel`: number of snapshots migrated concurrently (kopia default @@ -158,6 +160,14 @@ pub struct MigrateOptions { /// to the destination. Defaults to [`PolicyCopyMode::None`]. #[serde(default)] pub policies: PolicyCopyMode, + /// Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + /// migrate` has no speed flags, so each side is applied as `kopia repository + /// throttle set` on that side's connection; `source` overrides the source + /// repository's `moverDefaults.throttle` and `destination` the destination + /// repository's, field by field (a field left unset keeps that repository's + /// default). Absent: both sides use their repository's defaults. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub throttle: Option, } /// How `kopia snapshot migrate` treats the source's kopia policies. @@ -552,7 +562,7 @@ suspend: false assert!(ids.include[0].source_path.is_none()); assert_eq!(ids.exclude[0].source_path.as_deref(), Some("/scratch/*")); - let migrate = spec.migrate.expect("migrate set"); + let migrate = spec.migrate.clone().expect("migrate set"); assert_eq!(migrate.parallel, Some(4)); assert_eq!(migrate.policies, PolicyCopyMode::CopyOverwrite); diff --git a/crates/api/src/validate/mod.rs b/crates/api/src/validate/mod.rs index 017c6415..2233b0e5 100644 --- a/crates/api/src/validate/mod.rs +++ b/crates/api/src/validate/mod.rs @@ -256,6 +256,34 @@ pub fn require_min(field: &str, value: i64, min: i64) -> Option }) } +/// Validate every SET knob of a [`Throttle`](crate::common::Throttle): a cap is a +/// positive rate, so each present field must be >= 1. `field` names the block's +/// path for the message (e.g. `"SnapshotReplication spec.migrate.throttle.source"`), +/// and each knob's own camelCase name is appended. +/// +/// A `0` is refused rather than read as "unlimited": kopia's own "no limit" is the +/// ABSENT field, so accepting `0` would give one wire value two plausible meanings +/// (uncapped vs. stalled) — and the one it actually has in kopia is a hard stop. +/// Exhaustively destructured, so a fifth knob cannot be added without deciding its +/// bound here. +pub fn validate_throttle(field: &str, throttle: &crate::common::Throttle) -> Vec { + let crate::common::Throttle { + upload_bytes_per_second, + download_bytes_per_second, + read_ops_per_second, + write_ops_per_second, + } = throttle; + [ + ("uploadBytesPerSecond", upload_bytes_per_second), + ("downloadBytesPerSecond", download_bytes_per_second), + ("readOpsPerSecond", read_ops_per_second), + ("writeOpsPerSecond", write_ops_per_second), + ] + .into_iter() + .filter_map(|(knob, value)| value.and_then(|v| require_min(&format!("{field}.{knob}"), v, 1))) + .collect() +} + /// Validate a `MoverSpec`. `context` names the owning resource for the message (e.g. /// `"Restore mover"`). /// diff --git a/crates/api/src/validate/repository.rs b/crates/api/src/validate/repository.rs index c8848b54..dcb47395 100644 --- a/crates/api/src/validate/repository.rs +++ b/crates/api/src/validate/repository.rs @@ -1027,10 +1027,28 @@ pub fn validate_snapshot_replication(spec: &SnapshotReplicationSpec) -> Vec= 1" rule the throttle + // knobs carry everywhere, applied to each side independently so a + // message names the side that is wrong. + if let Some(throttle) = migrate.throttle.as_ref() { + for (side, block) in [ + ("source", throttle.source.as_ref()), + ("destination", throttle.destination.as_ref()), + ] { + if let Some(block) = block { + errs.extend(validate_throttle( + &format!("SnapshotReplication spec.migrate.throttle.{side}"), + block, + )); + } + } + } } if let Some(pruning) = &spec.pruning { // Exhaustive: a new pruning mode cannot compile without deciding its diff --git a/crates/api/src/validate/tests.rs b/crates/api/src/validate/tests.rs index aaf2dc6f..6daa29c6 100644 --- a/crates/api/src/validate/tests.rs +++ b/crates/api/src/validate/tests.rs @@ -5832,6 +5832,7 @@ fn srepl_spec(source: RepositoryRef, dest: RepositoryRef, cron: &str) -> Snapsho #[test] fn srepl_valid_full_spec_has_no_errors() { + use crate::common::{MigrateThrottle, Throttle}; use crate::snapshot_replication::{ IdentityMatcher, IdentitySelection, MigrateOptions, PolicyCopyMode, Pruning, SelectionSpec, }; @@ -5860,6 +5861,17 @@ fn srepl_valid_full_spec_has_no_errors() { spec.migrate = Some(MigrateOptions { parallel: Some(4), policies: PolicyCopyMode::Copy, + throttle: Some(MigrateThrottle { + source: Some(Throttle { + download_bytes_per_second: Some(20 * 1024 * 1024), + ..Default::default() + }), + destination: Some(Throttle { + upload_bytes_per_second: Some(5 * 1024 * 1024), + write_ops_per_second: Some(50), + ..Default::default() + }), + }), }); spec.pruning = Some(Pruning::Retention(Retention { keep_daily: Some(7), @@ -6139,6 +6151,127 @@ fn srepl_migrate_parallel_zero_is_rejected() { assert!(validate_snapshot_replication(&spec).is_empty()); } +#[test] +fn srepl_migrate_throttle_rejects_non_positive_rates_per_side() { + use crate::common::{MigrateThrottle, Throttle}; + use crate::snapshot_replication::MigrateOptions; + let mut spec = srepl_spec( + srepl_ref(RepositoryKind::Repository, "a", None), + srepl_ref(RepositoryKind::Repository, "b", None), + "0 6 * * *", + ); + // A zero on one side and a negative on the other: BOTH must be reported, and + // each message must name the side + the knob (a per-side cap is useless + // guidance if the operator can't tell which connection is wrong). + spec.migrate = Some(MigrateOptions { + throttle: Some(MigrateThrottle { + source: Some(Throttle { + download_bytes_per_second: Some(0), + ..Default::default() + }), + destination: Some(Throttle { + upload_bytes_per_second: Some(-1), + write_ops_per_second: Some(0), + ..Default::default() + }), + }), + ..Default::default() + }); + let fields: Vec = validate_snapshot_replication(&spec) + .iter() + .filter_map(|e| match e { + ValidationError::InvalidFieldValue { field, .. } => Some(field.clone()), + _ => None, + }) + .collect(); + assert_eq!( + fields, + vec![ + "SnapshotReplication spec.migrate.throttle.source.downloadBytesPerSecond", + "SnapshotReplication spec.migrate.throttle.destination.uploadBytesPerSecond", + "SnapshotReplication spec.migrate.throttle.destination.writeOpsPerSecond", + ], + "every offending knob is named, source side first" + ); + + // 1 is the smallest accepted rate; unset knobs are always fine. + spec.migrate = Some(MigrateOptions { + throttle: Some(MigrateThrottle { + source: Some(Throttle { + download_bytes_per_second: Some(1), + ..Default::default() + }), + destination: Some(Throttle { + upload_bytes_per_second: Some(1), + read_ops_per_second: Some(1), + write_ops_per_second: Some(1), + ..Default::default() + }), + }), + ..Default::default() + }); + assert!(validate_snapshot_replication(&spec).is_empty()); + + // An empty MigrateThrottle constrains nothing and is accepted (both sides + // simply inherit their repository's moverDefaults). + spec.migrate = Some(MigrateOptions { + throttle: Some(MigrateThrottle::default()), + ..Default::default() + }); + assert!(validate_snapshot_replication(&spec).is_empty()); +} + +#[test] +fn srepl_migrate_throttle_roundtrips_camel_case_through_the_apiserver_path() { + // YAML → JSON → typed, the way the cluster parses it (serde_yaml 0.9 would + // mis-encode the nested sub-objects). + use crate::snapshot_replication::SnapshotReplicationSpec; + use crate::testutil::from_yaml; + let spec: SnapshotReplicationSpec = from_yaml( + r#" +sourceRef: { name: nas-primary } +destinationRef: { name: offsite } +schedule: { cron: "0 6 * * *" } +migrate: + parallel: 2 + throttle: + source: + downloadBytesPerSecond: 20971520 + readOpsPerSecond: 200 + destination: + uploadBytesPerSecond: 5242880 + writeOpsPerSecond: 50 +"#, + ); + let t = spec + .migrate + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .expect("migrate.throttle set"); + let src = t.source.as_ref().expect("source side set"); + assert_eq!(src.download_bytes_per_second, Some(20 * 1024 * 1024)); + assert_eq!(src.read_ops_per_second, Some(200)); + assert_eq!(src.upload_bytes_per_second, None); + let dst = t.destination.as_ref().expect("destination side set"); + assert_eq!(dst.upload_bytes_per_second, Some(5 * 1024 * 1024)); + assert_eq!(dst.write_ops_per_second, Some(50)); + + // Re-serializing keeps the camelCase wire shape and elides unset knobs. + let json = serde_json::to_value(&spec).expect("serialize"); + assert_eq!( + json["migrate"]["throttle"]["source"]["readOpsPerSecond"], + 200 + ); + assert!( + json["migrate"]["throttle"]["source"] + .get("uploadBytesPerSecond") + .is_none(), + "unset knobs are elided: {json}" + ); + let reparsed: SnapshotReplicationSpec = serde_json::from_value(json).expect("reparse"); + assert_eq!(spec, reparsed); +} + #[test] fn srepl_retention_that_keeps_nothing_is_rejected() { use crate::snapshot_replication::{MirrorSourcePruning, NoPruning, Pruning}; diff --git a/crates/controller/src/io/repo.rs b/crates/controller/src/io/repo.rs index f616e974..426115e5 100644 --- a/crates/controller/src/io/repo.rs +++ b/crates/controller/src/io/repo.rs @@ -7,7 +7,7 @@ use kopiur_api::backend::Backend; use kopiur_api::common::{ CatalogBounds, DeletionProtectionSpec, Encryption, IdentityDefaults, MoverDefaults, NamespaceDeletePolicy, PhaseLabel, RepositoryKind, RepositoryMode, RepositoryRef, - ScheduleDefaults, + ScheduleDefaults, Throttle, }; use kopiur_api::preflight::{PreflightInputs, UNKNOWN_AGE}; use kopiur_api::repository::{RepositoryHealthStatus, RepositoryPhase, StorageStats}; @@ -36,6 +36,35 @@ pub fn throttle_spec(defaults: Option<&MoverDefaults>) -> kopiur_mover::workspec kopiur_mover::workspec::ThrottleSpec::from_mover_defaults(defaults) } +/// Resolve ONE side's throttle for a two-repository run (a `SnapshotReplication` +/// migrate, a seed): that side's repository `moverDefaults.throttle` overlaid by +/// the CR's own per-side override, **field by field**. +/// +/// Field-wise, not block-wise, because the two layers answer different questions: +/// the repository default says "never let anything hammer this backend harder +/// than X", the CR override says "and this particular run is the one I want to +/// hold back further". Replacing the whole block would silently drop the repo's +/// other caps the moment a CR pinned one knob — the classic partial-override +/// footgun that `resolve_mover` avoids for every other `moverDefaults` field. +/// +/// Both absent ⇒ an empty spec, which the mover reads as "skip `throttle set` +/// entirely" (kopia keeps whatever limits its config already carries). +pub fn merged_throttle( + defaults: Option<&MoverDefaults>, + override_: Option<&Throttle>, +) -> kopiur_mover::workspec::ThrottleSpec { + let base = throttle_spec(defaults); + let Some(o) = override_ else { return base }; + kopiur_mover::workspec::ThrottleSpec { + upload_bytes_per_second: o.upload_bytes_per_second.or(base.upload_bytes_per_second), + download_bytes_per_second: o + .download_bytes_per_second + .or(base.download_bytes_per_second), + read_ops_per_second: o.read_ops_per_second.or(base.read_ops_per_second), + write_ops_per_second: o.write_ops_per_second.or(base.write_ops_per_second), + } +} + /// The credentials a mover Job needs, sourced from a repository's /// `encryption.passwordSecretRef`. The Secret is mounted as env (`envFrom`), so /// here we only need its name; the key resolution is documented for callers that @@ -882,6 +911,64 @@ mod tests { assert_eq!(names, vec!["repo-pw", "s3-keys"]); // password first, order-stable } + // --- merged_throttle (per-side override over that repo's moverDefaults) --- + + fn throttle(up: Option, down: Option, r: Option, w: Option) -> Throttle { + Throttle { + upload_bytes_per_second: up, + download_bytes_per_second: down, + read_ops_per_second: r, + write_ops_per_second: w, + } + } + + fn defaults_with(t: Throttle) -> MoverDefaults { + MoverDefaults { + throttle: Some(t), + ..Default::default() + } + } + + #[test] + fn merged_throttle_is_empty_when_neither_layer_sets_anything() { + assert!(merged_throttle(None, None).is_empty()); + assert!(merged_throttle(Some(&MoverDefaults::default()), None).is_empty()); + // An override present but all-None is still nothing to apply. + assert!(merged_throttle(None, Some(&Throttle::default())).is_empty()); + } + + #[test] + fn merged_throttle_falls_back_to_the_repository_defaults() { + let d = defaults_with(throttle(Some(1), Some(2), Some(3), Some(4))); + let out = merged_throttle(Some(&d), None); + assert_eq!(out.upload_bytes_per_second, Some(1)); + assert_eq!(out.download_bytes_per_second, Some(2)); + assert_eq!(out.read_ops_per_second, Some(3)); + assert_eq!(out.write_ops_per_second, Some(4)); + } + + #[test] + fn merged_throttle_override_wins_field_by_field() { + // The footgun this exists to prevent: pinning ONE knob on the CR must not + // drop the repository's other three. + let d = defaults_with(throttle(Some(1), Some(2), Some(3), Some(4))); + let o = throttle(Some(99), None, None, Some(88)); + let out = merged_throttle(Some(&d), Some(&o)); + assert_eq!(out.upload_bytes_per_second, Some(99), "override wins"); + assert_eq!(out.download_bytes_per_second, Some(2), "default survives"); + assert_eq!(out.read_ops_per_second, Some(3), "default survives"); + assert_eq!(out.write_ops_per_second, Some(88), "override wins"); + } + + #[test] + fn merged_throttle_override_alone_stands_without_defaults() { + let o = throttle(None, Some(7), None, None); + let out = merged_throttle(None, Some(&o)); + assert_eq!(out.download_bytes_per_second, Some(7)); + assert_eq!(out.upload_bytes_per_second, None); + assert!(!out.is_empty()); + } + #[test] fn mover_creds_secrets_is_names_only_projection() { let backend = Backend::Filesystem(FilesystemBackend { diff --git a/crates/controller/src/snapshot_replication.rs b/crates/controller/src/snapshot_replication.rs index 14337178..291ee67a 100644 --- a/crates/controller/src/snapshot_replication.rs +++ b/crates/controller/src/snapshot_replication.rs @@ -1275,7 +1275,20 @@ pub fn build_snapshot_replication_work_spec( ) -> MoverWorkSpec { let selection = repl.spec.selection.as_ref(); let (include, exclude) = selection_matchers(repl); - let migrate = repl.spec.migrate.unwrap_or_default(); + let migrate = repl.spec.migrate.clone().unwrap_or_default(); + // `snapshot migrate` has no speed flags: each side's cap must ride its own + // kopia connection, so resolve TWO specs — the source's from the source + // repository's `moverDefaults` and the destination's from the destination's, + // each overlaid field-wise by that side of `spec.migrate.throttle`. + let migrate_throttle = migrate.throttle.as_ref(); + let source_throttle = io::merged_throttle( + source.mover_defaults.as_ref(), + migrate_throttle.and_then(|t| t.source.as_ref()), + ); + let destination_throttle = io::merged_throttle( + dest.mover_defaults.as_ref(), + migrate_throttle.and_then(|t| t.destination.as_ref()), + ); MoverWorkSpec { version: 1, operation: Operation::SnapshotReplicate(SnapshotReplicateOp { @@ -1297,6 +1310,7 @@ pub fn build_snapshot_replication_work_spec( parallel: migrate.parallel, policies: policy_copy_mode_spec(migrate.policies), pruning: pruning_spec(repl.spec.pruning.as_ref()), + destination_throttle, }), // Replication does not snapshot; a stable sentinel identity (like // maintenance / blob replication). @@ -1315,7 +1329,7 @@ pub fn build_snapshot_replication_work_spec( hook_plan: Default::default(), options: MoverOptions::default(), cache: Default::default(), - throttle: io::throttle_spec(source.mover_defaults.as_ref()), + throttle: source_throttle, } } @@ -1831,17 +1845,47 @@ mod tests { r.spec.migrate = Some(MigrateOptions { parallel: Some(4), policies: PolicyCopyMode::CopyOverwrite, + // Per-side caps, each PARTIAL, over repositories that both carry + // their own `moverDefaults.throttle` below — so this asserts the + // full two-layer, two-side resolution, not just "the field arrived". + throttle: Some(kopiur_api::common::MigrateThrottle { + source: Some(kopiur_api::common::Throttle { + download_bytes_per_second: Some(11), + ..Default::default() + }), + destination: Some(kopiur_api::common::Throttle { + upload_bytes_per_second: Some(22), + ..Default::default() + }), + }), }); r.spec.pruning = Some(Pruning::Retention( serde_json::from_value(serde_json::json!({ "keepDaily": 7, "keepWeekly": 4 })).unwrap(), )); - let ws = build_snapshot_replication_work_spec( - &r, - &sample_source(), - &sample_dest(), - "ns", - "offsite", - ); + // Each repository brings its OWN moverDefaults.throttle; the CR's + // per-side override must win field-by-field WITHOUT dropping the rest, + // and the two sides must never cross (a source default landing on the + // destination block would be invisible in production until the wrong + // link got saturated). + let mut source = sample_source(); + source.mover_defaults = Some(kopiur_api::common::MoverDefaults { + throttle: Some(kopiur_api::common::Throttle { + download_bytes_per_second: Some(1), + read_ops_per_second: Some(2), + ..Default::default() + }), + ..Default::default() + }); + let mut dest = sample_dest(); + dest.mover_defaults = Some(kopiur_api::common::MoverDefaults { + throttle: Some(kopiur_api::common::Throttle { + upload_bytes_per_second: Some(3), + write_ops_per_second: Some(4), + ..Default::default() + }), + ..Default::default() + }); + let ws = build_snapshot_replication_work_spec(&r, &source, &dest, "ns", "offsite"); match &ws.operation { Operation::SnapshotReplicate(op) => { assert_eq!(op.include.len(), 1); @@ -1859,9 +1903,32 @@ mod tests { } other => panic!("expected retention pruning, got {other:?}"), } + // DESTINATION side: the CR's upload override wins, the dest + // repo's writeOps default survives, and NOTHING from the source + // repo's defaults leaks in. + assert_eq!(op.destination_throttle.upload_bytes_per_second, Some(22)); + assert_eq!(op.destination_throttle.write_ops_per_second, Some(4)); + assert_eq!( + op.destination_throttle.download_bytes_per_second, None, + "the SOURCE repo's download default must not reach the destination" + ); + assert_eq!( + op.destination_throttle.read_ops_per_second, None, + "the SOURCE repo's readOps default must not reach the destination" + ); } other => panic!("expected snapshot-replicate op, got {}", other.kind_str()), } + // SOURCE side rides the work spec's own throttle: CR override wins on + // download, the source repo's readOps default survives, and the DEST + // repo's upload/writeOps defaults stay out. + assert_eq!(ws.throttle.download_bytes_per_second, Some(11)); + assert_eq!(ws.throttle.read_ops_per_second, Some(2)); + assert_eq!( + ws.throttle.upload_bytes_per_second, None, + "the DESTINATION repo's upload default must not reach the source" + ); + assert_eq!(ws.throttle.write_ops_per_second, None); } #[test] diff --git a/crates/e2e/tests/snapshot_replication.rs b/crates/e2e/tests/snapshot_replication.rs index 22181b17..c8f05a0f 100644 --- a/crates/e2e/tests/snapshot_replication.rs +++ b/crates/e2e/tests/snapshot_replication.rs @@ -48,7 +48,7 @@ use kopiur_api::consts::{ CONFIG_LABEL, ORIGIN_LABEL, REPOSITORY_UID_LABEL, SNAPSHOT_ID_LABEL, SNAPSHOT_REPLICATION_LABEL, }; use kopiur_api::{Repository, Snapshot, SnapshotPolicy, SnapshotReplication}; -use kopiur_e2e::{E2E_NAMESPACE, Need, World, default_timeout, poll_interval, wait_until}; +use kopiur_e2e::{E2E_NAMESPACE, Need, World, consts, default_timeout, poll_interval, wait_until}; /// In-pod mount path every DESTINATION repository pins (see module docs). const DEST_REPO_PATH: &str = "/repo-dst"; @@ -217,6 +217,34 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { // 1. Source history: seed (repo + policy + snapshot 1), then a second // distinct backup under the same policy. + // + // The source repository is created HERE rather than by `ensure_seed` (which + // is idempotent and will find it present) so it can carry its own + // `moverDefaults.throttle` — the base layer the per-CR override below has to + // merge over. Caps are deliberately GENEROUS (100 MiB/s, 10k ops/s): the + // point is to prove the limits reach kopia on both connections, and a low + // byte cap is far more punitive than nominal on a small-object cold-cache + // migrate (a 2 MB/s cap measured ~14s for 28 KiB), which would only make + // this scenario slow and flaky. + ensure_repo(&client, "srepl-src").await; + { + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + create_idempotent( + &repos, + &cr(repository_json( + REPO_SRC, + "srepl-src", + serde_json::json!({ + "moverDefaults": { "throttle": { + "downloadBytesPerSecond": 104857600, + "readOpsPerSecond": 10000 + } } + }), + )), + "create source Repository with moverDefaults.throttle", + ) + .await; + } ensure_seed(&client, REPO_SRC, POLICY, B1, "srepl-src").await; let backups: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); create_idempotent( @@ -274,7 +302,13 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { "srepl-dst", DST_SECRET, serde_json::json!({ - "maintenance": { "enabled": false } + "maintenance": { "enabled": false }, + // The DESTINATION's own defaults — a separate repository, so a + // separate throttle layer (kopia's limits are per connection). + "moverDefaults": { "throttle": { + "uploadBytesPerSecond": 104857600, + "writeOpsPerSecond": 10000 + } } }), ) .await; @@ -291,12 +325,38 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { "spec": { "sourceRef": { "kind": "Repository", "name": REPO_SRC }, "destinationRef": { "kind": "Repository", "name": REPO_DST }, - "schedule": { "cron": "* * * * *" } + "schedule": { "cron": "* * * * *" }, + // Per-CR, per-side overrides (#374). Each side sets ONE knob, so + // the merged block must also carry the knob that side's + // repository defaulted — the field-wise merge, end to end. + "migrate": { "throttle": { + "source": { "downloadBytesPerSecond": 209715200 }, + "destination": { "uploadBytesPerSecond": 209715200 } + } } } })), "create SnapshotReplication", ) .await; + // The CRD must actually carry the new fields — an apiserver whose CRD schema + // predates them PRUNES them silently, which is the exact #374 symptom this + // whole change is about. Read it back before asserting on downstream effects. + { + let live = serde_json::to_value(repls.get(REPL).await.expect("get SnapshotReplication")) + .unwrap_or_default(); + assert_eq!( + live.pointer("/spec/migrate/throttle/source/downloadBytesPerSecond") + .and_then(|v| v.as_u64()), + Some(209715200), + "spec.migrate.throttle survived admission (a pruned field means a stale CRD): {live}" + ); + assert_eq!( + live.pointer("/spec/migrate/throttle/destination/uploadBytesPerSecond") + .and_then(|v| v.as_u64()), + Some(209715200), + "{live}" + ); + } // The per-slot mover Job's inline work-spec env is the controller↔mover // contract seam (mirrors the replication.rs #216 guard): the operation must @@ -363,6 +423,53 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { "unset knob {absent} must be elided from the wire: {op}" ); } + // Both merged throttle blocks ride the wire, one per connection: the SOURCE + // side on the work spec's own `throttle`, the DESTINATION side on the op's + // `destinationThrottle`. Each must show the CR's override AND the knob its + // own repository defaulted — and NOT the other repository's knobs, which is + // how a crossed-sides merge would show up. + let src_throttle = spec + .get("throttle") + .unwrap_or_else(|| panic!("work spec must carry the merged SOURCE throttle: {spec}")); + assert_eq!( + src_throttle + .get("downloadBytesPerSecond") + .and_then(|v| v.as_u64()), + Some(209715200), + "the CR's source override wins: {src_throttle}" + ); + assert_eq!( + src_throttle + .get("readOpsPerSecond") + .and_then(|v| v.as_u64()), + Some(10000), + "the SOURCE repo's moverDefaults knob survives the partial override: {src_throttle}" + ); + assert!( + src_throttle.get("uploadBytesPerSecond").is_none(), + "the DESTINATION repo's upload default must not cross into the source: {src_throttle}" + ); + let dst_throttle = op + .get("destinationThrottle") + .unwrap_or_else(|| panic!("op must carry the merged DESTINATION throttle: {op}")); + assert_eq!( + dst_throttle + .get("uploadBytesPerSecond") + .and_then(|v| v.as_u64()), + Some(209715200), + "the CR's destination override wins: {dst_throttle}" + ); + assert_eq!( + dst_throttle + .get("writeOpsPerSecond") + .and_then(|v| v.as_u64()), + Some(10000), + "the DESTINATION repo's moverDefaults knob survives: {dst_throttle}" + ); + assert!( + dst_throttle.get("readOpsPerSecond").is_none(), + "the SOURCE repo's readOps default must not cross into the destination: {dst_throttle}" + ); // 4. A run succeeds and records lastReplicated + the run counters. The // counter assert tolerates an in-place nextest retry (copies may already @@ -398,6 +505,29 @@ async fn snapshot_replication_copies_history_between_filesystem_repos() { "the run must account for both source snapshots (copied {copied} + alreadyPresent {already}): {s1}" ); + // 4b. #374 regression guard: a run that succeeded must ALSO have capped both + // of its connections. The srepl mover opens exactly TWO repositories — + // the source (read-only, under its own kopia config, the only cap + // `snapshot migrate` can honor on the read side) and the destination — + // and neither goes through the generic connect funnel, so each is its own + // chance to ship #374 again. The line is emitted once per APPLICATION and + // only after kopia accepted the limits, so two occurrences in ONE pod can + // only mean both connections were capped; counting per pod (not across + // pods) is what keeps deleting ONE of the two applications from staying + // green on a suite that runs several replication pods. + kopiur_e2e::wait::wait_for_pod_log_times( + &client, + E2E_NAMESPACE, + &selector, + consts::THROTTLE_APPLIED_LOG, + 2, + ) + .await + .expect( + "the snapshot-replication mover must cap BOTH the source and the destination \ + connection (#374); one occurrence means one side's throttle is gone", + ); + // kstatus two-pass heal guard: the mover stamps the terminal status, the // controller heals Ready=True on a FOLLOW-UP reconcile — gate on the healed // condition like every sibling. diff --git a/crates/mover/src/main.rs b/crates/mover/src/main.rs index 25b5b813..110384eb 100644 --- a/crates/mover/src/main.rs +++ b/crates/mover/src/main.rs @@ -413,8 +413,10 @@ async fn connect_and_throttle( /// `throttle: Default::default()`, so `spec.throttle` is structurally always /// empty; capping it means teaching the CLI to resolve `moverDefaults` first. /// * **snapshot-replication source and destination** — a dual connect under two -/// kopia configs, with per-CR per-side overrides; wired by the next milestone -/// (M3c), which adds the destination's own `destinationThrottle`. +/// kopia configs, so ONE funnel call cannot serve both: each side is connected +/// directly and then throttled in place from its own resolved block (the work +/// spec's `throttle` for the source, the op's `destinationThrottle` for the +/// destination). Both applications are terminal, via the flow's `srepl_terminal`. /// * **a seed's SOURCE (replica) connect** — same shape, wired by M3d together /// with its `replica_throttle`. /// @@ -3133,6 +3135,22 @@ async fn srepl_connect_source( unreachable!("srepl_terminal_kopia always errors"); } + // Cap the SOURCE read side on this connection. `snapshot migrate` reopens + // this very config, and kopia persists the limits into it — that is the only + // lever migrate has, since it carries no speed flags of its own. Applied + // straight onto the read-only connect: `repository throttle set` is accepted + // there (pinned by the kopia crate's integration test), so no read-write + // flip is involved. + // + // Deliberately BEFORE the no-env-password probe below, not after: the probe's + // whole job is to prove the config migrate will reopen is usable, so it + // should see that config in its FINAL state — throttle limits written and + // all — rather than a state no later step ever runs against. + if let Err(e) = apply_repository_throttle(&source_client, &spec.throttle).await { + srepl_terminal(&spec.target_ref, e, None).await?; + unreachable!("srepl_terminal always errors"); + } + let probe_client = srepl_client_builder(spec, kopia_binary) .env(kopiur_kopia::env::CONFIG_PATH_ENV, &paths.source_config) .env(kopiur_kopia::env::CACHE_DIRECTORY_ENV, &paths.source_cache) @@ -3215,6 +3233,15 @@ async fn srepl_connect_dest( srepl_terminal_kopia(&spec.target_ref, Op::SnapshotReplicateDestConnect, e).await?; unreachable!("srepl_terminal_kopia always errors"); } + // Cap the DESTINATION write side on ITS connection, from the op's own + // `destinationThrottle` (the destination repository's `moverDefaults` + // overlaid by `spec.migrate.throttle.destination`). kopia's limits are + // per-connection, so the source's cap above says nothing here — this is the + // block that keeps a migrate from saturating the off-site link it writes to. + if let Err(e) = apply_repository_throttle(&dest_client, &op.destination_throttle).await { + srepl_terminal(&spec.target_ref, e, None).await?; + unreachable!("srepl_terminal always errors"); + } Ok(dest_client) } @@ -3982,11 +4009,13 @@ mod tests { ), ( "srepl_connect_source", - "snapshot-replication source; M3c (1 call)", + "snapshot-replication source; dual-config connect, throttled in place \ + from spec.throttle immediately after (1 call)", ), ( "srepl_connect_dest", - "snapshot-replication destination; M3c (1 call)", + "snapshot-replication destination; dual-config connect, throttled in \ + place from op.destinationThrottle immediately after (1 call)", ), ( "run_browse_session_flow", diff --git a/crates/mover/src/workspec/mod.rs b/crates/mover/src/workspec/mod.rs index fbd38e6a..a8c709c4 100644 --- a/crates/mover/src/workspec/mod.rs +++ b/crates/mover/src/workspec/mod.rs @@ -1677,6 +1677,16 @@ pub struct SnapshotReplicateOp { /// [`PruningSpec::None`] (never prune) — absent on the wire means none. #[serde(default, skip_serializing_if = "PruningSpec::is_none")] pub pruning: PruningSpec, + /// Throttle limits for the DESTINATION connection (`kopia repository + /// throttle set` after the dest connect). The SOURCE side rides the work + /// spec's own [`MoverWorkSpec::throttle`] — a snapshot replication opens two + /// repositories under two kopia configs, and kopia's limits are per + /// connection, so each side needs its own resolved block. The controller + /// merges the destination repository's `moverDefaults.throttle` with + /// `spec.migrate.throttle.destination` field by field. All-`None` ⇒ the + /// mover skips the dest throttle call. + #[serde(default, skip_serializing_if = "ThrottleSpec::is_empty")] + pub destination_throttle: ThrottleSpec, } /// A fully-resolved repository CR reference on the wire (kind + name + diff --git a/crates/mover/src/workspec/tests.rs b/crates/mover/src/workspec/tests.rs index 86631ebc..e43ea5cb 100644 --- a/crates/mover/src/workspec/tests.rs +++ b/crates/mover/src/workspec/tests.rs @@ -1687,6 +1687,11 @@ fn sample_snapshot_replicate_op() -> SnapshotReplicateOp { keep_weekly: Some(4), ..Default::default() }), + destination_throttle: ThrottleSpec { + upload_bytes_per_second: Some(5 * 1024 * 1024), + write_ops_per_second: Some(50), + ..Default::default() + }, } } @@ -1733,6 +1738,31 @@ fn snapshot_replicate_roundtrip_and_wire_shape() { assert_eq!(op["pruning"]["retention"]["keepDaily"], 7); assert_eq!(op["pruning"]["retention"]["keepWeekly"], 4); assert!(op["pruning"]["retention"].get("keepLatest").is_none()); + // The DESTINATION-side cap rides the op (the source's rides the work spec's + // own `throttle`): two connections, two blocks, camelCase, unset knobs elided. + assert_eq!(op["destinationThrottle"]["uploadBytesPerSecond"], 5242880); + assert_eq!(op["destinationThrottle"]["writeOpsPerSecond"], 50); + assert!( + op["destinationThrottle"] + .get("downloadBytesPerSecond") + .is_none(), + "unset dest knobs are elided: {op}" + ); +} + +#[test] +fn snapshot_replicate_empty_destination_throttle_is_elided_from_the_wire() { + // The common case must not grow a key: an all-None dest throttle is the + // mover's "skip `throttle set`" signal, so it serializes to nothing at all. + let op = SnapshotReplicateOp { + destination_throttle: ThrottleSpec::default(), + ..sample_snapshot_replicate_op() + }; + let v = serde_json::to_value(&op).unwrap(); + assert!( + v.get("destinationThrottle").is_none(), + "an empty dest throttle is elided: {v}" + ); } #[test] @@ -1774,6 +1804,10 @@ fn snapshot_replicate_old_wire_decodes_with_selection_and_pruning_defaulted() { assert!(op.pruning.is_none()); assert_eq!(op.destination_repository.namespace, None); assert_eq!(op.source_repository.namespace, None); + // A spec stamped before `destinationThrottle` existed decodes to an empty + // block — i.e. an old controller driving a new mover leaves the destination + // uncapped rather than failing to decode. + assert!(op.destination_throttle.is_empty()); } #[test] diff --git a/crates/xtask/tests/snapshots/snapshots_test__snapshot_replication_crd.snap b/crates/xtask/tests/snapshots/snapshots_test__snapshot_replication_crd.snap index ff013f22..5e5542ae 100644 --- a/crates/xtask/tests/snapshots/snapshots_test__snapshot_replication_crd.snap +++ b/crates/xtask/tests/snapshots/snapshots_test__snapshot_replication_crd.snap @@ -110,6 +110,75 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + migrate` has no speed flags, so each side is applied as `kopia repository + throttle set` on that side's connection; `source` overrides the source + repository's `moverDefaults.throttle` and `destination` the destination + repository's, field by field (a field left unset keeps that repository's + default). Absent: both sides use their repository's defaults. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object mover: description: Mover (Job pod) overrides for the replication run. diff --git a/deploy/crds/all-crds.yaml b/deploy/crds/all-crds.yaml index 0335f6b8..68887d51 100644 --- a/deploy/crds/all-crds.yaml +++ b/deploy/crds/all-crds.yaml @@ -13065,6 +13065,75 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + migrate` has no speed flags, so each side is applied as `kopia repository + throttle set` on that side's connection; `source` overrides the source + repository's `moverDefaults.throttle` and `destination` the destination + repository's, field by field (a field left unset keeps that repository's + default). Absent: both sides use their repository's defaults. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object mover: description: Mover (Job pod) overrides for the replication run. diff --git a/deploy/crds/snapshotreplications.yaml b/deploy/crds/snapshotreplications.yaml index 197e684b..3bfb4893 100644 --- a/deploy/crds/snapshotreplications.yaml +++ b/deploy/crds/snapshotreplications.yaml @@ -107,6 +107,75 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + migrate` has no speed flags, so each side is applied as `kopia repository + throttle set` on that side's connection; `source` overrides the source + repository's `moverDefaults.throttle` and `destination` the destination + repository's, field by field (a field left unset keeps that repository's + default). Absent: both sides use their repository's defaults. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object mover: description: Mover (Job pod) overrides for the replication run. diff --git a/deploy/examples/39-snapshot-replication.yaml b/deploy/examples/39-snapshot-replication.yaml index 24e34a3d..66e276c1 100644 --- a/deploy/examples/39-snapshot-replication.yaml +++ b/deploy/examples/39-snapshot-replication.yaml @@ -105,6 +105,27 @@ spec: # Whether kopia POLICIES ride along: none (default — a Kopiur-managed # destination keeps retention CR-driven), copy, or copyOverwrite. policies: none + # Bandwidth/ops caps for THIS replication's runs, one block per side. + # `kopia snapshot migrate` has no speed flags, so each side is applied as + # `kopia repository throttle set` on that side's connection before the + # migrate runs. Each side overrides THAT side's repository's + # `moverDefaults.throttle` field by field: a knob set here wins, a knob left + # unset keeps the repository's value. Every set knob must be >= 1. + # + # Caveat worth knowing before you tune these down: byte caps only bite on + # COLD backend traffic (cached content bypasses the limiter), and on + # small-object workloads the real throughput lands far below the nominal + # rate. Treat them as a ceiling for large transfers and measure. + throttle: + # Reading out of nas-primary — on the LAN, so leave it roomy. + source: + downloadBytesPerSecond: 209715200 # 200 MiB/s + # Writing to the off-site bucket — the scarce link, and the one worth + # capping. Unset knobs (downloadBytesPerSecond, readOpsPerSecond) fall + # back to the destination Repository's own moverDefaults.throttle. + destination: + uploadBytesPerSecond: 10485760 # 10 MiB/s + writeOpsPerSecond: 100 # What happens to already-replicated copies on later runs. Exactly one of: # none: never prune (the default when `pruning` is omitted) # mirrorSource: delete a copy when its snapshot vanished from the source diff --git a/deploy/helm/kopiur/crds/snapshotreplications.yaml b/deploy/helm/kopiur/crds/snapshotreplications.yaml index 197e684b..3bfb4893 100644 --- a/deploy/helm/kopiur/crds/snapshotreplications.yaml +++ b/deploy/helm/kopiur/crds/snapshotreplications.yaml @@ -107,6 +107,75 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS replication's runs, per side. `snapshot + migrate` has no speed flags, so each side is applied as `kopia repository + throttle set` on that side's connection; `source` overrides the source + repository's `moverDefaults.throttle` and `destination` the destination + repository's, field by field (a field left unset keeps that repository's + default). Absent: both sides use their repository's defaults. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object mover: description: Mover (Job pod) overrides for the replication run. diff --git a/docs/field-reference.md b/docs/field-reference.md index 9f5bbe11..3a22025d 100644 --- a/docs/field-reference.md +++ b/docs/field-reference.md @@ -3757,6 +3757,32 @@ Externally tagged — set **exactly one** of: `pvcConsumer` · `snapshot` · `wo | --- | --- | --- | --- | | `parallel` | integer | —
min 0 | `--parallel`: number of snapshots migrated concurrently (kopia default `1` — sequential). Must be >= 1 when set. | | `policies` | enum: none \| copy \| copyOverwrite | `none` | Whether kopia **policies** attached to the copied sources are also copied to the destination. Defaults to `PolicyCopyMode::None`. | +| `throttle` | [object](#snapshotreplication-spec-migrate-throttle) | — | Bandwidth/ops caps for THIS replication's runs, per side. `snapshot migrate` has no speed flags, so each side is applied as `kopia repository throttle set` on that side's connection; `source` overrides the source repository's `moverDefaults.throttle` and `destination` the destination repository's, field by field (a field left unset keeps that repository's default). Absent: both sides use their repository's defaults. | + +##### `spec.migrate.throttle` { #snapshotreplication-spec-migrate-throttle } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `destination` | [object](#snapshotreplication-spec-migrate-throttle-destination) | — | Caps for the DESTINATION (write) side, overriding the destination repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the destination connection the migrate writes through. | +| `source` | [object](#snapshotreplication-spec-migrate-throttle-source) | — | Caps for the SOURCE (read) side, overriding the source repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the migrate's read-only source connection — accepted there, so a read-only source is throttled like any other. | + +###### `spec.migrate.throttle.destination` { #snapshotreplication-spec-migrate-throttle-destination } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | + +###### `spec.migrate.throttle.source` { #snapshotreplication-spec-migrate-throttle-source } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | #### `spec.mover` { #snapshotreplication-spec-mover } diff --git a/docs/install.md b/docs/install.md index 009d825f..ead5ce49 100644 --- a/docs/install.md +++ b/docs/install.md @@ -262,6 +262,8 @@ The 9 CRDs ship in the chart's special `crds/` directory. Helm treats that direc > ``` > > A GitOps flow (Flux/Argo with a `CreateReplace` sync policy) applies CRD changes automatically. Anyone managing CRDs entirely out of band just applies `deploy/crds/` themselves — `helm install` skips the ones that already exist. +> +> Skipping this is usually **silent**, not loud: when a release adds a new spec field, an apiserver still running the old schema **prunes** that field out of every object you apply. The object admits cleanly, `kubectl get -o yaml` shows no trace of the field, and the feature you configured simply never happens. If a newly-documented field seems to be ignored, check the live CRD before anything else (`kubectl get crd .kopiur.home-operations.com -o yaml | grep `); `kubectl kopiur doctor` flags known-stale schemas too. The CRDs and RBAC shipped by the chart are **generated** by `cargo xtask gen-crds` / `cargo xtask gen-rbac` and checked in under `deploy/crds/` and `deploy/rbac/`. Those xtasks are the source of truth. diff --git a/docs/reference/crds/shared-types.md b/docs/reference/crds/shared-types.md index b0dd53fa..104eb373 100644 --- a/docs/reference/crds/shared-types.md +++ b/docs/reference/crds/shared-types.md @@ -100,10 +100,35 @@ cache. See [movers](../../movers.md). mover Job so finished Jobs self-GC. Defaults to 1h when neither the repo nor the recipe sets one. - **`throttle`** — repository throttle limits (`kopia repository throttle set`) - applied by every mover after it connects, so a run doesn't saturate the link or - hammer the object store: `uploadBytesPerSecond`, `downloadBytesPerSecond`, - `readOpsPerSecond`, `writeOpsPerSecond` (each unset leaves kopia's current - limit). + applied by every mover after it connects to this repository, so a run doesn't + saturate the link or hammer the object store: `uploadBytesPerSecond`, + `downloadBytesPerSecond`, `readOpsPerSecond`, `writeOpsPerSecond` (each unset + leaves kopia's current limit). Every batch mover honors it — bootstrap, backup, + restore, maintenance, verification, deletion, repository replication and + snapshot replication. kopia's limits are **per connection**, so a run that + opens two repositories caps each from *that* repository's own `moverDefaults`: + a [`SnapshotReplication`](snapshot-replication.md) caps its source connection + from the source repo's defaults and its destination connection from the + destination repo's, each overridable per CR via + [`spec.migrate.throttle`](snapshot-replication.md#migrate). The exceptions: + the interactive [`serve`/browse](../../server.md) session (a read-only UI + session, not a batch transfer), and a `RepositoryReplication`'s **destination** + side — `kopia repository sync-to` copies blobs without opening the destination + as a repository, so its caps live on + [`spec.sync`](repository-replication.md) instead. + +/// warning | A byte cap bites much harder than its number suggests + +A `*BytesPerSecond` cap is enforced against **cold** backend traffic only — +content already in the mover's kopia cache is read without touching the limiter, +so a warm re-run can look completely unthrottled. And on small-object workloads +the effective throughput lands far below the nominal rate: a measured 2 MB/s cap +took ~14 s to move a 28 KiB cold repository, while caps of 10 MB/s and up often +did not bind at all at that size. Treat these as a **ceiling for large transfers** +— set them generously and verify against your own data, rather than tuning them +down to a number that looks safe. + +/// The `securityContext`/`podSecurityContext`/`resources`/`cache` fields resolve by field-wise merge: a repo-wide default composes with a partial per-recipe diff --git a/docs/reference/crds/snapshot-replication.md b/docs/reference/crds/snapshot-replication.md index 1ff614f7..61db81aa 100644 --- a/docs/reference/crds/snapshot-replication.md +++ b/docs/reference/crds/snapshot-replication.md @@ -33,6 +33,9 @@ Tuning for the underlying `kopia snapshot migrate`: | --- | --- | | `parallel` | Snapshots migrated concurrently (kopia default `1` — sequential; must be `>= 1` when set, webhook-enforced). The main knob for large first runs. | | `policies` | Whether kopia **policy** objects ride along: `none` (default — Kopiur pins retention CR-side, so imported kopia policies are usually unwanted), `copy` (copy where absent), `copyOverwrite` (copy and overwrite). | +| `throttle.source` / `throttle.destination` | Bandwidth/ops caps for this replication's runs, **per side** — each a [`throttle`](shared-types.md#moverdefaults) block (`uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, `writeOpsPerSecond`; every set knob must be `>= 1`, webhook-enforced). Each side **overrides that side's repository's `moverDefaults.throttle` field by field** — a knob set here wins, a knob left unset keeps the repository's value. Absent: both sides use their repository's defaults. See [Throttling a replication](../../snapshot-replication.md#throttling-a-replication). | + +`kopia snapshot migrate` has **no speed flags of its own**, so `throttle` is not passed to the migrate command: each side is applied as `kopia repository throttle set` on that side's connection *before* the migrate runs, and kopia persists the limits into that connection's config for the migrate to pick up. Two repositories, two connections, two independent blocks — the source cap says nothing about the destination and vice versa. Applying a cap to the read-only source connection is supported (kopia accepts `throttle set` there), so a `mode: ReadOnly` source is throttled like any other. ### `pruning` diff --git a/docs/snapshot-replication.md b/docs/snapshot-replication.md index 6c0e5552..7369349d 100644 --- a/docs/snapshot-replication.md +++ b/docs/snapshot-replication.md @@ -53,6 +53,7 @@ $ kubectl -n billing get snapshots -l kopiur.home-operations.com/origin=replicat | `selection.latestOnly` | `true` = only each identity's most recent snapshot (cheap seed); default `false` = full history. | | `migrate.parallel` | Snapshots migrated concurrently (kopia default: 1, sequential) — the main knob for large first runs. | | `migrate.policies` | Whether kopia **policies** ride along: `none` (default — a Kopiur-managed destination keeps retention CR-driven), `copy`, or `copyOverwrite`. | +| `migrate.throttle.source` / `.destination` | Bandwidth/ops caps for this replication's runs, one block **per side** — see [Throttling a replication](#throttling-a-replication). Each overrides that side's repository's `moverDefaults.throttle` field by field. | | `pruning` | What happens to already-made copies on later runs: exactly one of `none` / `mirrorSource` / `retention` — see [Pruning](#pruning-the-copies). | | `mover` | Per-run mover overrides (resources, scheduling, security context). Inherits the source repository's `moverDefaults`. `inheritSecurityContextFrom` is rejected here, as for `RepositoryReplication` — there is no workload to inherit from. | | `credentialProjection` | Opt in to [credential projection](movers.md#let-kopiur-project-the-credentials-secret-recommended-for-shared-repos) for a `ClusterRepository` source/destination whose Secret lives elsewhere. | @@ -110,6 +111,90 @@ selection: Matching zero identities is a **successful no-op** (`NoIdentitiesMatched`), not an error — a fresh source simply has nothing to copy yet. Incomplete (interrupted) source snapshots are never copied. +## Throttling a replication + +A replication is often the heaviest thing Kopiur does to your network: it reads a +whole history out of one repository and writes it into another, frequently across +a WAN link that other people are also using. `spec.migrate.throttle` caps it — +**per side**, because a replication is two repositories. + +```yaml +migrate: + throttle: + source: # reading out of nas-primary (on-LAN, be generous) + downloadBytesPerSecond: 209715200 # 200 MiB/s + destination: # writing to the off-site bucket (the scarce link) + uploadBytesPerSecond: 10485760 # 10 MiB/s + writeOpsPerSecond: 100 +``` + +Each side takes the same four knobs as a repository's +[`moverDefaults.throttle`](reference/crds/shared-types.md#moverdefaults): +`uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, +`writeOpsPerSecond`. Every knob you set must be at least `1` (a `0` is rejected at +admission — kopia's "no limit" is the *absent* knob, not a zero). + +**Two layers, merged field by field.** Each repository can already declare +`moverDefaults.throttle`, which every mover that touches it honors. `migrate.throttle` +sits on top of that, per side: + +- `throttle.source` overrides the **source** repository's defaults; + `throttle.destination` overrides the **destination** repository's. +- The merge is per knob: a knob you set here wins, a knob you leave unset keeps + the repository's value. Pinning one knob never drops the repository's others. +- Omit `migrate.throttle` entirely and each side simply uses its own repository's + defaults. + +The two sides never mix. A cap on the source constrains only the read connection, +because kopia's limits are **per connection** and a replication opens two of them +under two separate kopia configurations. + +/// note | Why the cap isn't a flag on `snapshot migrate` + +`kopia snapshot migrate` has no speed options at all. The only lever kopia offers +is `kopia repository throttle set`, which writes the limits into a *connection's* +config — so Kopiur connects each side, applies that side's limits, and lets the +migrate (which reopens those configs) inherit them. That is also why each side +needs its own block: there is nothing repository-wide to inherit. Applying a cap +to the read-only source connection is fine — kopia accepts `throttle set` there — +so a `mode: ReadOnly` source is throttled like any other. + +If either application fails, the run **fails** rather than proceeding uncapped: +saturating exactly the link you asked to protect is worse than not running. The +mover logs `applied repository throttle` once per capped connection (twice for a +healthy replication), which is the quickest way to confirm the limits landed: + +```console +$ kubectl -n billing logs job/nas-primary-to-offsite-srepl-1765440000 | grep 'applied repository throttle' +``` + +/// + +/// warning | A byte cap bites much harder than its number suggests + +Byte-rate caps apply to **cold** backend traffic only — content already in the +mover's kopia cache is served without touching the limiter, so a warm re-run can +look entirely unthrottled. And on small-object workloads the effective throughput +lands far below the nominal number: a measured 2 MB/s cap took ~14 s to move a +28 KiB cold repository, while caps of 10 MB/s and above often did not bind at all +at that size. Set these as a **ceiling for large transfers**, generously, and +measure against your own data — a number picked to "feel safe" can turn a nightly +replication into one that never finishes. + +/// + +/// warning | New CRD fields need `kubectl apply`, not just `helm upgrade` + +`migrate.throttle` is a new CRD field. Helm's `crds/` directory is **install-only** +— a `helm upgrade` never updates CRD schemas — so on a helm-CLI upgrade an +apiserver running the old schema **silently prunes** the field from your manifest: +the object admits cleanly, `kubectl get -o yaml` shows no `throttle`, and the runs +stay uncapped with nothing to see. Apply the CRDs first (`kubectl apply +--server-side -f deploy/crds/`), or use a GitOps flow with a `CreateReplace` CRD +policy. See [CRD lifecycle](install.md#crd-lifecycle). + +/// + ## Pruning the copies `pruning` is exactly one of three modes; **absent means `none`**. Whatever the mode, pruning only ever considers snapshots **this replication created** (the copy CRs it labels) — never the destination's own directly-written snapshots, and never another replication's copies. @@ -137,7 +222,7 @@ Matching zero identities is a **successful no-op** (`NoIdentitiesMatched`), not - **A dedicated mover ServiceAccount.** Replication movers create/patch/delete `Snapshot` CRs (the copies), which the ordinary backup mover must never be able to do — so they run as the dedicated `kopiur-snapshot-replication-mover` ServiceAccount with its own narrowly-scoped Role, generated alongside the rest of the [RBAC](rbac.md). - **No destination maintenance behind your back.** The mover always disables kopia's auto-maintenance; the destination's own [`Maintenance`](maintenance.md) remains the only compaction that runs there. - **Every run is counted.** `kopiur_replication_runs_total{kind,trigger,outcome}` records each finished run, so "the nightly copy has been failing" is alertable without watching conditions. `trigger` separates `cron` from the [requested](#run-it-now) runs. -- **Size the first run.** A full-history first replication of a large repository moves everything once (idempotent thereafter). Raise `migrate.parallel`, consider `latestOnly: true` for seeding, and note the mover Job's deadline can be tuned via `mover`/`failurePolicy` knobs on big estates. +- **Size the first run.** A full-history first replication of a large repository moves everything once (idempotent thereafter). Raise `migrate.parallel`, consider `latestOnly: true` for seeding, and note the mover Job's deadline can be tuned via `mover`/`failurePolicy` knobs on big estates. If that first run is what worries the network, cap it with [`migrate.throttle`](#throttling-a-replication) rather than by shrinking the selection — but leave headroom, or the deadline becomes the thing that fails. ## See also From 2cca789832681cf3442691d7c9945eac7031725f Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 12:39:00 -0700 Subject: [PATCH 10/15] docs(shared-types): stop claiming batched deletion honors the throttle (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reworked `moverDefaults.throttle` bullet enumerated the movers that honor it and listed `deletion` — but the SnapshotDeleteBatch and SnapshotPin work specs are built with `throttle: Default::default()` unconditionally, so the generic mover arm applies nothing for them. A precise-looking enumeration that is wrong is worse than the vague claim it replaced: someone capping a repository to protect an object store would not get that cap on bulk deletes, and nothing would say so. Deletion moves to the exceptions, which are now their own paragraph (serve/browse, a RepositoryReplication's destination side, batched deletions and pins), each with its reason, and deletion's actual bound named — per-repository batching plus `maxConcurrentDeleteJobs` — so the gap reads as scoped rather than unqualified. The builders are left alone; populating them stays out of scope. Two adjacent cleanups: - `srepl_terminal` logs `class` for a `MoverError::Kopia` the way `srepl_terminal_kopia` already does, so whether an operator can filter on it no longer depends on which helper a call site picked. - The throttle integration test's prose said M3c/M3d "will ship" the read-write flip sequence. They didn't: the flip was empirically shown unnecessary and `repository set-client` stays unwired. The test now says so and states its own role as a contingency-only guard. --- .../tests/integration_set_client_throttle.rs | 16 ++++++----- crates/mover/src/main.rs | 11 +++++++- docs/reference/crds/shared-types.md | 27 +++++++++++-------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/crates/kopia/tests/integration_set_client_throttle.rs b/crates/kopia/tests/integration_set_client_throttle.rs index 694ddd7b..e4d24281 100644 --- a/crates/kopia/tests/integration_set_client_throttle.rs +++ b/crates/kopia/tests/integration_set_client_throttle.rs @@ -21,9 +21,13 @@ //! directly on a `--readonly` connection — contradicting the design premise //! this milestone was written to confirm (kopia registers it as a write action, //! but it never puts a blob, so read-only storage never rejects it). The -//! sequence M3c/M3d will actually ship is therefore `connect --readonly` → -//! `throttle set` → migrate, with no flip; that is what steps (1b)→(6) assert, -//! end to end, and it is the sequence the timing probe measures. +//! sequence that shipped is therefore `connect --readonly` → `throttle set` → +//! migrate, with **no flip** — that is what steps (1b)→(6) assert, end to end, +//! and it is the sequence the timing probe measures. `repository set-client +//! --read-only/--read-write` consequently stays UNWIRED in the mover flows; +//! the flip coverage below is a contingency guard, kept so that a future kopia +//! turning `throttle set` into a real write action finds the alternative +//! already proven rather than having to discover it under an outage. //! * **…and that sequence leaves the source READ-ONLY.** `throttle set` rewrites //! the config file, so "the bit survives the rewrite" is a data-safety //! invariant for a replication source, not a formality. Step (1c) asserts it @@ -351,9 +355,9 @@ async fn set_client_flip_brackets_throttle_and_migrate_honors_persisted_limits() // (1b) THE PRODUCTION SEQUENCE, and the premise this milestone was written to // check — which does NOT hold as the plan stated it. `repository throttle set` // SUCCEEDS on the read-only connection: it only rewrites the local config's - // `throttlingLimits`, it never puts a blob. So what M3c/M3d will actually ship - // is `connect --readonly` → `throttle set` → migrate, with NO flip, and that - // is the sequence measured in step (6). + // `throttlingLimits`, it never puts a blob. So what shipped is `connect + // --readonly` → `throttle set` → migrate, with NO flip, and that is the + // sequence measured in step (6). // // If a future kopia turns this into a real write action, this fails loudly and // the flip below stops being defensive and becomes mandatory. diff --git a/crates/mover/src/main.rs b/crates/mover/src/main.rs index 110384eb..4cc26042 100644 --- a/crates/mover/src/main.rs +++ b/crates/mover/src/main.rs @@ -3045,6 +3045,12 @@ async fn srepl_terminal_kopia( /// PATCH a terminal failure body for a typed mover error in the snapshot /// replication flow (with whatever run counters exist) and return it. +/// +/// A [`MoverError::Kopia`] arriving here (a throttle application, say) carries a +/// `class` exactly like one going through [`srepl_terminal_kopia`], so it is +/// logged with the same structured field — otherwise whether the operator can +/// filter on `class` would depend on which helper the call site happened to +/// pick. Non-kopia errors have no class to report and log plainly. async fn srepl_terminal( target: &workspec::TargetRef, err: MoverError, @@ -3055,7 +3061,10 @@ async fn srepl_terminal( &snapshot_replicate_failed_body(&err.to_string(), stats), ) .await; - error!("{err}"); + match &err { + MoverError::Kopia { .. } => error!(class = %err.kopia_class(), "{err}"), + _ => error!("{err}"), + } Err(err) } diff --git a/docs/reference/crds/shared-types.md b/docs/reference/crds/shared-types.md index 104eb373..b12c665c 100644 --- a/docs/reference/crds/shared-types.md +++ b/docs/reference/crds/shared-types.md @@ -103,19 +103,24 @@ cache. See [movers](../../movers.md). applied by every mover after it connects to this repository, so a run doesn't saturate the link or hammer the object store: `uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, `writeOpsPerSecond` (each unset - leaves kopia's current limit). Every batch mover honors it — bootstrap, backup, - restore, maintenance, verification, deletion, repository replication and - snapshot replication. kopia's limits are **per connection**, so a run that - opens two repositories caps each from *that* repository's own `moverDefaults`: - a [`SnapshotReplication`](snapshot-replication.md) caps its source connection + leaves kopia's current limit). Honored by the transfer movers — bootstrap, + backup, restore, maintenance, verification, repository replication and snapshot + replication. kopia's limits are **per connection**, so a run that opens two + repositories caps each from *that* repository's own `moverDefaults`: a + [`SnapshotReplication`](snapshot-replication.md) caps its source connection from the source repo's defaults and its destination connection from the destination repo's, each overridable per CR via - [`spec.migrate.throttle`](snapshot-replication.md#migrate). The exceptions: - the interactive [`serve`/browse](../../server.md) session (a read-only UI - session, not a batch transfer), and a `RepositoryReplication`'s **destination** - side — `kopia repository sync-to` copies blobs without opening the destination - as a repository, so its caps live on - [`spec.sync`](repository-replication.md) instead. + [`spec.migrate.throttle`](snapshot-replication.md#migrate). + + Three things it does **not** cap today: the interactive + [`serve`/browse](../../server.md) session (a read-only UI session, not a batch + transfer); a `RepositoryReplication`'s **destination** side, because `kopia + repository sync-to` copies blobs without opening the destination as a + repository — its caps live on [`spec.sync`](repository-replication.md) instead; + and **batched snapshot deletions and pins**, whose work specs leave the throttle + block empty, so a repository-wide cap does not currently reach them (deletion + load is instead bounded by per-repository batching and the cluster-wide + `maxConcurrentDeleteJobs` backstop). /// warning | A byte cap bites much harder than its number suggests From a75c96300f3c0c9ec5a00f5a023bec5e83035fe4 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 13:00:36 -0700 Subject: [PATCH 11/15] feat(repository): seed-migrate throttles (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A seed is the single heaviest transfer kopiur performs — a whole repository pulled across a link on the worst day you have had all year — and migrate mode had no cap surface at all. Blob mode at least offered `sync-to`'s speed flags; a migrate seed offered nothing, because `kopia snapshot migrate` carries no speed flags either. The only lever is `repository throttle set` on each CONNECTION, and a migrate seed opens two under two kopia configs: the replica it reads and the repository it writes. - API: `SeedMigrateOptions.throttle: Option`, reusing the same per-side {source, destination} block SnapshotReplication's `spec.migrate.throttle` uses — here `source` is the REPLICA named by `seed.from.repository` and `destination` is this repository. `SeedMigrateOptions` loses `Copy`. Admission rejects a non-positive rate on either side, naming the side and the knob (the shared `validate_throttle`); a throttle on a BLOB seed stays refused by the existing mode-pairing rule, as an inert field. - Controller: the replica's block is resolved WITH the source it belongs to, inside `seed_op_for`, so the two can never describe different repositories — that repository's own `moverDefaults.throttle` overlaid field-wise by the CR's override, and empty in blob mode. It rides the op as `replicaThrottle`. This repository's side rides the bootstrap work spec's `throttle`, now `seed_bootstrap_throttle`: the same merge, gated on the seed being ARMED so a number chosen for a one-time copy does not go on capping every later connect. Both repository kinds take the resolved block instead of `moverDefaults`, so they cannot drift. - Mover: `seed_connect_source` applies the block in place immediately after the read-only replica connect (no read-write flip — kopia accepts `throttle set` there), BEFORE the persisted-password probe, so the probe sees the config `snapshot migrate` will reopen in its final state. Terminal, like every other application. - Tests: a seed glue guard over two repositories that BOTH set defaults, asserting each side's partial override merges and that neither side's knobs cross into the other, plus the unarmed and blob arms; wire shape + old-wire decode for `replicaThrottle`; per-side validation; a camelCase round-trip through the apiserver path. - E2e: the migrate scenario now throttles the replica repo, the seeded repo and both sides of the CR, reads the CR back to prove the new fields survived admission, and requires the seeding pod to log `applied repository throttle` THREE times — replica, seed-local, post-seed reconnect (the probe connect fails on an uninitialized backend, which is what arms the seed, so it never reaches its own). - Docs: per-side semantics and precedence, why migrate has no speed flags, why the override is armed-only, the measured caveats (byte caps bite far harder than nominal on small-object cold-cache work; warm reads bypass throttling), and the helm-`crds/`-is-install-only trap that silently prunes new fields. --- crates/api/src/seed.rs | 93 ++++++++- crates/api/src/validate/repository.rs | 18 ++ crates/api/src/validate/tests.rs | 60 ++++++ crates/controller/src/cluster_repository.rs | 18 +- crates/controller/src/repo_seed.rs | 180 +++++++++++++++++- crates/controller/src/repository.rs | 40 +++- crates/e2e/tests/seed.rs | 123 +++++++++--- crates/mover/src/bootstrap.rs | 1 + crates/mover/src/main.rs | 49 ++++- crates/mover/src/workspec/mod.rs | 15 ++ crates/mover/src/workspec/tests.rs | 57 ++++++ .../snapshots_test__repository_crd.snap | 79 ++++++++ deploy/crds/all-crds.yaml | 158 +++++++++++++++ deploy/crds/clusterrepositories.yaml | 79 ++++++++ deploy/crds/repositories.yaml | 79 ++++++++ .../42-repository-seed-from-repository.yaml | 29 ++- .../helm/kopiur/crds/clusterrepositories.yaml | 79 ++++++++ deploy/helm/kopiur/crds/repositories.yaml | 79 ++++++++ docs/field-reference.md | 52 +++++ docs/reference/crds/repository.md | 17 ++ docs/reference/crds/shared-types.md | 6 +- docs/repositories.md | 88 +++++++++ 22 files changed, 1342 insertions(+), 57 deletions(-) diff --git a/crates/api/src/seed.rs b/crates/api/src/seed.rs index 4bd7b4a6..4242bb14 100644 --- a/crates/api/src/seed.rs +++ b/crates/api/src/seed.rs @@ -36,7 +36,7 @@ //! standing in a GitOps manifest forever. use crate::backend::Backend; -use crate::common::{CredentialProjection, FailurePolicy, RepositoryRef}; +use crate::common::{CredentialProjection, FailurePolicy, MigrateThrottle, RepositoryRef}; use crate::snapshot_replication::PolicyCopyMode; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -246,7 +246,10 @@ pub struct SeedSyncOptions { } /// Migrate-mode tuning for `kopia snapshot migrate`. -#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Default, JsonSchema)] +/// +/// Scalars plus an optional throttle sub-object, so `Eq` but NOT `Copy` +/// ([`Throttle`](crate::common::Throttle) isn't). +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Default, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SeedMigrateOptions { /// `--parallel`: snapshots migrated concurrently (kopia default `1` — @@ -267,6 +270,24 @@ pub struct SeedMigrateOptions { /// operator's back. #[serde(default)] pub policies: PolicyCopyMode, + /// Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + /// two repositories under two kopia connections and `snapshot migrate` has + /// no speed flags of its own, so each side is applied as `kopia repository + /// throttle set` on that side's connection: + /// + /// * `source` caps the REPLICA — the repository named by + /// `spec.seed.from.repository`, opened read-only — overriding **its** + /// `moverDefaults.throttle`; + /// * `destination` caps THIS repository, the one being seeded, overriding + /// **its own** `moverDefaults.throttle`. + /// + /// Each side overrides field by field: a knob set here wins, a knob left + /// unset keeps that repository's default. Absent: both sides use their + /// repository's defaults. Applies only while the seed is armed — an + /// ordinary connect to the now-initialized repository is capped by + /// `moverDefaults.throttle` alone. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub throttle: Option, } /// What the last seed attempt did, pinned on `Repository`/`ClusterRepository` @@ -501,6 +522,74 @@ credentialProjection: ); } + #[test] + fn migrate_throttle_parses_per_side_and_round_trips() { + // Two repositories, two kopia connections, two independent caps — so + // the wire shape has to keep them apart. Parsed the cluster's way + // (YAML -> JSON value -> typed), which is also what proves the + // camelCase knob names survive. + let seed: SeedSpec = from_yaml( + r#" +from: + repository: + name: offsite +migrate: + parallel: 4 + throttle: + source: + downloadBytesPerSecond: 20000000 + readOpsPerSecond: 500 + destination: + uploadBytesPerSecond: 10000000 +"#, + ); + let throttle = seed + .migrate + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .expect("per-side migrate throttle"); + let source = throttle.source.as_ref().expect("source side"); + assert_eq!(source.download_bytes_per_second, Some(20_000_000)); + assert_eq!(source.read_ops_per_second, Some(500)); + assert_eq!( + source.upload_bytes_per_second, None, + "an unset knob stays unset rather than defaulting to a cap" + ); + let destination = throttle.destination.as_ref().expect("destination side"); + assert_eq!(destination.upload_bytes_per_second, Some(10_000_000)); + assert_eq!( + destination.download_bytes_per_second, None, + "the SOURCE side's knobs must not leak into the destination" + ); + + let json = serde_json::to_value(&seed).expect("serialize"); + assert_eq!( + json.pointer("/migrate/throttle/source/downloadBytesPerSecond"), + Some(&serde_json::json!(20_000_000)), + "{json}" + ); + assert_eq!( + json.pointer("/migrate/throttle/destination/uploadBytesPerSecond"), + Some(&serde_json::json!(10_000_000)), + "{json}" + ); + let reparsed: SeedSpec = serde_json::from_value(json).expect("reparse"); + assert_eq!(seed, reparsed); + + // Absent on a migrate seed that does not cap anything: the common case + // must not grow a key (and must not show up in a GitOps diff). + let plain: SeedSpec = from_yaml( + r#" +from: + repository: + name: offsite +migrate: + parallel: 2 +"#, + ); + assert!(plain.migrate.expect("migrate options").throttle.is_none()); + } + #[test] fn unknown_source_variant_is_rejected() { // An externally-tagged enum must refuse a key it does not know rather diff --git a/crates/api/src/validate/repository.rs b/crates/api/src/validate/repository.rs index dcb47395..6a48aaf8 100644 --- a/crates/api/src/validate/repository.rs +++ b/crates/api/src/validate/repository.rs @@ -1452,6 +1452,24 @@ pub fn validate_repository_seed( { errs.push(e); } + // Per-side seed caps, the same "a rate is >= 1" rule the throttle + // knobs carry everywhere. Validated per side so the message names + // the side that is wrong: `source` is the REPLICA being read, + // `destination` this repository — and mixing them up is the likely + // authoring mistake. + if let Some(throttle) = seed.migrate.as_ref().and_then(|m| m.throttle.as_ref()) { + for (side, block) in [ + ("source", throttle.source.as_ref()), + ("destination", throttle.destination.as_ref()), + ] { + if let Some(block) = block { + errs.extend(validate_throttle( + &format!("spec.seed.migrate.throttle.{side}"), + block, + )); + } + } + } } } errs diff --git a/crates/api/src/validate/tests.rs b/crates/api/src/validate/tests.rs index 6daa29c6..1e00f0d6 100644 --- a/crates/api/src/validate/tests.rs +++ b/crates/api/src/validate/tests.rs @@ -6761,6 +6761,66 @@ fn seed_numeric_knobs_and_failure_policy_must_be_positive() { ); } +#[test] +fn a_seed_migrate_throttle_is_rejected_per_side_and_names_the_knob() { + // #374: a cap is a positive rate on either side. Both sides are checked + // independently and the message names the side, because "source" is the + // REPLICA and "destination" is this repository — mixing them up is the + // likely authoring mistake, and a message that only named the knob would + // not help at all. + let errs = validate_repository(&repo_with_seed(&format!( + "{MIGRATE_SEED} migrate:\n throttle:\n source: {{ downloadBytesPerSecond: 0 }}\n \ + destination: {{ uploadBytesPerSecond: -1, writeOpsPerSecond: 0 }}\n" + ))); + let fields: Vec = errs + .iter() + .filter_map(|e| match e { + ValidationError::InvalidFieldValue { field, .. } => Some(field.clone()), + _ => None, + }) + .collect(); + assert_eq!( + fields, + vec![ + "spec.seed.migrate.throttle.source.downloadBytesPerSecond", + "spec.seed.migrate.throttle.destination.uploadBytesPerSecond", + "spec.seed.migrate.throttle.destination.writeOpsPerSecond", + ], + "every offending knob is named, source side first: {errs:?}" + ); + + // 1 is the smallest accepted rate, unset knobs are always fine, and an + // empty block constrains nothing (both sides inherit their repository's + // moverDefaults). + assert!( + validate_repository(&repo_with_seed(&format!( + "{MIGRATE_SEED} migrate:\n throttle:\n source: {{ downloadBytesPerSecond: 1 }}\n \ + destination: {{ uploadBytesPerSecond: 1, readOpsPerSecond: 1, writeOpsPerSecond: 1 }}\n" + ))) + .is_empty() + ); + assert!( + validate_repository(&repo_with_seed(&format!( + "{MIGRATE_SEED} migrate: {{ throttle: {{}} }}\n" + ))) + .is_empty() + ); + + // A throttle on a BLOB seed is refused as an inert field by the existing + // mode-pairing rule — `migrate` belongs to `from.repository`, throttle and + // all — rather than silently doing nothing. + let errs = validate_repository(&repo_with_seed(&format!( + "{BLOB_SEED} migrate: {{ throttle: {{ source: {{ downloadBytesPerSecond: 1 }} }} }}\n" + ))); + assert!( + errs.iter().any(|e| matches!( + e, + ValidationError::SeedTuningNotApplicable { field, .. } if field == "migrate" + )), + "{errs:?}" + ); +} + #[test] fn a_cluster_repository_seed_ref_must_not_carry_a_namespace() { // `validate_repository_ref`'s existing rule reaches the seed source too. diff --git a/crates/controller/src/cluster_repository.rs b/crates/controller/src/cluster_repository.rs index be974476..e1648196 100644 --- a/crates/controller/src/cluster_repository.rs +++ b/crates/controller/src/cluster_repository.rs @@ -1380,7 +1380,11 @@ async fn bootstrap_cluster_via_mover( &job_ns, create_enabled, repo.spec.create.as_ref(), - repo.spec.mover_defaults.as_ref(), + crate::repo_seed::seed_bootstrap_throttle( + seed_armed, + repo.spec.seed.as_ref(), + repo.spec.mover_defaults.as_ref(), + ), cluster, foreign, read_only, @@ -1621,7 +1625,10 @@ fn cluster_bootstrap_work_spec( job_ns: &str, auto_create: bool, create: Option<&kopiur_api::common::CreateBehavior>, - mover_defaults: Option<&kopiur_api::common::MoverDefaults>, + // The cap for every connection this bootstrap opens to THIS repository, + // resolved by the caller through `repo_seed::seed_bootstrap_throttle` — + // identical rule to the `Repository` twin, so the two kinds cannot drift. + throttle: kopiur_mover::workspec::ThrottleSpec, cluster: Option<&str>, foreign: ForeignSnapshots, read_only: bool, @@ -1694,7 +1701,7 @@ fn cluster_bootstrap_work_spec( options: MoverOptions::default(), // Bootstrap is a connect/create probe, not a data run: kopia defaults. cache: Default::default(), - throttle: io::throttle_spec(mover_defaults), + throttle, } } @@ -2581,7 +2588,7 @@ mod tests { "kopia-system", true, None, - None, + Default::default(), None, ForeignSnapshots::Fallback, read_only, @@ -2619,6 +2626,7 @@ mod tests { migrate: None, allow_empty_source: true, resume: false, + replica_throttle: Default::default(), }), ); let carried = armed.seed.expect("the seed rides the op"); @@ -2650,7 +2658,7 @@ mod tests { "kopia-system", true, None, - None, + Default::default(), cluster, ForeignSnapshots::Fallback, read_only, diff --git a/crates/controller/src/repo_seed.rs b/crates/controller/src/repo_seed.rs index 614f13e8..8639515b 100644 --- a/crates/controller/src/repo_seed.rs +++ b/crates/controller/src/repo_seed.rs @@ -24,7 +24,7 @@ use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector}; use kube::api::Api; use kopiur_api::backend::Backend; -use kopiur_api::common::{RepositoryKind, RepositoryRef}; +use kopiur_api::common::{MoverDefaults, RepositoryKind, RepositoryRef, Throttle}; use kopiur_api::seed::{SeedMode, SeedSource, SeedSpec, SeedStatus}; use kopiur_mover::bootstrap::SeedOutcome; use kopiur_mover::jobs::VolumeMountSpec; @@ -155,10 +155,63 @@ pub(crate) struct SeedSourceRepository<'a> { pub kind: RepositoryKind, /// The source CR's name. pub name: &'a str, - /// The source's resolved backend/encryption/CA surface. + /// The source's resolved backend/encryption/CA surface — and its + /// `moverDefaults`, which is where the REPLICA side's throttle comes from + /// (kopia's limits are per connection, so the replica's cap can only be the + /// replica's own default, never the seeded repository's). pub repo: &'a ResolvedRepository, } +/// **Pure.** The per-CR override for ONE side of a migrate seed's copy, or +/// `None` when this seed caps nothing (blob mode included — `spec.seed.migrate` +/// is refused alongside `from.backend`, so the block cannot exist there). +/// +/// Two named accessors rather than one indexed helper because the two sides are +/// two different repositories with two different failure modes, and a caller +/// that picked the wrong one would produce a run that caps the link it meant to +/// leave alone. +fn seed_source_throttle(seed: &SeedSpec) -> Option<&Throttle> { + seed.migrate + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.source.as_ref()) +} + +/// The DESTINATION-side counterpart of [`seed_source_throttle`]: the override +/// for THIS repository, the one being seeded. +fn seed_destination_throttle(seed: &SeedSpec) -> Option<&Throttle> { + seed.migrate + .as_ref() + .and_then(|m| m.throttle.as_ref()) + .and_then(|t| t.destination.as_ref()) +} + +/// **Pure.** The throttle a bootstrap work spec carries — the cap that lands on +/// every connection the bootstrap mover opens to THIS repository. +/// +/// Ordinarily that is just this repository's own `moverDefaults.throttle`. While +/// a seed is ARMED it also picks up `spec.seed.migrate.throttle.destination`, +/// field by field: a seeding bootstrap's local connect is the write side of the +/// heaviest transfer kopiur performs, and `kopia snapshot migrate` has no speed +/// flags of its own, so the limits persisted into that connection's config are +/// the only cap it can honor. +/// +/// Gated on `armed` deliberately. The destination override describes the SEED +/// RUN, not the repository; leaving it in force on every later connect to the +/// now-initialized repository would cap routine work with a number chosen for a +/// one-time copy. `armed == false` therefore reproduces the pre-#374 value byte +/// for byte. +pub(crate) fn seed_bootstrap_throttle( + armed: bool, + seed: Option<&SeedSpec>, + defaults: Option<&MoverDefaults>, +) -> kopiur_mover::workspec::ThrottleSpec { + io::merged_throttle( + defaults, + seed.filter(|_| armed).and_then(seed_destination_throttle), + ) +} + /// **Pure.** Build the mover's seed payload from `spec.seed`. /// /// `blob_ca_bundle_pem` is the resolved `tls.caBundleRef` content of a BLOB @@ -181,11 +234,21 @@ pub(crate) fn seed_op_for( source: Option<&SeedSourceRepository<'_>>, resume: bool, ) -> Option { - let from = match (&seed.from, source) { - (SeedSource::Backend(backend), _) => SeedConnectSource::Backend(Box::new( - backend_to_repository_connect(backend, blob_ca_bundle_pem), - )), - (SeedSource::Repository(_), Some(src)) => { + // Resolved together with the source it belongs to, so the two can never + // describe different repositories: kopia's limits are per CONNECTION, and + // the replica's cap comes from the REPLICA's own `moverDefaults.throttle` + // (overlaid field-wise by `spec.seed.migrate.throttle.source`) — never from + // the repository being seeded. Blob mode caps its copy through `sync-to`'s + // own speed flags instead, so it carries none. + let (from, replica_throttle) = match (&seed.from, source) { + (SeedSource::Backend(backend), _) => ( + SeedConnectSource::Backend(Box::new(backend_to_repository_connect( + backend, + blob_ca_bundle_pem, + ))), + Default::default(), + ), + (SeedSource::Repository(_), Some(src)) => ( SeedConnectSource::Repository(Box::new(SeedRepositoryConnect { kind: io::repo_kind_str(src.kind).to_string(), name: src.name.to_string(), @@ -194,8 +257,9 @@ pub(crate) fn seed_op_for( &src.repo.backend, src.repo.ca_bundle_pem.clone(), ), - })) - } + })), + io::merged_throttle(src.repo.mover_defaults.as_ref(), seed_source_throttle(seed)), + ), (SeedSource::Repository(_), None) => return None, }; Some(SeedOpSpec { @@ -211,13 +275,14 @@ pub(crate) fn seed_op_for( max_download_speed_bytes_per_second: s.max_download_speed_bytes_per_second, max_upload_speed_bytes_per_second: s.max_upload_speed_bytes_per_second, }), - migrate: seed.migrate.map(|m| SeedMigrateSpec { + migrate: seed.migrate.as_ref().map(|m| SeedMigrateSpec { parallel: m.parallel, latest_only: m.latest_only, policies: crate::snapshot_replication::policy_copy_mode_spec(m.policies), }), allow_empty_source: seed.allow_empty_source, resume, + replica_throttle, }) } @@ -1100,6 +1165,101 @@ mod tests { } } + /// The #374 seed glue guard: two repositories, two kopia connections, two + /// independently resolved caps — and neither side's knobs may reach the + /// other. A crossed merge is invisible in production until the wrong link + /// gets saturated, which is exactly what this feature exists to prevent. + #[test] + fn seed_throttles_resolve_per_side_from_each_repositorys_own_defaults() { + use kopiur_api::common::{MoverDefaults, Throttle}; + let throttle = |up, down, r, w| Throttle { + upload_bytes_per_second: up, + download_bytes_per_second: down, + read_ops_per_second: r, + write_ops_per_second: w, + }; + // The REPLICA brings its own defaults; so does the repository being + // seeded. The CR overrides ONE knob per side, so a correct merge shows + // the override AND that side's surviving default — and nothing else. + let mut src = resolved_source(s3("source-bucket"), Some("backups")); + src.mover_defaults = Some(MoverDefaults { + throttle: Some(throttle(None, Some(1), Some(2), None)), + ..Default::default() + }); + let local_defaults = MoverDefaults { + throttle: Some(throttle(Some(3), None, None, Some(4))), + ..Default::default() + }; + let seed = seed_spec(serde_json::json!({ + "from": { "repository": { "name": "offsite" } }, + "migrate": { "throttle": { + "source": { "downloadBytesPerSecond": 11 }, + "destination": { "uploadBytesPerSecond": 22 }, + } }, + })); + let source = SeedSourceRepository { + kind: RepositoryKind::Repository, + name: "offsite", + repo: &src, + }; + let op = seed_op_for(&seed, None, Some(&source), false).expect("migrate op"); + // REPLICA side: the CR's download override wins, the replica repo's + // readOps default survives, and NOTHING from the local repository's + // defaults leaks in. + assert_eq!(op.replica_throttle.download_bytes_per_second, Some(11)); + assert_eq!(op.replica_throttle.read_ops_per_second, Some(2)); + assert_eq!( + op.replica_throttle.upload_bytes_per_second, None, + "the SEEDED repository's upload default must not reach the replica" + ); + assert_eq!( + op.replica_throttle.write_ops_per_second, None, + "the SEEDED repository's writeOps default must not reach the replica" + ); + + // LOCAL (destination) side rides the bootstrap work spec's own throttle: + // the CR's upload override wins, this repository's writeOps default + // survives, and the REPLICA's knobs stay out. + let armed = seed_bootstrap_throttle(true, Some(&seed), Some(&local_defaults)); + assert_eq!(armed.upload_bytes_per_second, Some(22)); + assert_eq!(armed.write_ops_per_second, Some(4)); + assert_eq!( + armed.download_bytes_per_second, None, + "the REPLICA's download default must not reach the seeded repository" + ); + assert_eq!(armed.read_ops_per_second, None); + + // NOT armed — every ordinary bootstrap, including every later connect to + // the now-initialized repository: byte-for-byte this repository's own + // defaults, with the seed-run override nowhere in sight. + let unarmed = seed_bootstrap_throttle(false, Some(&seed), Some(&local_defaults)); + assert_eq!(unarmed.upload_bytes_per_second, Some(3)); + assert_eq!(unarmed.write_ops_per_second, Some(4)); + assert!(unarmed.download_bytes_per_second.is_none()); + + // A migrate seed that caps nothing leaves both sides on their + // repositories' defaults (and an all-empty pair skips `throttle set`). + let plain = seed_spec(serde_json::json!({ + "from": { "repository": { "name": "offsite" } }, + })); + let op = seed_op_for(&plain, None, Some(&source), false).expect("migrate op"); + assert_eq!(op.replica_throttle.download_bytes_per_second, Some(1)); + assert!(seed_bootstrap_throttle(true, Some(&plain), None).is_empty()); + + // BLOB mode has no source repository CR at all — its copy is capped by + // `sync-to`'s own speed flags — so the replica block stays empty rather + // than inheriting this repository's defaults. + let blob = seed_spec(serde_json::json!({ + "from": { "backend": { "s3": { "bucket": "offsite" } } }, + "sync": { "maxDownloadSpeedBytesPerSecond": 20000000 }, + })); + let op = seed_op_for(&blob, None, None, false).expect("blob op"); + assert!(op.replica_throttle.is_empty()); + // …and admission refuses `migrate` beside `from.backend`, so a blob seed + // can never carry a destination override either. + assert!(seed_bootstrap_throttle(true, Some(&blob), None).is_empty()); + } + #[test] fn a_migrate_seed_without_a_resolved_source_yields_no_payload() { // Unreachable from the caller (it resolves or parks first) and diff --git a/crates/controller/src/repository.rs b/crates/controller/src/repository.rs index 81cecd3a..01e882fc 100644 --- a/crates/controller/src/repository.rs +++ b/crates/controller/src/repository.rs @@ -1302,7 +1302,11 @@ async fn bootstrap_via_mover( create_enabled, true, repo.spec.create.as_ref(), - repo.spec.mover_defaults.as_ref(), + repo_seed::seed_bootstrap_throttle( + seed_armed, + repo.spec.seed.as_ref(), + repo.spec.mover_defaults.as_ref(), + ), cluster, foreign, read_only, @@ -1632,7 +1636,11 @@ fn bootstrap_work_spec( auto_create: bool, scan_catalog: bool, create: Option<&kopiur_api::common::CreateBehavior>, - mover_defaults: Option<&kopiur_api::common::MoverDefaults>, + // The cap for every connection this bootstrap opens to THIS repository, + // resolved by the caller through `repo_seed::seed_bootstrap_throttle`: the + // repository's own `moverDefaults.throttle`, plus — while a seed is armed — + // `spec.seed.migrate.throttle.destination` over it, field by field. + throttle: kopiur_mover::workspec::ThrottleSpec, cluster: Option<&str>, foreign: ForeignSnapshots, read_only: bool, @@ -1709,8 +1717,10 @@ fn bootstrap_work_spec( options: MoverOptions::default(), // Bootstrap is a connect/create probe, not a data run: kopia defaults. cache: Default::default(), - // Apply the repo throttle on the bootstrap connection too (§13(e)). - throttle: io::throttle_spec(mover_defaults), + // Apply the repo throttle on the bootstrap connection too (§13(e)) — + // and, on a seeding bootstrap, on the seed's own local connect, which + // is the only cap `kopia snapshot migrate` can honor on the write side. + throttle, } } @@ -2715,7 +2725,7 @@ mod tests { true, true, None, - None, + Default::default(), None, ForeignSnapshots::Fallback, read_only, @@ -2752,6 +2762,7 @@ mod tests { migrate: None, allow_empty_source: false, resume: true, + replica_throttle: Default::default(), }; let armed = build(false, Some(op.clone())); let carried = armed.seed.expect("the seed rides the op"); @@ -2861,8 +2872,21 @@ mod tests { }); let prefilter = |cluster: Option<&str>, foreign: ForeignSnapshots| { let spec = bootstrap_work_spec( - &backend, "nas", "billing", true, true, None, None, cluster, foreign, false, true, - false, None, None, None, + &backend, + "nas", + "billing", + true, + true, + None, + Default::default(), + cluster, + foreign, + false, + true, + false, + None, + None, + None, ); match spec.operation { Operation::BootstrapRepository(op) => op.catalog_foreign_prefilter_cluster, @@ -2902,7 +2926,7 @@ mod tests { true, true, None, - None, + Default::default(), cluster, ForeignSnapshots::Fallback, read_only, diff --git a/crates/e2e/tests/seed.rs b/crates/e2e/tests/seed.rs index 3a347fa2..e6822987 100644 --- a/crates/e2e/tests/seed.rs +++ b/crates/e2e/tests/seed.rs @@ -516,7 +516,33 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident // DIFFERENT from the shared `kopia-creds` password — the whole point. const SEEDED_PASSWORD: &str = "e2e-seed-migrate-password-789"; - // The source repository, with real history under a real identity. + // The source repository, with real history under a real identity. Created + // HERE rather than by `ensure_seed` (which is idempotent and will find it + // present) so the REPLICA can carry its own `moverDefaults.throttle` — the + // base layer the per-CR `seed.migrate.throttle.source` override below has to + // merge over. Caps are deliberately GENEROUS: the point is to prove the + // limits reach kopia on the replica's connection, and a low byte cap is far + // more punitive than nominal on a small-object cold-cache migrate, which + // would only make this scenario slow and flaky. + ensure_repo(&client, "seed-mig-src").await; + { + let repos: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); + create_idempotent( + &repos, + &cr(repository_json( + SOURCE, + "seed-mig-src", + serde_json::json!({ + "moverDefaults": { "throttle": { + "downloadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND, + "readOpsPerSecond": 10000 + } } + }), + )), + "create the seed SOURCE Repository with moverDefaults.throttle", + ) + .await; + } ensure_seed(&client, SOURCE, POLICY, BACKUP, "seed-mig-src").await; let backups: Api = Api::namespaced(client.clone(), E2E_NAMESPACE); let source_identity = snapshot_identity(&backups, BACKUP).await; @@ -553,23 +579,33 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident // itself, and a seed-armed bootstrap never takes the create // fallback — so reaching Ready here proves both at once. "create": { "enabled": false }, - // #374: a migrate seed opens TWO connections to this repository - // under two kopia configs — the seed's own local client (which - // `snapshot migrate` reopens, and which has no speed flags of its - // own, making this the ONLY cap on the heaviest transfer kopiur - // performs) and the post-seed bootstrap reconnect. Both must be - // capped; the assertion below counts the mover's log line per pod - // to prove the seed-local one specifically fired. Non-binding at - // 100 MiB/s over a local hostPath. + // #374: a migrate seed opens THREE kopia connections in one pod + // — the replica (read-only), this repository's own seed-local + // client (which `snapshot migrate` reopens, and which has no + // speed flags of its own, making it the ONLY cap on the heaviest + // transfer kopiur performs) and the post-seed bootstrap + // reconnect. All must be capped; the assertion below counts the + // mover's log line per pod. Non-binding at 100 MiB/s over a + // local hostPath. "moverDefaults": { "throttle": { "uploadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND, - "downloadBytesPerSecond": consts::THROTTLE_BYTES_PER_SECOND + "writeOpsPerSecond": 10000 } }, "seed": { "from": { "repository": { "kind": "Repository", "name": SOURCE } }, - "migrate": { "parallel": 2 } + // Per-CR, per-side overrides (#374). Each side sets ONE knob, + // so the merged block must also carry the knob that side's + // repository defaulted — the field-wise merge, end to end. + // `source` is the REPLICA, `destination` this repository. + "migrate": { + "parallel": 2, + "throttle": { + "source": { "downloadBytesPerSecond": 209715200 }, + "destination": { "uploadBytesPerSecond": 209715200 } + } + } } }), )), @@ -597,6 +633,39 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident Some(consts::THROTTLE_BYTES_PER_SECOND), "moverDefaults.throttle must survive onto the seeded Repository" ); + // The per-side seed overrides too — an apiserver whose CRD schema predates + // them PRUNES them silently, which is the exact #374 symptom this change is + // about, and the log-count assertion below would then pass for the wrong + // reason (the repositories' own defaults alone). + { + let seed = live + .spec + .seed + .as_ref() + .and_then(|s| s.migrate.as_ref()) + .and_then(|m| m.throttle.as_ref()) + .unwrap_or_else(|| { + panic!( + "spec.seed.migrate.throttle survived admission (a pruned field means a stale \ + CRD): {:?}", + live.spec.seed + ) + }); + assert_eq!( + seed.source + .as_ref() + .and_then(|t| t.download_bytes_per_second), + Some(209_715_200), + "the REPLICA-side override must survive: {seed:?}" + ); + assert_eq!( + seed.destination + .as_ref() + .and_then(|t| t.upload_bytes_per_second), + Some(209_715_200), + "the DESTINATION-side override must survive: {seed:?}" + ); + } wait_seeded(&repos, SEEDED, "True", "Seeded").await; wait_phase(&repos, SEEDED, "Ready") @@ -619,25 +688,35 @@ async fn migrate_seed_copies_history_from_another_repository_and_keeps_its_ident "a migrate seed must report the manifests it copied; status.seed: {seed_status}" ); - // #374 regression guard for the highest-value call site of the fix: the - // seed's LOCAL repository connect. This one bootstrap pod opens two capped - // connections — the seed's local client (under its own kopia config, the - // only cap `snapshot migrate` can honor) and the post-seed reconnect — so it - // must log the line TWICE. Counting per pod is load-bearing: the same - // selector also matches later catalog-rescan/probe pods that log it once - // each, and a bare `contains` (or a count across pods) would stay green with - // the seed-local call deleted. + // #374 regression guard for the highest-value call sites of the fix: every + // connection a migrate seed opens. The arithmetic for THIS pod, which is why + // the number is 3: + // + // probe connect 0 — the backend is uninitialized, so the connect FAILS + // (that failure is what arms the seed) and never + // reaches its throttle; + // replica connect +1 — the read-only source open, capped from + // `replicaThrottle` (source repo defaults ⊕ + // seed.migrate.throttle.source); + // seed-local +1 — this repository under its own kopia config, the + // only cap `snapshot migrate` can honor; + // post-seed +1 — the bootstrap reconnect after the copy. + // + // Counting per pod is load-bearing: the same selector also matches later + // catalog-rescan/probe pods that log it once each, and a bare `contains` (or + // a count across pods) would stay green with any one application deleted. kopiur_e2e::wait::wait_for_pod_log_times( &client, E2E_NAMESPACE, &format!("kopiur.home-operations.com/repository={SEEDED}"), consts::THROTTLE_APPLIED_LOG, - 2, + 3, ) .await .expect( - "the seeding mover must cap BOTH the seed's local connect and the post-seed \ - reconnect (#374); one occurrence means the seed-local throttle is gone", + "the seeding mover must cap ALL THREE of its connections — the replica read, the \ + seed's local connect and the post-seed reconnect (#374); two occurrences means one \ + of them is uncapped", ); // Identity survives the migrate, so a restore keyed on the SOURCE's diff --git a/crates/mover/src/bootstrap.rs b/crates/mover/src/bootstrap.rs index 01a31ba1..d2abbcf1 100644 --- a/crates/mover/src/bootstrap.rs +++ b/crates/mover/src/bootstrap.rs @@ -1010,6 +1010,7 @@ mod tests { migrate: None, allow_empty_source: false, resume, + replica_throttle: Default::default(), } } diff --git a/crates/mover/src/main.rs b/crates/mover/src/main.rs index 4cc26042..ffa1920f 100644 --- a/crates/mover/src/main.rs +++ b/crates/mover/src/main.rs @@ -417,8 +417,10 @@ async fn connect_and_throttle( /// directly and then throttled in place from its own resolved block (the work /// spec's `throttle` for the source, the op's `destinationThrottle` for the /// destination). Both applications are terminal, via the flow's `srepl_terminal`. -/// * **a seed's SOURCE (replica) connect** — same shape, wired by M3d together -/// with its `replica_throttle`. +/// * **a seed's SOURCE (replica) connect** — same shape: the seed holds its own +/// repository and the replica open at once under two kopia configs, so the +/// replica is connected directly and then throttled in place from the op's +/// `replicaThrottle`. Terminal, and reported as the bootstrap's failure. /// /// No limits set → no kopia process is spawned (the common case stays free). /// @@ -1361,6 +1363,13 @@ fn seed_source_builder( /// `--persist-credentials` writes the password beside the config, which is what /// `snapshot migrate --source-config` reads in migrate mode; it is harmless in /// blob mode and kept symmetrical so both arms share this function. +/// +/// The connection is capped in place from the op's `replicaThrottle` right after +/// it opens — this is one of the connects that cannot go through +/// [`connect_and_throttle`], because a seed holds two repositories open under two +/// kopia configs and the work spec's own `throttle` belongs to the other one. +/// Empty in blob mode (no source repository CR to take `moverDefaults` from; its +/// caps ride `sync-to`'s own speed flags), so that arm spawns no kopia process. async fn seed_connect_source( spec: &MoverWorkSpec, seed: &workspec::SeedOpSpec, @@ -1398,12 +1407,35 @@ async fn seed_connect_source( ) .await { - Ok(()) => Ok(SeedSource { - client, - connect: source, - }), - Err(e) => Err(Box::new(seed_source_connect_failure(seed, &e))), + Ok(()) => {} + Err(e) => return Err(Box::new(seed_source_connect_failure(seed, &e))), + } + // Cap the REPLICA read side on this connection, from the op's own + // `replicaThrottle` (the SOURCE repository's `moverDefaults.throttle` + // overlaid by `spec.seed.migrate.throttle.source`). kopia's limits are + // per-connection, so the cap the bootstrap put on THIS repository says + // nothing here — and in migrate mode `snapshot migrate` reopens this very + // config, which is the only lever it has (it carries no speed flags). + // Applied straight onto the read-only connect: `repository throttle set` is + // accepted there, so no read-write flip is involved. + // + // Deliberately BEFORE migrate mode's persisted-password probe: that probe's + // whole job is to prove the config `snapshot migrate` will reopen is usable, + // so it should see that config in its FINAL state — throttle limits written + // and all. + // + // Terminal, like every other throttle application: proceeding would pull a + // whole repository across exactly the link the user asked to protect, and a + // seed is the largest read kopiur ever makes of a replica that is usually + // still another cluster's live off-site copy. + if let Err(e) = apply_repository_throttle(&client, &seed.replica_throttle).await { + error!(class = %e.kopia_class(), "could not cap the seed source connection"); + return Err(Box::new(BootstrapResult::from_mover_error(&e))); } + Ok(SeedSource { + client, + connect: source, + }) } /// Classify a failed seed-source connect. Split out so the "the backend @@ -4014,7 +4046,8 @@ mod tests { ), ( "seed_connect_source", - "seed SOURCE/replica; its own cap lands with M3d (1 call)", + "seed SOURCE/replica; dual-config connect, throttled in place from \ + op.replicaThrottle immediately after (1 call)", ), ( "srepl_connect_source", diff --git a/crates/mover/src/workspec/mod.rs b/crates/mover/src/workspec/mod.rs index a8c709c4..3bee77ab 100644 --- a/crates/mover/src/workspec/mod.rs +++ b/crates/mover/src/workspec/mod.rs @@ -779,6 +779,21 @@ pub struct SeedOpSpec { /// `(SourceInfo, StartTime)`. Absent on old work specs (serde default). #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub resume: bool, + /// Throttle limits for the seed SOURCE (replica) connection — applied with + /// `kopia repository throttle set` right after that read-only connect. + /// + /// THIS repository's own side rides the work spec's + /// [`MoverWorkSpec::throttle`]: a migrate seed opens two repositories under + /// two kopia configs, and kopia's limits are per connection, so each side + /// needs its own resolved block. The controller merges the SOURCE + /// repository's `moverDefaults.throttle` with + /// `spec.seed.migrate.throttle.source` field by field; blob mode leaves it + /// empty (its source has no repository CR, and its caps ride + /// [`SeedSyncSpec`]'s `sync-to` speed flags). All-`None` ⇒ the mover skips + /// the call. Absent on old work specs (serde default ⇒ an uncapped replica + /// read, the pre-#374 behavior). + #[serde(default, skip_serializing_if = "ThrottleSpec::is_empty")] + pub replica_throttle: ThrottleSpec, } impl SeedOpSpec { diff --git a/crates/mover/src/workspec/tests.rs b/crates/mover/src/workspec/tests.rs index e43ea5cb..2eeb4ce1 100644 --- a/crates/mover/src/workspec/tests.rs +++ b/crates/mover/src/workspec/tests.rs @@ -2404,6 +2404,9 @@ fn blob_seed() -> SeedOpSpec { migrate: None, allow_empty_source: false, resume: false, + // Blob mode caps its copy through `sync-to`'s own speed flags above; + // there is no source repository CR to carry `moverDefaults`. + replica_throttle: ThrottleSpec::default(), } } @@ -2426,6 +2429,11 @@ fn migrate_seed() -> SeedOpSpec { }), allow_empty_source: true, resume: false, + replica_throttle: ThrottleSpec { + download_bytes_per_second: Some(20 * 1024 * 1024), + read_ops_per_second: Some(200), + ..Default::default() + }, } } @@ -2550,6 +2558,55 @@ fn resume_rides_the_wire_and_defaults_off_on_old_specs() { assert_eq!(serde_json::from_value::(v).unwrap(), resuming); } +#[test] +fn the_replica_throttle_rides_the_seed_op_and_defaults_empty_on_old_specs() { + // The SOURCE (replica) side's cap rides the op; THIS repository's side rides + // the work spec's own `throttle`. Two repositories, two kopia connections, + // two independent blocks — a migrate seed is the only flow that opens both. + let v = serde_json::to_value(migrate_seed()).unwrap(); + assert_eq!( + v["replicaThrottle"]["downloadBytesPerSecond"], + 20 * 1024 * 1024 + ); + assert_eq!(v["replicaThrottle"]["readOpsPerSecond"], 200); + assert!( + v["replicaThrottle"].get("uploadBytesPerSecond").is_none(), + "unset replica knobs are elided: {v}" + ); + assert_eq!( + serde_json::from_value::(v).unwrap(), + migrate_seed() + ); + + // The common case must not grow a key: an all-None block is the mover's + // "skip `throttle set`" signal, so it serializes to nothing at all. + let uncapped = SeedOpSpec { + replica_throttle: ThrottleSpec::default(), + ..migrate_seed() + }; + assert!( + serde_json::to_value(&uncapped) + .unwrap() + .get("replicaThrottle") + .is_none(), + "an empty replica throttle is elided from the wire" + ); + + // Old→new: a spec stamped before `replicaThrottle` existed decodes to an + // empty block, i.e. an old controller driving a new mover leaves the replica + // read uncapped rather than failing to decode. + let old: SeedOpSpec = serde_json::from_value(serde_json::json!({ + "from": { "repository": { + "kind": "Repository", + "name": "offsite", + "connect": { "filesystem": { "path": "/mnt/offsite" } } + } }, + "sourceDescription": "Repository/offsite" + })) + .unwrap(); + assert!(old.replica_throttle.is_empty()); +} + #[test] fn seed_migrate_policies_default_to_an_explicit_no_policies() { // kopia's OWN default copies the source's policies, retention included, diff --git a/crates/xtask/tests/snapshots/snapshots_test__repository_crd.snap b/crates/xtask/tests/snapshots/snapshots_test__repository_crd.snap index aa53fb1c..e90adff9 100644 --- a/crates/xtask/tests/snapshots/snapshots_test__repository_crd.snap +++ b/crates/xtask/tests/snapshots/snapshots_test__repository_crd.snap @@ -2949,6 +2949,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/deploy/crds/all-crds.yaml b/deploy/crds/all-crds.yaml index 68887d51..9f22ceca 100644 --- a/deploy/crds/all-crds.yaml +++ b/deploy/crds/all-crds.yaml @@ -2946,6 +2946,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- @@ -6723,6 +6802,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/deploy/crds/clusterrepositories.yaml b/deploy/crds/clusterrepositories.yaml index b84afb33..33bf31f6 100644 --- a/deploy/crds/clusterrepositories.yaml +++ b/deploy/crds/clusterrepositories.yaml @@ -2999,6 +2999,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/deploy/crds/repositories.yaml b/deploy/crds/repositories.yaml index 3ea9ba08..a64a0301 100644 --- a/deploy/crds/repositories.yaml +++ b/deploy/crds/repositories.yaml @@ -2946,6 +2946,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/deploy/examples/42-repository-seed-from-repository.yaml b/deploy/examples/42-repository-seed-from-repository.yaml index 0a0f209c..e425e30d 100644 --- a/deploy/examples/42-repository-seed-from-repository.yaml +++ b/deploy/examples/42-repository-seed-from-repository.yaml @@ -33,7 +33,8 @@ # absent namespace resolves in this CR's own namespace); `seed.migrate` is # migrate-mode-only tuning and is rejected alongside a `backend:` source; # `policies` is the same enum SnapshotReplication uses and defaults to `none` -# (an explicit `--no-policies`). +# (an explicit `--no-policies`); `seed.migrate.throttle` is the same per-side +# {source, destination} shape SnapshotReplication's `spec.migrate.throttle` uses. # # The `# --8<-- [start:…]` / `# --8<-- [end:…]` lines are snippet-section markers # (docs/repositories.md pulls the Repository on its own). Plain YAML comments — @@ -101,6 +102,32 @@ spec: # window; imported kopia policies could delete manifests behind the # operator's back. policies: none + # PER-SIDE bandwidth/ops caps for THIS seed. `kopia snapshot migrate` has + # no speed flags of its own, so each side is applied as `kopia repository + # throttle set` on that side's connection before the copy runs — and a + # migrate seed is two repositories under two kopia configs, so the two caps + # are independent. `source` is the REPLICA being read; `destination` is + # THIS repository, the one being written. + # + # Each side OVERRIDES that side's repository's `moverDefaults.throttle` + # field by field: a knob set here wins, a knob left unset keeps that + # repository's value. Only applied while the seed is armed — later connects + # to this (now-initialized) repository are capped by `moverDefaults.throttle` + # alone. + # + # Every knob set must be >= 1 (a 0 is rejected: kopia's "no limit" is the + # ABSENT knob). Caps bind only COLD backend traffic, and on small objects + # bite far harder than the nominal rate — set them as a generous ceiling + # for the bulk transfer, and remember a seed already runs against + # failurePolicy.activeDeadlineSeconds below. + throttle: + source: + # Protect the off-site link this replica lives behind. + downloadBytesPerSecond: 10485760 # 10 MiB/s + readOpsPerSecond: 200 + destination: + # Writing into the new primary — usually the roomier side. + uploadBytesPerSecond: 209715200 # 200 MiB/s allowEmptySource: false # Copy the SOURCE repository's credential Secrets into the seeding Job's # namespace for the duration of the run. Needed whenever the source's Secrets diff --git a/deploy/helm/kopiur/crds/clusterrepositories.yaml b/deploy/helm/kopiur/crds/clusterrepositories.yaml index b84afb33..33bf31f6 100644 --- a/deploy/helm/kopiur/crds/clusterrepositories.yaml +++ b/deploy/helm/kopiur/crds/clusterrepositories.yaml @@ -2999,6 +2999,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/deploy/helm/kopiur/crds/repositories.yaml b/deploy/helm/kopiur/crds/repositories.yaml index 3ea9ba08..a64a0301 100644 --- a/deploy/helm/kopiur/crds/repositories.yaml +++ b/deploy/helm/kopiur/crds/repositories.yaml @@ -2946,6 +2946,85 @@ spec: - copy - copyOverwrite type: string + throttle: + description: |- + Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens + two repositories under two kopia connections and `snapshot migrate` has + no speed flags of its own, so each side is applied as `kopia repository + throttle set` on that side's connection: + + * `source` caps the REPLICA — the repository named by + `spec.seed.from.repository`, opened read-only — overriding **its** + `moverDefaults.throttle`; + * `destination` caps THIS repository, the one being seeded, overriding + **its own** `moverDefaults.throttle`. + + Each side overrides field by field: a knob set here wins, a knob left + unset keeps that repository's default. Absent: both sides use their + repository's defaults. Applies only while the seed is armed — an + ordinary connect to the now-initialized repository is capped by + `moverDefaults.throttle` alone. + nullable: true + properties: + destination: + description: |- + Caps for the DESTINATION (write) side, overriding the destination + repository's `moverDefaults.throttle` field by field. Applied with + `repository throttle set` on the destination connection the migrate + writes through. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + source: + description: |- + Caps for the SOURCE (read) side, overriding the source repository's + `moverDefaults.throttle` field by field. Applied with `repository + throttle set` on the migrate's read-only source connection — accepted + there, so a read-only source is throttled like any other. + nullable: true + properties: + downloadBytesPerSecond: + description: Cap download throughput in bytes/sec (`--download-bytes-per-second`). + format: int64 + nullable: true + type: integer + readOpsPerSecond: + description: Cap read/list ops/sec (`--read-requests-per-second`). + format: int64 + nullable: true + type: integer + uploadBytesPerSecond: + description: Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). + format: int64 + nullable: true + type: integer + writeOpsPerSecond: + description: Cap write ops/sec (`--write-requests-per-second`). + format: int64 + nullable: true + type: integer + type: object + type: object type: object sync: description: |- diff --git a/docs/field-reference.md b/docs/field-reference.md index 3a22025d..50814437 100644 --- a/docs/field-reference.md +++ b/docs/field-reference.md @@ -858,6 +858,32 @@ Externally tagged — set **exactly one** of: `nfs` · `pvc`. | `latestOnly` | boolean | — | Copy only each source identity's most recent snapshot instead of its full history (`kopia snapshot migrate --latest`). Default `false` — a seed exists to recover history, so the full copy is the sane default; set this when you only need the latest restore point back quickly. | | `parallel` | integer | —
min 0 | `--parallel`: snapshots migrated concurrently (kopia default `1` — sequential). Must be >= 1 when set. | | `policies` | enum: none \| copy \| copyOverwrite | `none` | Whether the source's kopia **policies** are copied along with the snapshots. Defaults to `PolicyCopyMode::None` (an explicit `--no-policies`), not kopia's own copy-by-default: retention in a kopiur-managed repository is driven by `Snapshot` CRs, and importing the source's kopia-side policies could delete manifests behind the operator's back. | +| `throttle` | [object](#repository-spec-seed-migrate-throttle) | — | Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens two repositories under two kopia connections and `snapshot migrate` has no speed flags of its own, so each side is applied as `kopia repository throttle set` on that side's connection:
* `source` caps the REPLICA — the repository named by `spec.seed.from.repository`, opened read-only — overriding **its** `moverDefaults.throttle`; * `destination` caps THIS repository, the one being seeded, overriding **its own** `moverDefaults.throttle`.
Each side overrides field by field: a knob set here wins, a knob left unset keeps that repository's default. Absent: both sides use their repository's defaults. Applies only while the seed is armed — an ordinary connect to the now-initialized repository is capped by `moverDefaults.throttle` alone. | + +###### `spec.seed.migrate.throttle` { #repository-spec-seed-migrate-throttle } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `destination` | [object](#repository-spec-seed-migrate-throttle-destination) | — | Caps for the DESTINATION (write) side, overriding the destination repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the destination connection the migrate writes through. | +| `source` | [object](#repository-spec-seed-migrate-throttle-source) | — | Caps for the SOURCE (read) side, overriding the source repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the migrate's read-only source connection — accepted there, so a read-only source is throttled like any other. | + +###### `spec.seed.migrate.throttle.destination` { #repository-spec-seed-migrate-throttle-destination } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | + +###### `spec.seed.migrate.throttle.source` { #repository-spec-seed-migrate-throttle-source } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | ##### `spec.seed.sync` { #repository-spec-seed-sync } @@ -1890,6 +1916,32 @@ Externally tagged — set **exactly one** of: `nfs` · `pvc`. | `latestOnly` | boolean | — | Copy only each source identity's most recent snapshot instead of its full history (`kopia snapshot migrate --latest`). Default `false` — a seed exists to recover history, so the full copy is the sane default; set this when you only need the latest restore point back quickly. | | `parallel` | integer | —
min 0 | `--parallel`: snapshots migrated concurrently (kopia default `1` — sequential). Must be >= 1 when set. | | `policies` | enum: none \| copy \| copyOverwrite | `none` | Whether the source's kopia **policies** are copied along with the snapshots. Defaults to `PolicyCopyMode::None` (an explicit `--no-policies`), not kopia's own copy-by-default: retention in a kopiur-managed repository is driven by `Snapshot` CRs, and importing the source's kopia-side policies could delete manifests behind the operator's back. | +| `throttle` | [object](#clusterrepository-spec-seed-migrate-throttle) | — | Bandwidth/ops caps for THIS seed's copy, per side. A migrate seed opens two repositories under two kopia connections and `snapshot migrate` has no speed flags of its own, so each side is applied as `kopia repository throttle set` on that side's connection:
* `source` caps the REPLICA — the repository named by `spec.seed.from.repository`, opened read-only — overriding **its** `moverDefaults.throttle`; * `destination` caps THIS repository, the one being seeded, overriding **its own** `moverDefaults.throttle`.
Each side overrides field by field: a knob set here wins, a knob left unset keeps that repository's default. Absent: both sides use their repository's defaults. Applies only while the seed is armed — an ordinary connect to the now-initialized repository is capped by `moverDefaults.throttle` alone. | + +###### `spec.seed.migrate.throttle` { #clusterrepository-spec-seed-migrate-throttle } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `destination` | [object](#clusterrepository-spec-seed-migrate-throttle-destination) | — | Caps for the DESTINATION (write) side, overriding the destination repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the destination connection the migrate writes through. | +| `source` | [object](#clusterrepository-spec-seed-migrate-throttle-source) | — | Caps for the SOURCE (read) side, overriding the source repository's `moverDefaults.throttle` field by field. Applied with `repository throttle set` on the migrate's read-only source connection — accepted there, so a read-only source is throttled like any other. | + +###### `spec.seed.migrate.throttle.destination` { #clusterrepository-spec-seed-migrate-throttle-destination } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | + +###### `spec.seed.migrate.throttle.source` { #clusterrepository-spec-seed-migrate-throttle-source } + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `downloadBytesPerSecond` | integer | — | Cap download throughput in bytes/sec (`--download-bytes-per-second`). | +| `readOpsPerSecond` | integer | — | Cap read/list ops/sec (`--read-requests-per-second`). | +| `uploadBytesPerSecond` | integer | — | Cap upload throughput in bytes/sec (`--upload-bytes-per-second`). | +| `writeOpsPerSecond` | integer | — | Cap write ops/sec (`--write-requests-per-second`). | ##### `spec.seed.sync` { #clusterrepository-spec-seed-sync } diff --git a/docs/reference/crds/repository.md b/docs/reference/crds/repository.md index a76144ab..258855cc 100644 --- a/docs/reference/crds/repository.md +++ b/docs/reference/crds/repository.md @@ -50,6 +50,23 @@ holding zero snapshots. `failurePolicy.activeDeadlineSeconds` defaults to **8640 (24 h) while seeding, versus the 120 s a routine connect gets. See [Repositories → `seed`](../../repositories.md#seed--initialize-a-new-repository-from-a-replica). +Capping the copy differs by mode. Blob mode uses `sync.maxDownloadSpeedBytesPerSecond` +/ `maxUploadSpeedBytesPerSecond` — `kopia repository sync-to` takes real speed +flags. Migrate mode's `kopia snapshot migrate` takes **none**, so +`migrate.throttle` is applied as `kopia repository throttle set` on each +connection before the copy runs, **per side**: `throttle.source` caps the +replica's read connection and `throttle.destination` this repository's write +connection, each a [`throttle`](shared-types.md#moverdefaults) block +(`uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, +`writeOpsPerSecond`; every set knob must be `>= 1`, webhook-enforced). Each side +**overrides that side's repository's `moverDefaults.throttle` field by field** — +a knob set here wins, a knob left unset keeps the repository's value — and only +while the seed is armed; later connects are capped by `moverDefaults.throttle` +alone. Two repositories, two connections, two independent blocks: the replica cap +says nothing about this repository's and vice versa. Applying a cap to the +read-only replica connection is supported (kopia accepts `throttle set` there). +See [Throttling a seed](../../repositories.md#throttling-a-seed). + ### `moverDefaults` Base mover configuration — security context, pod security context, resources, diff --git a/docs/reference/crds/shared-types.md b/docs/reference/crds/shared-types.md index b12c665c..4ba502a0 100644 --- a/docs/reference/crds/shared-types.md +++ b/docs/reference/crds/shared-types.md @@ -110,7 +110,11 @@ cache. See [movers](../../movers.md). [`SnapshotReplication`](snapshot-replication.md) caps its source connection from the source repo's defaults and its destination connection from the destination repo's, each overridable per CR via - [`spec.migrate.throttle`](snapshot-replication.md#migrate). + [`spec.migrate.throttle`](snapshot-replication.md#migrate). A migrate-mode + [`seed`](repository.md#seed) is the same shape: the replica's connection is + capped from the *replica's* defaults and this repository's from its own, each + overridable while the seed is armed via `spec.seed.migrate.throttle.source` / + `.destination`. Three things it does **not** cap today: the interactive [`serve`/browse](../../server.md) session (a read-only UI session, not a batch diff --git a/docs/repositories.md b/docs/repositories.md index 2f4d41f6..1a6a104e 100644 --- a/docs/repositories.md +++ b/docs/repositories.md @@ -231,6 +231,93 @@ Both modes preserve each snapshot's `username@hostname:path` identity and its times, so seeded history stays restorable by `Restore.source.identity` and by `fromPolicy`. +### Throttling a seed + +A seed is the single heaviest transfer Kopiur ever performs — a whole repository, +pulled across a link that other people are also using, usually on the worst day +you have had all year. Both modes can be capped, but they take different knobs +because they are different kopia operations. + +**Blob mode** copies at the storage layer, so it uses `sync-to`'s own speed +flags: `seed.sync.maxDownloadSpeedBytesPerSecond` and +`maxUploadSpeedBytesPerSecond`. + +**Migrate mode** has no speed flags at all — `kopia snapshot migrate` offers +none. The only lever is `kopia repository throttle set` on each *connection*, so +the cap is expressed **per side**, because a migrate seed is two repositories: + +```yaml +seed: + from: + repository: { kind: ClusterRepository, name: offsite } + migrate: + throttle: + source: # reading OUT of the replica (the off-site link) + downloadBytesPerSecond: 10485760 # 10 MiB/s + readOpsPerSecond: 100 + destination: # writing INTO this repository (on-LAN, be generous) + uploadBytesPerSecond: 209715200 # 200 MiB/s +``` + +Each side takes the same four knobs as a repository's +[`moverDefaults.throttle`](reference/crds/shared-types.md#moverdefaults): +`uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, +`writeOpsPerSecond`. Every knob you set must be at least `1` (a `0` is rejected at +admission — kopia's "no limit" is the *absent* knob, not a zero). + +**Two layers, merged field by field.** Each repository can already declare +`moverDefaults.throttle`, which every mover that touches it honors. +`seed.migrate.throttle` sits on top of that, per side: + +- `throttle.source` overrides the **replica's** defaults — the repository named by + `seed.from.repository`, not this one; +- `throttle.destination` overrides **this** repository's defaults; +- the merge is per knob: a knob you set here wins, a knob you leave unset keeps + the repository's value. Pinning one knob never drops the repository's others; +- omit `migrate.throttle` entirely and each side simply uses its own repository's + defaults. + +The two sides never mix — kopia's limits are **per connection**, and a migrate +seed opens both repositories under two separate kopia configurations. The +override applies only while the seed is **armed**: every later connect to the +now-initialized repository is capped by `moverDefaults.throttle` alone, so a +number chosen for a one-time copy does not go on constraining routine work. + +If any of the applications fails, the bootstrap **fails** rather than proceeding +uncapped: saturating exactly the link you asked to protect is worse than not +running. The mover logs `applied repository throttle` once per capped connection +— three times for a healthy migrate seed (the replica, this repository's +seed-local connect, and the post-seed reconnect): + +```console +$ kubectl -n billing logs job/rebuilt-nas-bootstrap | grep 'applied repository throttle' +``` + +/// warning | A byte cap bites much harder than its number suggests + +Byte-rate caps apply to **cold** backend traffic only — content already in the +mover's kopia cache is served without touching the limiter, so a warm re-run can +look entirely unthrottled. And on small-object workloads the effective throughput +lands far below the nominal number: a measured 2 MB/s cap took ~14 s to move a +28 KiB cold repository, while caps of 10 MB/s and above often did not bind at all +at that size. Set these as a **ceiling for large transfers**, generously, and +measure against your own data. A seed already runs against a 24 h deadline; a cap +picked to "feel safe" is a good way to make that deadline the thing that fails. + +/// + +/// warning | New CRD fields need `kubectl apply`, not just `helm upgrade` + +`seed.migrate.throttle` is a new CRD field. Helm's `crds/` directory is +**install-only** — a `helm upgrade` never updates CRD schemas — so on a helm-CLI +upgrade an apiserver running the old schema **silently prunes** the field from +your manifest: the object admits cleanly, `kubectl get -o yaml` shows no +`throttle`, and the seed runs uncapped with nothing to see. Apply the CRDs first +(`kubectl apply --server-side -f deploy/crds/`), or use a GitOps flow with a +`CreateReplace` CRD policy. See [CRD lifecycle](install.md#crd-lifecycle). + +/// + ### `create` and `seed` together | You set | What happens | @@ -252,6 +339,7 @@ times, so seeded history stays restorable by `Restore.source.identity` and by | `seed.migrate.parallel` | kopia's `1` | Migrate mode only: snapshots migrated concurrently. | | `seed.migrate.latestOnly` | `false` | Copy only each identity's newest snapshot instead of its full history. | | `seed.migrate.policies` | `none` | Whether the source's **kopia-side** policies come along. Default is an explicit `--no-policies`, unlike kopia's own copy-by-default: retention here is driven by `Snapshot` CRs, and imported kopia policies could delete manifests behind the operator's back. | +| `seed.migrate.throttle.source` / `.destination` | each repository's `moverDefaults.throttle` | Migrate mode only: bandwidth/ops caps for the seed run, **per side** — `source` for the replica, `destination` for this repository. Each overrides that side's repository's defaults field by field; every set knob must be `>= 1`. Armed seeds only. See [Throttling a seed](#throttling-a-seed). | | `seed.allowEmptySource` | `false` | Accept a source holding zero snapshots. Left at `false`, an empty source **fails the bootstrap** and retries. | | `seed.failurePolicy.activeDeadlineSeconds` | `86400` (24 h) | Wall-clock cap for the seeding Job. | | `seed.failurePolicy.backoffLimit` | as `bootstrap` | Pod retries within one seeding Job. | From ac58809196123ea8e6835dae99553dc09270c421 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 13:08:44 -0700 Subject: [PATCH 12/15] docs(mover): correct two seed-throttle comments the wiring outdated (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both described the world before the replica connect was capped, in the two places a reader is most likely to trust them. - `seed_create_and_connect_local` still said the seed SOURCE's cap was "wired in a later change" — this WAS that change, so the comment told a reader of the local connect that the replica runs uncapped. It now names `replicaThrottle` and where it is applied. - `apply_repository_throttle`'s doc used a seed as its worked example of "one line per APPLICATION" and put the count at two connections (the bootstrap's and the local repository's). A migrate seed now opens three, and the e2e asserts three. Restated as the general rule plus the exact seed arithmetic: three for a FIRST seed — the replica, the seed-local client and the post-seed reconnect, its probe connect having failed on the uninitialized backend, which is what armed the seed — and four when resuming, where that probe connect succeeds and is capped too. Same off-by-one in docs/repositories.md's grep hint, which claimed three unconditionally; it now distinguishes the first seed from a resume. Comments and prose only; no behavior change. --- crates/mover/src/main.rs | 13 +++++++++---- docs/repositories.md | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/mover/src/main.rs b/crates/mover/src/main.rs index ffa1920f..5431007b 100644 --- a/crates/mover/src/main.rs +++ b/crates/mover/src/main.rs @@ -436,9 +436,13 @@ async fn connect_and_throttle( /// scenarios assert on it, and its ABSENCE is the version-skew signal for a new /// controller driving an old mover image. Emitted only after kopia accepted /// them, so it can never claim a throttle that is not in force. One line per -/// APPLICATION, not per run: a flow that opens two connections (a seed: the -/// bootstrap's and the local repository's) logs it once per connection, which is -/// what lets an e2e assert that both were capped. +/// APPLICATION, not per run: a flow that opens several connections logs it once +/// per connection, which is what lets an e2e assert that every one of them was +/// capped. A first migrate seed opens three (the replica, the seed's local +/// client, the post-seed reconnect — its probe connect FAILS on the +/// uninitialized backend, which is what arms the seed, so it never reaches its +/// own application); a resuming one opens four, since that probe connect +/// succeeds. /// /// Applied on read-only connections too. `repository throttle set` registers a /// kopia *write* action, but is empirically accepted on a `--readonly` connect @@ -1679,7 +1683,8 @@ async fn seed_create_and_connect_local( // capped, and kopia's limits are per-config. `snapshot migrate` has no speed // flags of its own, so the limits persisted into this config are the ONLY cap // on the heaviest transfer kopiur performs. (The seed SOURCE's own cap is a - // separate knob, wired in a later change.) + // separate knob on a separate connection — `replicaThrottle`, applied in + // `seed_connect_source`; kopia's limits never cross between the two.) if let Err(e) = connect_and_throttle(&local_client, local_connect, spec.cache, &spec.throttle).await { diff --git a/docs/repositories.md b/docs/repositories.md index 1a6a104e..c3ffcc32 100644 --- a/docs/repositories.md +++ b/docs/repositories.md @@ -286,8 +286,10 @@ number chosen for a one-time copy does not go on constraining routine work. If any of the applications fails, the bootstrap **fails** rather than proceeding uncapped: saturating exactly the link you asked to protect is worse than not running. The mover logs `applied repository throttle` once per capped connection -— three times for a healthy migrate seed (the replica, this repository's -seed-local connect, and the post-seed reconnect): +— three times for a healthy first migrate seed (the replica, this repository's +seed-local connect, and the post-seed reconnect), and four when it is *resuming* +an interrupted attempt, whose opening probe connect finds the repository already +there and so gets capped too: ```console $ kubectl -n billing logs job/rebuilt-nas-bootstrap | grep 'applied repository throttle' From 425f5a8877f4d748e84b47d80872bc678a772e8b Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 13:20:08 -0700 Subject: [PATCH 13/15] =?UTF-8?q?docs:=20final-review=20sweep=20=E2=80=94?= =?UTF-8?q?=20truthful=20throttle-failure=20logs=20and=20stale=20prose=20(?= =?UTF-8?q?#374,=20#393,=20#394)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four small, behavior-neutral findings from final review: - crates/mover/src/main.rs: remove a doc-comment paragraph pair duplicated back-to-back on apply_repository_throttle. - crates/mover/src/main.rs: reword the maintenance/verify/replicate-source error! log literals, which also fire on a refused throttle set (ConnectFailure::Throttle) and previously read as pure connectivity failures. No change to error classes, propagation, or status bodies. - docs/dev/api-conventions.md: document the manualRun.completedAt exception to the blanket skip_serializing_if convention (#394). - crates/controller/src/restore/tests.rs: fix a stale doc comment referring to Cow; the type is CarriedRestore since 7c89887 (#393). - docs/reference/crds/shared-types.md: add "seed" to the throttle honored-flows list, consistent with the seed-shape sentence that follows. --- crates/controller/src/restore/tests.rs | 2 +- crates/mover/src/main.rs | 11 +++-------- docs/dev/api-conventions.md | 2 ++ docs/reference/crds/shared-types.md | 4 ++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/crates/controller/src/restore/tests.rs b/crates/controller/src/restore/tests.rs index 2e764466..540db08a 100644 --- a/crates/controller/src/restore/tests.rs +++ b/crates/controller/src/restore/tests.rs @@ -307,7 +307,7 @@ fn the_referent_gate_condition_clears_only_when_it_is_stale() { /// them; this test pins the property that keeps that true. /// /// The CALLER half of the contract is enforced by the compiler rather than here: -/// `RepositoryGate::Proceed` carries the `Cow` to continue with, so +/// `RepositoryGate::Proceed` carries the `CarriedRestore` to continue with, so /// `reconcile_inner` cannot get a `&Restore` for the rest of the pass without /// taking the carried one. This test pins the mechanism that carrying provides. #[test] diff --git a/crates/mover/src/main.rs b/crates/mover/src/main.rs index 5431007b..569cba04 100644 --- a/crates/mover/src/main.rs +++ b/crates/mover/src/main.rs @@ -427,11 +427,6 @@ async fn connect_and_throttle( /// A failure is TERMINAL for the run, never best-effort: continuing would /// saturate exactly the link the user capped, which is worse than not running. /// -/// No limits set → no kopia process is spawned (the common case stays free). -/// -/// A failure is TERMINAL for the run, never best-effort: continuing would -/// saturate exactly the link the user capped, which is worse than not running. -/// /// The `info!` line is the observable proof the limits reached kopia — the e2e /// scenarios assert on it, and its ABSENCE is the version-skew signal for a new /// controller driving an old mover image. Emitted only after kopia accepted @@ -2578,7 +2573,7 @@ async fn run_maintenance_flow( if let Err(e) = connect_and_throttle(client, connect, spec.cache, &spec.throttle).await { let e = e.into_mover_error(KopiaOp::MaintenanceConnect); patch_maintenance_status(&spec.target_ref, &maintenance_failed_body_from_mover(&e)).await; - error!(class = %e.kopia_class(), "maintenance connect failed"); + error!(class = %e.kopia_class(), "could not open a capped connection to the repository undergoing maintenance"); return Err(e); } @@ -2734,7 +2729,7 @@ async fn run_verify_flow( if let Err(e) = connect_and_throttle(client, connect, spec.cache, &spec.throttle).await { let e = e.into_mover_error(KopiaOp::VerifyConnect); patch_verify_status(&spec.target_ref, &verify_failed_body(&e.to_string())).await; - error!(class = %e.kopia_class(), "verify connect failed"); + error!(class = %e.kopia_class(), "could not open a capped connection to the repository being verified"); return Err(e); } @@ -2980,7 +2975,7 @@ async fn run_replicate_flow( if let Err(e) = connect_and_throttle(client, connect, spec.cache, &spec.throttle).await { let e = e.into_mover_error(KopiaOp::ReplicateConnect); patch_replicate_status(&spec.target_ref, &replicate_failed_body(&e.to_string())).await; - error!(class = %e.kopia_class(), "replication source connect failed"); + error!(class = %e.kopia_class(), "could not open a capped connection to the source repository a replication reads from"); return Err(e); } diff --git a/docs/dev/api-conventions.md b/docs/dev/api-conventions.md index 69141cf2..e6cb9300 100644 --- a/docs/dev/api-conventions.md +++ b/docs/dev/api-conventions.md @@ -49,6 +49,8 @@ Use `k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, Condition}` Every credential/policy/identity/schedule surface is a **sub-object**, not a leaf field. Optionals: `#[serde(default, skip_serializing_if = "Option::is_none")] pub x: Option`. Bools that default false: `#[serde(default, skip_serializing_if = "std::ops::Not::not")]`. Vecs: `#[serde(default, skip_serializing_if = "Vec::is_empty")]`. +Exception: `manualRun.completedAt` (`ReplicationManualRunStatus` and `maintenance::ManualRunStatus`) deliberately omits `skip_serializing_if` — a non-terminal-phase status patch must emit an explicit JSON `null` so the RFC-7386 merge-patch clears a stale timestamp left standing from a prior run; see the field comments and #394. + ## 4a. Schema defaults (`#[schemars(default = …)]`) A schemars `default` becomes a CRD `default:` that the **API server materializes diff --git a/docs/reference/crds/shared-types.md b/docs/reference/crds/shared-types.md index 4ba502a0..e914a4d7 100644 --- a/docs/reference/crds/shared-types.md +++ b/docs/reference/crds/shared-types.md @@ -104,8 +104,8 @@ cache. See [movers](../../movers.md). saturate the link or hammer the object store: `uploadBytesPerSecond`, `downloadBytesPerSecond`, `readOpsPerSecond`, `writeOpsPerSecond` (each unset leaves kopia's current limit). Honored by the transfer movers — bootstrap, - backup, restore, maintenance, verification, repository replication and snapshot - replication. kopia's limits are **per connection**, so a run that opens two + backup, restore, maintenance, verification, repository replication, snapshot + replication and seed. kopia's limits are **per connection**, so a run that opens two repositories caps each from *that* repository's own `moverDefaults`: a [`SnapshotReplication`](snapshot-replication.md) caps its source connection from the source repo's defaults and its destination connection from the From 47b564c821c46bb9f8d8655bc70a979c5145834f Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 21:09:12 -0700 Subject: [PATCH 14/15] fix(repository): stop re-stamping seed.seededAt on every finalize pass (#396 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A repository that finished a seed span its own reconciler at ~4 passes a second until its bootstrap Job's TTL expired — a self-sustaining hot loop that starved every other Repository in the cluster. `finalize_bootstrap` re-enters on EVERY reconcile for as long as the finished bootstrap Job exists: the Job is deleted only on the health-probe arm, and a seeding bootstrap is never a probe run. Each re-entry re-reads the same result ConfigMap, so `result.seed` is still the same `SeedOutcome`, forever — and `seed_success_fold` stamped `status.seed.seededAt = Utc::now()` unconditionally. A fresh timestamp on every pass meant `status_patch_is_noop` never held, the status PATCH bumped `resourceVersion`, the controller's own primary watch re-delivered the object as `object.reason="object updated"`, and round it went. The `RepositorySeeded` Event is gated on that same write, so the loop also minted one Event aggregation per pass. A second, independent defect kept the loop alive even with a stable clock: the no-op guard compares the whole `seed` value, while the fold wrote a PARTIAL sub-object that omitted the `startedAt` marker `seed_marker_patch` had stamped. A partial object can never equal the stored one, so the write repeated regardless of the timestamp. - `seed_success_fold` now takes the live `status.seed`. `seededAt` is stamped ONCE, at the first fold, and reused verbatim afterwards — a seed happens exactly once, so its completion time is a fact, not a heartbeat (the rule `seed_marker_patch` already applies to `startedAt`). The marker itself is carried forward into the patch so the fold's `seed` object equals the stored one byte for byte. Both repository kinds pass their own `status.seed`, so neither can drift. - Regression test: re-folding a finished MIGRATE seed minutes later must be a `status_patch_is_noop`, shaped like the e2e's `e2e-seed-mig-dst`. It covers both arms — the advancing wall clock and the missing marker — and pins that the FIRST fold still stamps `now`. Reverting either half of the fix fails it. This predates the branch: the unguarded `seededAt = now` arrived with 74665dc (#380 / #396) and reproduces identically at the merge-base of main. It only went red now because the consolidated e2e run puts the migrate-seed scenario (which deliberately leaves its seeded Repository standing) in the same cluster as `ready_repository_steady_state_is_quiet`, which counts fleet-wide Repository reconciles. The shared `status_patch_is_noop` shallow-compare stays as it is; making it a recursive-subset compare is a global guard-semantics change and is deferred to its own review. --- crates/controller/src/cluster_repository.rs | 11 +- crates/controller/src/repo_seed.rs | 128 +++++++++++++++++++- crates/controller/src/repository.rs | 11 +- 3 files changed, 137 insertions(+), 13 deletions(-) diff --git a/crates/controller/src/cluster_repository.rs b/crates/controller/src/cluster_repository.rs index e1648196..928d4842 100644 --- a/crates/controller/src/cluster_repository.rs +++ b/crates/controller/src/cluster_repository.rs @@ -1826,10 +1826,13 @@ async fn finalize_cluster_bootstrap( // #380: fold the seed outcome into the SAME conditions array — the // mover-skew guard has already refused a seed-armed success carrying no // outcome, so a present `result.seed` is proof the image understood it. - let seed_fold = result - .seed - .as_ref() - .map(|outcome| crate::repo_seed::seed_success_fold(outcome, &now)); + let seed_fold = result.seed.as_ref().map(|outcome| { + crate::repo_seed::seed_success_fold( + outcome, + repo.status.as_ref().and_then(|s| s.seed.as_ref()), + &now, + ) + }); let conditions = match seed_fold.as_ref() { Some(fold) => io::upsert_condition( &conditions, diff --git a/crates/controller/src/repo_seed.rs b/crates/controller/src/repo_seed.rs index 8639515b..1135aff6 100644 --- a/crates/controller/src/repo_seed.rs +++ b/crates/controller/src/repo_seed.rs @@ -383,14 +383,43 @@ pub(crate) struct SeedSuccess { /// storage, not manifests, so there is no per-snapshot copy count; its /// `snapshotCount` is the SOURCE listing the mover took before the copy, which /// for a byte-for-byte mirror is also what this repository ends up holding). -pub(crate) fn seed_success_fold(outcome: &SeedOutcome, now: &str) -> SeedSuccess { +/// +/// # Why `existing` — the fold must be BYTE-STABLE across re-folds +/// +/// `finalize_bootstrap` re-enters on EVERY reconcile for as long as the +/// finished bootstrap Job lingers (it is deleted only on the probe arm), and it +/// re-reads the same result ConfigMap each time — so a seeding Job's outcome is +/// re-folded, unchanged, hundreds of times. Two properties make that pass a +/// no-op under [`crate::io::status_patch_is_noop`], and both need `existing`: +/// +/// * `seededAt` is stamped ONCE, at the first fold, and reused verbatim +/// afterwards. Re-stamping `now` made every pass a fresh status write, which +/// bumped `resourceVersion`, re-triggered this reconciler through its own +/// primary watch and spun the repository at ~4 reconciles/second until the +/// Job's TTL. (A seed happens exactly once, so "when it completed" is a fact, +/// not a heartbeat — the same rule `seed_marker_patch` already applies to +/// `startedAt`.) +/// * the seed-attempt marker (`startedAt`) is carried FORWARD into the patch. +/// A merge patch would preserve it either way, but the no-op guard compares +/// the whole `seed` value: a partial sub-object can never equal the stored +/// one, so omitting the marker re-writes forever on its own. +pub(crate) fn seed_success_fold( + outcome: &SeedOutcome, + existing: Option<&SeedStatus>, + now: &str, +) -> SeedSuccess { let mode = seed_mode_of(outcome); let mut status = serde_json::json!({ "mode": mode, "source": outcome.source, }); + if let Some(started_at) = existing.and_then(|s| s.started_at.as_deref()) { + status["startedAt"] = serde_json::Value::String(started_at.to_string()); + } if outcome.performed { - status["seededAt"] = serde_json::Value::String(now.to_string()); + // Set once; every later re-fold of the same lingering result reuses it. + let seeded_at = existing.and_then(|s| s.seeded_at.as_deref()).unwrap_or(now); + status["seededAt"] = serde_json::Value::String(seeded_at.to_string()); if let Some(n) = outcome.snapshot_count { status["snapshotCount"] = serde_json::json!(n); } @@ -1451,7 +1480,7 @@ mod tests { 412, Some(412), ); - let fold = seed_success_fold(&performed, now); + let fold = seed_success_fold(&performed, None, now); assert_eq!(fold.reason, kopiur_api::consts::SEEDED_REASON); assert_eq!(fold.outcome, crate::metrics::SeedOutcomeLabel::Seeded); assert_eq!(fold.status["seed"]["seededAt"], serde_json::json!(now)); @@ -1469,7 +1498,7 @@ mod tests { // manifests — and the post-seed catalog listing reports the count on // storageStats instead. let blob = SeedOutcome::performed(SeedModeSpec::Blob, "S3".into(), 7, None); - let fold = seed_success_fold(&blob, now); + let fold = seed_success_fold(&blob, None, now); assert_eq!(fold.status["seed"]["snapshotCount"], serde_json::json!(7)); assert!(fold.status["seed"].get("snapshotsCopied").is_none()); @@ -1477,7 +1506,7 @@ mod tests { // reporting 0 would be a lie), its own reason and metric label, and NO // Event — nothing happened. let noop = SeedOutcome::already_initialized(SeedModeSpec::Blob, "S3".into()); - let fold = seed_success_fold(&noop, now); + let fold = seed_success_fold(&noop, None, now); assert_eq!(fold.reason, kopiur_api::consts::ALREADY_INITIALIZED_REASON); assert_eq!( fold.outcome, @@ -1488,6 +1517,94 @@ mod tests { assert!(fold.event.is_none()); } + /// HOT-LOOP REGRESSION (#396 follow-up). `finalize_bootstrap` re-enters on + /// every reconcile while the finished bootstrap Job lingers and re-folds the + /// SAME `SeedOutcome` out of the same result ConfigMap. That steady-state + /// pass must be a no-op under `io::status_patch_is_noop`, or the write bumps + /// `resourceVersion`, the primary watch re-delivers the object and the + /// repository spins until the Job's TTL. + /// + /// Shaped like the e2e's `e2e-seed-mig-dst`: a completed MIGRATE seed on a + /// `Ready` repository. Both failure modes are covered — the wall clock + /// advancing between passes, and the marker (`startedAt`) the stored + /// sub-object carries but the fold used to omit. + #[test] + fn re_folding_a_finished_seed_is_a_byte_stable_no_op() { + use kopiur_api::repository::RepositoryStatus; + + let outcome = SeedOutcome::performed( + SeedModeSpec::Migrate, + "Repository/e2e-seed-mig-src".into(), + 1, + Some(1), + ); + // The marker stamped by `seed_marker_patch` before the seeding Job ran. + let marker = SeedStatus { + started_at: Some("2026-08-26T03:33:20+00:00".into()), + ..Default::default() + }; + + // FIRST fold: nothing recorded yet, so `now` is what gets stamped. + let first = seed_success_fold(&outcome, Some(&marker), "2026-08-26T03:33:26+00:00"); + assert_eq!( + first.status["seed"]["seededAt"], + serde_json::json!("2026-08-26T03:33:26+00:00"), + "the first fold stamps the completion time" + ); + assert_eq!( + first.status["seed"]["startedAt"], + serde_json::json!("2026-08-26T03:33:20+00:00"), + "the seed-attempt marker rides the patch, or the no-op compare below \ + can never hold" + ); + + // What that first pass persisted, read back the way the reconciler reads + // it (`serde_json::to_value(&repo.status)`). + let stored = RepositoryStatus { + phase: Some(kopiur_api::RepositoryPhase::Ready), + observed_generation: Some(1), + unique_id: Some("c36584a9".into()), + backend: Some("Filesystem".into()), + seed: Some(SeedStatus { + started_at: marker.started_at.clone(), + seeded_at: Some("2026-08-26T03:33:26+00:00".into()), + mode: Some(SeedMode::Migrate), + source: Some("Repository/e2e-seed-mig-src".into()), + snapshot_count: Some(1), + snapshots_copied: Some(1), + }), + ..Default::default() + }; + let current = serde_json::to_value(Some(&stored)).expect("status serializes"); + let existing = stored.seed.as_ref(); + + // EVERY later pass, minutes later on the wall clock, is a no-op. + let mut later = serde_json::json!({ + "phase": "Ready", + "backend": "Filesystem", + "uniqueId": "c36584a9", + "observedGeneration": 1, + }); + merge_seed_status( + &mut later, + &seed_success_fold(&outcome, existing, "2026-08-26T03:51:54+00:00").status, + ); + assert!( + io::status_patch_is_noop(Some(¤t), &later), + "the steady-state re-fold must not re-write status (hot loop): {later}" + ); + + // The standing no-op arm carries the marker forward too, so an + // `AlreadyInitialized` repository is equally quiet. + let adopted = SeedOutcome::already_initialized(SeedModeSpec::Blob, "S3".into()); + let fold = seed_success_fold(&adopted, Some(&marker), "2026-08-26T03:51:54+00:00"); + assert_eq!( + fold.status["seed"]["startedAt"], + serde_json::json!("2026-08-26T03:33:20+00:00") + ); + assert!(fold.status["seed"].get("seededAt").is_none()); + } + #[test] fn merging_the_seed_fold_never_drops_a_key_already_in_the_patch() { // `status_patch` is assembled key-by-key before ONE patch, so a plain @@ -1498,6 +1615,7 @@ mod tests { }); let fold = seed_success_fold( &SeedOutcome::performed(SeedModeSpec::Blob, "S3".into(), 3, None), + None, "2026-08-17T03:00:00Z", ); merge_seed_status(&mut patch, &fold.status); diff --git a/crates/controller/src/repository.rs b/crates/controller/src/repository.rs index 01e882fc..a09691a6 100644 --- a/crates/controller/src/repository.rs +++ b/crates/controller/src/repository.rs @@ -1946,10 +1946,13 @@ async fn finalize_bootstrap( // patch. `bootstrap_outcome` has already refused a seed-armed success that // carries no outcome (the mover-skew guard), so `result.seed` being present // here is proof the running mover understood the request. - let seed_fold = result - .seed - .as_ref() - .map(|outcome| repo_seed::seed_success_fold(outcome, &now)); + let seed_fold = result.seed.as_ref().map(|outcome| { + repo_seed::seed_success_fold( + outcome, + repo.status.as_ref().and_then(|s| s.seed.as_ref()), + &now, + ) + }); let conditions = match seed_fold.as_ref() { Some(fold) => io::upsert_condition( &conditions, From 1c4e5aad937a4f239bc58285ea6846b31836d2ab Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Tue, 25 Aug 2026 21:11:38 -0700 Subject: [PATCH 15/15] fix(e2e): accept the apiserver storing an explicit completedAt null (#394) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #394 e2e demanded the stale `completedAt` key be ABSENT after the non-terminal patch nulled it. Against a real apiserver it is not: on k8s 1.33 a nullable CRD field answers a merge-patch null by STORING the null rather than deleting the key. The scenario failed on a property it never meant to assert. #394's semantics are correct either way — an absent key and an explicit null both decode to `None`, both re-serialize to `null`, and neither leaves the previous run's timestamp standing, which is the whole point. Only the assertion and the prose were over-specified. - E2e: assert there is no completedAt VALUE (absent OR explicit null is fine; a non-null string is the bug), with the message rewritten to name that property instead of the deletion mechanic. - Comments at both serde sites (`common::ReplicationManualRunStatus`, `maintenance::ManualRunStatus`) and in the api/controller tests now say the apiserver either deletes the key or stores the null, and why both converge. The `Patch::Merge`-not-`Patch::Apply` dependency is unchanged and still called out. - `manual_run_patch_converges_after_clearing_a_stale_completed_at` now replays BOTH apiserver outcomes against the noop guard, so the stored-null shape is pinned rather than assumed. --- crates/api/src/common/mod.rs | 22 ++++++++----- crates/api/src/common/tests.rs | 6 ++-- crates/api/src/maintenance.rs | 20 +++++++----- crates/controller/src/io/tests.rs | 41 +++++++++++++++++------- crates/controller/src/replication_run.rs | 5 +-- crates/e2e/tests/replication.rs | 22 ++++++++----- 6 files changed, 77 insertions(+), 39 deletions(-) diff --git a/crates/api/src/common/mod.rs b/crates/api/src/common/mod.rs index 64fffdcf..81fbc96d 100644 --- a/crates/api/src/common/mod.rs +++ b/crates/api/src/common/mod.rs @@ -1275,17 +1275,23 @@ pub struct ReplicationManualRunStatus { /// RFC3339 instant the run reached a terminal phase. // Deliberately serialized EVEN WHEN `None` — no `skip_serializing_if`, unlike // every other optional in this struct. A non-terminal phase must emit - // `"completedAt": null` so the RFC-7386 merge-patch the controller sends - // DELETES the previous run's stamp. Omitting the key instead leaves the old - // instant standing, which is both a lie in `kubectl get -o yaml` and — - // because the replication controllers build the noop guard's `current` by - // re-serializing this very struct — a guard that never converges: `desired` - // would omit the key while `current` carries the stale value, so every - // queued pass re-fires a PATCH the apiserver no-ops (#394). + // `"completedAt": null` so the merge-patch the controller sends CLEARS the + // previous run's stamp. On receiving that null the apiserver either deletes + // the key (plain RFC-7386) or stores the null verbatim — a nullable CRD + // field on k8s 1.33 was observed doing the latter — and BOTH converge here: + // an absent key decodes to `None`, an explicit null decodes to `None`, and + // re-serializing either yields `null` again. + // + // Omitting the key instead leaves the old instant standing, which is both a + // lie in `kubectl get -o yaml` and — because the replication controllers + // build the noop guard's `current` by re-serializing this very struct — a + // guard that never converges: `desired` would omit the key while `current` + // carries the stale value, so every queued pass re-fires a PATCH the + // apiserver no-ops (#394). // // This depends on `patch_status` sending `kube::api::Patch::Merge` // (`crates/controller/src/io/apply.rs`). Under `Patch::Apply` an explicit - // null does NOT delete, and this contract silently breaks. + // null does NOT clear the field, and this contract silently breaks. // // Kept a plain comment rather than rustdoc on purpose: doc comments become // the CRD `description` (`kubectl explain`, docs/field-reference.md), and diff --git a/crates/api/src/common/tests.rs b/crates/api/src/common/tests.rs index 80b24945..c3f9fb47 100644 --- a/crates/api/src/common/tests.rs +++ b/crates/api/src/common/tests.rs @@ -1525,8 +1525,10 @@ fn replication_manual_run_status_roundtrips_camel_case() { assert_eq!(back, st); // The contract (#394): `requestedAt`/`phase` serialize away when absent, but // `completedAt` ALWAYS emits — as an explicit null when there is no - // completion instant — so the merge-patch DELETES a previous run's stamp - // instead of leaving it standing over a fresh non-terminal phase. + // completion instant — so the merge-patch CLEARS a previous run's stamp + // instead of leaving it standing over a fresh non-terminal phase. The + // apiserver either deletes the key or stores the null; both decode back to + // `None`, so neither leaves a stale timestamp. let empty = serde_json::to_value(ReplicationManualRunStatus::default()).unwrap(); assert_eq!(empty, serde_json::json!({ "completedAt": null })); // …and that explicit null reads back as "no completion instant", not as a diff --git a/crates/api/src/maintenance.rs b/crates/api/src/maintenance.rs index 30ec7ca8..e14011d1 100644 --- a/crates/api/src/maintenance.rs +++ b/crates/api/src/maintenance.rs @@ -611,15 +611,18 @@ pub struct ManualRunStatus { /// RFC3339 instant the run reached a terminal phase. // Deliberately serialized EVEN WHEN `None` (no `skip_serializing_if`), for // the same reason as `common::ReplicationManualRunStatus::completed_at`: a - // non-terminal phase emits `"completedAt": null` so the RFC-7386 merge-patch - // DELETES the previous run's stamp instead of leaving it standing over a - // fresh `Running` (#394). Maintenance patches `manualRun` unconditionally - // (no noop guard), so here the stake is a truthful status rather than a - // non-converging write loop. + // non-terminal phase emits `"completedAt": null` so the merge-patch CLEARS + // the previous run's stamp instead of leaving it standing over a fresh + // `Running` (#394). The apiserver answers that null by deleting the key + // (plain RFC-7386) or by storing the null verbatim — a nullable CRD field on + // k8s 1.33 was observed doing the latter — and both decode back to `None`, + // so no stale timestamp survives either way. Maintenance patches `manualRun` + // unconditionally (no noop guard), so here the stake is a truthful status + // rather than a non-converging write loop. // // This depends on `patch_status` sending `kube::api::Patch::Merge` // (`crates/controller/src/io/apply.rs`). Under `Patch::Apply` an explicit - // null does NOT delete, and this contract silently breaks. + // null does NOT clear the field, and this contract silently breaks. // // Kept a plain comment rather than rustdoc on purpose: doc comments become // the CRD `description` (`kubectl explain`, docs/field-reference.md). @@ -710,8 +713,9 @@ mod tests { assert_eq!(json["completedAt"], "2026-06-11T12:01:42Z"); // #394: a non-terminal run emits an EXPLICIT null completedAt, so the - // RFC-7386 merge-patch deletes the previous run's stamp rather than - // leaving it standing over a fresh `Running`. + // merge-patch clears the previous run's stamp rather than leaving it + // standing over a fresh `Running`. Whether the apiserver deletes the key + // or stores the null, it decodes back to `None`. let running = serde_json::to_value(ManualRunStatus { requested_at: Some("2026-06-11T13:00:00Z".into()), mode: Some(ManualRunMode::Quick), diff --git a/crates/controller/src/io/tests.rs b/crates/controller/src/io/tests.rs index ca98c93a..5de70564 100644 --- a/crates/controller/src/io/tests.rs +++ b/crates/controller/src/io/tests.rs @@ -853,7 +853,7 @@ fn non_terminal_manual_run_patch_carries_an_explicit_null_completed_at() { desired["manualRun"].get("completedAt"), Some(&serde_json::Value::Null), "a non-terminal manualRun must NAME completedAt so the merge-patch \ - deletes the previous run's stamp; got {desired}" + clears the previous run's stamp; got {desired}" ); } @@ -875,23 +875,42 @@ fn manual_run_patch_converges_after_clearing_a_stale_completed_at() { "answering a NEW request must write" ); - // The apiserver applies the merge patch (this is `Patch::Merge` — an - // explicit null DELETES the key). + // The apiserver applies the merge patch (this is `Patch::Merge`). Given the + // explicit null it either DELETES the key — plain RFC-7386, what + // `json_patch::merge` models — or STORES the null verbatim; a nullable CRD + // field on k8s 1.33 was observed doing the latter. Both outcomes are + // replayed below, because the guard has to converge under either. json_patch::merge(&mut stored, &desired); assert!( stored["manualRun"].get("completedAt").is_none(), - "the merge-patch must have deleted the stale stamp; got {stored}" + "the merge-patch must have cleared the stale stamp; got {stored}" ); // Next reconcile: `current` is the typed status of the refreshed object, // re-serialized — the exact round trip `patch_manual_run` performs. - let typed: kopiur_api::common::ReplicationManualRunStatus = - serde_json::from_value(stored["manualRun"].clone()).expect("stored manualRun decodes"); - let current = serde_json::json!({ "manualRun": typed }); - assert!( - status_patch_is_noop(Some(¤t), &desired), - "the guard must CONVERGE once the stamp is cleared; current {current}, desired {desired}" - ); + let converges = |stored: &serde_json::Value| { + let typed: kopiur_api::common::ReplicationManualRunStatus = + serde_json::from_value(stored["manualRun"].clone()).expect("stored manualRun decodes"); + let current = serde_json::json!({ "manualRun": typed }); + assert!( + status_patch_is_noop(Some(¤t), &desired), + "the guard must CONVERGE once the stamp is cleared; current {current}, \ + desired {desired}" + ); + }; + converges(&stored); + + // The other apiserver outcome: the null is STORED rather than deleted. It + // decodes to `None` just the same (`#[serde(default)]` on an `Option`), so + // the rebuilt `current` is identical and the guard converges here too. + let stored_null = serde_json::json!({ + "manualRun": { + "requestedAt": "2026-06-11T13:00:00Z", + "phase": "Pending", + "completedAt": null, + }, + }); + converges(&stored_null); } #[test] diff --git a/crates/controller/src/replication_run.rs b/crates/controller/src/replication_run.rs index 8f80dc92..798cd484 100644 --- a/crates/controller/src/replication_run.rs +++ b/crates/controller/src/replication_run.rs @@ -623,8 +623,9 @@ mod tests { let st = manual_run_status(&req, phase.clone(), now); assert!(st.completed_at.is_none(), "{phase:?} has not completed"); // #394: and it must SERIALIZE as an explicit null, not vanish — the - // merge-patch only deletes a previous run's stamp when the key is - // present with a null value. + // merge-patch only clears a previous run's stamp when the key is + // present with a null value (the apiserver then deletes the key or + // stores the null; either way no stale timestamp survives). let json = serde_json::to_value(&st).unwrap(); assert_eq!( json.get("completedAt"), diff --git a/crates/e2e/tests/replication.rs b/crates/e2e/tests/replication.rs index 5f22a41c..19e2f0e9 100644 --- a/crates/e2e/tests/replication.rs +++ b/crates/e2e/tests/replication.rs @@ -438,8 +438,9 @@ async fn repository_replication_s3_to_s3_uses_destination_scoped_credentials() { /// Job), and `status.manualRun` answers the exact timestamp requested. /// /// It then pins issue #394 against a REAL apiserver: a follow-up request parked -/// as `Pending` must have the finished run's `completedAt` merge-patched away, -/// which only the explicit-null serialization achieves. +/// as `Pending` must carry no `completedAt` VALUE from the finished run, which +/// only the explicit-null serialization achieves (the apiserver may delete the +/// key or store the null; either way the stale stamp is gone). #[tokio::test] #[ignore = "requires the e2e harness (mise run //crates/e2e:test): kind + built images + helm install"] async fn repository_replication_runs_on_demand_from_the_run_requested_annotation() { @@ -593,9 +594,11 @@ async fn repository_replication_runs_on_demand_from_the_run_requested_annotation ); // Issue #394: a FOLLOW-UP request must not inherit the finished run's - // `completedAt`. The non-terminal patch now serializes an explicit null, and - // an RFC-7386 merge-patch DELETES the key it is nulling — so the stale stamp - // disappears from the stored object rather than sitting under a `Pending`. + // `completedAt`. The non-terminal patch serializes an explicit null; the + // apiserver then either DELETES the key (plain RFC-7386) or stores the null + // verbatim — a nullable CRD field on k8s 1.33 was observed keeping it. Both + // converge on the property that matters and the assertion below tests + // exactly that: no stale timestamp survives under the fresh `Pending`. // // Suspending is what makes the park deterministic (#380 records an // unanswered request as `Pending` and stops). Ordering hazard: the @@ -638,9 +641,12 @@ async fn repository_replication_runs_on_demand_from_the_run_requested_annotation .await .expect("the suspended replication should record the new request as Pending"); assert!( - parked.get("completedAt").is_none(), - "a non-terminal manualRun must carry NO completedAt — the previous run's \ - stamp must have been merge-patched away, not left standing; got {parked}" + parked + .get("completedAt") + .is_none_or(serde_json::Value::is_null), + "a non-terminal manualRun must carry NO completedAt VALUE — absent or an \ + explicit null are both fine (the apiserver picks), but the previous run's \ + stamp must not still be standing; got {parked}" ); let _ = repls.delete(name, &DeleteParams::default()).await;