Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cloud/guardian/alert.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ const URGENT_WARNINGS = [
/no 72-hour figure/,
/was stopped on Floor/,
/guardian was ejected on Floor/,
// A followed Floor config replaying rows at or before the start slot. With
// a late start slot (an endpoint ahead of its chain) they landed after the
// start, and their events (a key replaced or removed, a restore) are never
// reported; a lagging node logs it too, and the two cannot be told apart.
// Only rows listed after the first good poll: that poll's replay is silent.
/listed transactions at or before the start slot/,
];

/** Actions sent on chain: each one is news. */
Expand Down
63 changes: 58 additions & 5 deletions cloud/guardian/follow.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -179,9 +179,10 @@ export async function followAccount(fetchPage, { cursor = null, oldestWanted = n
/**
* One poll of each watched Floor config (`configs`: `{ address, program }`):
* followAccount from its cursor in `cursors`, then `ingest(sigs, live,
* checkpoint, program)`. A config not yet in `followed` resumes as a replay
* (not live), walking back to `oldestWanted()` at most (a day back, as the gap
* message says). One config whose walk or ingest fails no longer stops the
* checkpoint, program)`. A config not yet in `followed` resumes, walking back
* to `oldestWanted()` at most (a day back, as the gap message says), and what
* landed before the start is a replay (not live; see below). One config whose
* walk or ingest fails no longer stops the
* configs after it: the others are followed, and then this throws, naming
* each that failed, so a failure that lasts raises the stall alert. A plain
* loop used to stop at the first failure, and the later programs' executions
Expand All @@ -190,8 +191,22 @@ export async function followAccount(fetchPage, { cursor = null, oldestWanted = n
* scanAll's scans do: ingesting acts and checkpoints, and walked together, a
* config listed twice would be read twice from the same cursor. So an outage
* of every endpoint costs one timeout per config.
*
* The line is the slot this process started at (`liveAfterSlot`, from
* startSlot), on every poll: the first signature listed in a slot at or
* before it, and all listed after it, are replayed; the newer ones are
* live. It used to be the first good poll: a restart
* whose replay failed stayed a replay until one succeeded, and one that
* succeeded replayed what landed during the Vault startup, so a recovery
* key replaced, a stop or an ejection after the start was read silently.
* Without `liveAfterSlot`, a config resuming replays all it reads, and a
* followed one acts on all. The node lists newest-first by slot (the walk
* relies on it too), so the older part is replayed first and the newer one
* then ingested live. A followed config that lists rows from before the
* start (a lagging node, or a start slot from an endpoint ahead of its
* chain) says so, with the slots, once they are read.
*/
export async function followConfigs(configs, { page, cursors, followed, oldestWanted, knows, ingest, log, saved = () => {} }) {
export async function followConfigs(configs, { page, cursors, followed, oldestWanted, knows, ingest, log, saved = () => {}, liveAfterSlot = null }) {
const failed = [];
for (const { address, program } of configs) {
try {
Expand All @@ -206,7 +221,19 @@ export async function followConfigs(configs, { page, cursors, followed, oldestWa
else cursors.delete(address);
// No signature at all yet: ask again next poll, never from genesis.
if (r.status === "empty") continue;
await ingest(r.sigs, !resuming, (signature, s) => cursors.set(address, { signature, slot: s.slot ?? 0 }), program);
const checkpoint = (signature, s) => cursors.set(address, { signature, slot: s.slot ?? 0 });
const k = liveAfterSlot == null ? (resuming ? 0 : r.sigs.length) : newerThan(r.sigs, liveAfterSlot);
const live = r.sigs.slice(0, k);
const replay = r.sigs.slice(k);
if (replay.length) await ingest(replay, false, checkpoint, program);
// Once read, so a replay that keeps failing is not said again each poll.
if (!resuming && replay.length) {
// A row with no slot can sit in the replay part; the first row has one.
const known = replay.map((s) => s.slot).filter((s) => s != null);
const slots = [Math.min(...known), Math.max(...known)];
log({ level: "warn", msg: `a followed Floor config listed transactions at or before the start slot: replayed, not acted on. A lagging RPC node (they landed before this process started), or a start slot from an endpoint ahead of its chain (they landed after it, and their events were not reported): ${replay.length} rows in slots ${slots[0]}-${slots[1]}, start slot ${liveAfterSlot}`, program, rows: replay.length, startSlot: liveAfterSlot, slots });
}
if (live.length) await ingest(live, true, checkpoint, program);
followed.add(address);
} catch (e) {
failed.push(`${program}: ${e?.message ?? e}`);
Expand All @@ -217,6 +244,32 @@ export async function followConfigs(configs, { page, cursors, followed, oldestWa
if (failed.length) throw new Error(failed.join("; "));
}

/** How many of newest-first `sigs` come before the first in a slot at or
* before `slot`. A row with no slot (null over JSON-RPC) never ends them, so
* one among the newest is acted on (twice at worst, never unsaid); one below
* the line is older than it, and replayed with it. */
function newerThan(sigs, slot) {
const k = sigs.findIndex((s) => s.slot != null && s.slot <= slot);
return k < 0 ? sigs.length : k;
}

/**
* The slot a restart's Floor replay runs up to (followConfigs'
* `liveAfterSlot`): the highest confirmed slot that any of a FailoverRpc's
* endpoints gives. One endpoint behind would put the line early, and what
* landed while the guardian was down would be acted on as new. If none
* answers, or `rpc` has no endpoints (a plain Rpc), `rpc` itself is asked:
* an endpoint asked directly leaves at once when busy (`hurry`), and
* FailoverRpc.call waits for one once. Throws when that fails too.
*/
export async function startSlot(rpc) {
const ask = (e) => e.call("getSlot", [{ commitment: "confirmed" }]).catch(() => null);
let slots = (await Promise.all((rpc.endpoints ?? []).map(ask))).filter(Number.isInteger);
if (!slots.length) slots = [await ask(rpc)].filter(Number.isInteger);
if (!slots.length) throw new Error("no RPC endpoint gave its slot, so the Floor replay has no start");
return Math.max(...slots);
}

/** Process one oldest-first signature batch, checkpointing only completed items. */
export async function ingestSignatures(signatures, ingestOne, checkpoint = () => {}) {
for (const signature of [...signatures].reverse()) {
Expand Down
20 changes: 14 additions & 6 deletions cloud/guardian/main.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ import { Rpc, FailoverRpc, b58encode, decodeTicket, key, decodeProtocol, decodeF
import { Keypair, sendWithFallback, sendEscalating, priorityPrice } from "../lib/signer.mjs";
import { VaultClient, FloorClient } from "../lib/bulkhead.mjs";
import { assess, watchedEvents, checkFloors, guardianSetCheck, newState, observeProtocolFreeze, pendingCancels, rememberTicket, retirePendingFreeze, decisionTime, judgeReplayedWindow, queueFailedVeto, openIncident, observeFreezeEnd, incidentHold, guardianStanding, afterHeartbeat, heartbeatEveryAfterAccount, HEARTBEAT_ALERT_AFTER, pendingChangeVeto, pendingExitVeto, pendingUpgradeVeto, dueApprovedChange, dueApprovedFloor, vetoRetryDisposition, readChangeVeto, readPendingChange, checkPolicy, checkConfig, GENESIS, checkProposedChange, spendExecution, observeExecutions, unpinnedChangeApprovals, parseApproval, changeApproval, floorApproval, afterCancelFailure, CANCEL_RETRY_MS, DEFAULT_POLICY, DAY_S, outflowAlerts, OUTFLOW_KEEP_S, ticketAlreadyGone, freezeAlreadyCovers, replayedBreachTime, ticketScanDue } from "./policy.mjs";
import { ingestBudgeted, ingestLagS, signatureWalk, signatureWalkSinceSlot, followConfigs, cursorGone, PAGE_LOG_INTERVAL, requireTransaction, transactionOrSkip } from "./follow.mjs";
import { ingestBudgeted, ingestLagS, signatureWalk, signatureWalkSinceSlot, followConfigs, startSlot, cursorGone, PAGE_LOG_INTERVAL, requireTransaction, transactionOrSkip } from "./follow.mjs";
import { executeChangeIx as executeChangeIxFor } from "./keeper.mjs";
import { alerter, stallWatch, scanAll, actionLine } from "./alert.mjs";
import { activeTickets, activeChanges, ticketRelease, settlementRows, truncatedOutflows, unexplainedOutflow } from "./chain.mjs";
Expand Down Expand Up @@ -1240,8 +1240,10 @@ async function step(name, run) {
}
}

// Watched Floor configs read at least once by this process (followFloor).
// Watched Floor configs read at least once by this process (followFloor), and
// the slot it started at (main): a replay covers only what landed up to it.
const floorFollowed = new Set();
let floorLiveAfterSlot = null;

/**
* Floor's events (a proposal, an execution, a replaced recovery key) are
Expand All @@ -1251,10 +1253,12 @@ const floorFollowed = new Set();
* own, and only that program's Floor events are read from it.
*
* Its cursor is kept in the state file. A restart reads on from it, back to
* at most the Vault startup's day, as a replay (nothing is sent: the account
* scans veto what is still pending), so an execution while this guardian was
* down still spends its approval. Each restart used to start at the newest
* signature, and those executions were never seen (docs/09, finding 71).
* at most the Vault startup's day, as a replay of what landed up to the slot
* this process started at (nothing is sent: the account scans veto what is
* still pending), and what is newer live, so an execution while this
* guardian was down still spends its approval. Each restart used to start at
* the newest signature, and those executions were never seen (docs/09,
* finding 71).
* Without a cursor (a first start), it anchors at the newest signature; the
* account scans cover what came before.
*/
Expand All @@ -1263,6 +1267,7 @@ async function followFloor() {
page: (address, q) => rpc.call("getSignaturesForAddress", [address, q]),
cursors: floorCursors,
followed: floorFollowed,
liveAfterSlot: floorLiveAfterSlot,
oldestWanted: () => wallNow() - DAY_S,
knows: knowsSignature,
// Live, bounded like the protocol's ingest (finding 76): what is left is
Expand Down Expand Up @@ -1337,6 +1342,9 @@ async function main() {
for (const w of configWarnings) log({ level: "warn", msg: w });
if (stateProblem) log({ level: "error", msg: stateProblem });
await must(checkCluster);
// Fatal, like the cluster check: with no start slot, a restart would replay
// all that lands until its first good poll.
if (floorWatch.length) floorLiveAfterSlot = await must(() => startSlot(rpc));
await checkBalance().catch((e) => log({ level: "warn", msg: `could not read the guardian key's balance: ${e.message}` }));
if (state.incidentSince != null) log({ msg: "incident hold resumed from saved state", incidentSince: state.incidentSince, stateFile });
const unpinned = unpinnedChangeApprovals(policy);
Expand Down
Loading