feat: enhanced lightnode reorg observability

Author: jelias2

op-node/metrics/metrics.go

@@ -35,6 +35,8 @@ type Metricer interface {
 	SetSequencerState(active bool)
 	RecordPipelineReset()
 	RecordFollowSourceRequest(result string)
+	RecordFollowSourceReorg(action string)
+	RecordSuperAuthorityReorgSignal(reason string)
 	RecordSequencingError()
 	RecordPublishingError()
 	RecordDerivationError()
@@ -84,8 +86,10 @@ type Metrics struct {
 	L1SourceCache *metrics.CacheMetrics
 	L2SourceCache *metrics.CacheMetrics
 
-	L2FollowSourceCache  *metrics.CacheMetrics
-	FollowSourceRequests *prometheus.CounterVec
+	L2FollowSourceCache        *metrics.CacheMetrics
+	FollowSourceRequests       *prometheus.CounterVec
+	FollowSourceReorgs         *prometheus.CounterVec
+	SuperAuthorityReorgSignals *prometheus.CounterVec
 
 	DerivationIdle prometheus.Gauge
 
@@ -198,6 +202,8 @@ func NewMetrics(procName string, labels prometheus.Labels) *Metrics {
 			Name:      "follow_source_requests_total",
 			Help:      "Count of follow source requests by result",
 		}, []string{"result"}),
+		FollowSourceReorgs:         factory.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "follow_source_reorgs_total", Help: "Count of follow source reorg decisions by action"}, []string{"action"}),
+		SuperAuthorityReorgSignals: factory.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "super_authority_reorg_signals_total", Help: "Count of super authority reorg signals by reason"}, []string{"reason"}),
 
 		DerivationIdle: factory.NewGauge(prometheus.GaugeOpts{
 			Namespace: ns,
@@ -515,6 +521,14 @@ func (m *Metrics) RecordFollowSourceRequest(result string) {
 	m.FollowSourceRequests.WithLabelValues(result).Inc()
 }
 
+func (m *Metrics) RecordFollowSourceReorg(action string) {
+	m.FollowSourceReorgs.WithLabelValues(action).Inc()
+}
+
+func (m *Metrics) RecordSuperAuthorityReorgSignal(reason string) {
+	m.SuperAuthorityReorgSignals.WithLabelValues(reason).Inc()
+}
+
 func (m *Metrics) RecordSequencerReset() {
 	m.SequencerResets.Record()
 }
@@ -670,6 +684,12 @@ func (n *noopMetricer) RecordPipelineReset() {
 func (n *noopMetricer) RecordFollowSourceRequest(result string) {
 }
 
+func (n *noopMetricer) RecordFollowSourceReorg(action string) {
+}
+
+func (n *noopMetricer) RecordSuperAuthorityReorgSignal(reason string) {
+}
+
 func (n *noopMetricer) RecordSequencingError() {
 }
 

op-node/rollup/engine/engine_controller.go

@@ -251,6 +251,7 @@ func (e *EngineController) resolveVerifiedAsSafe(block eth.BlockID) eth.L2BlockR
 	}
 	br, err := e.engine.L2BlockRefByHash(e.ctx, block.Hash)
 	if err != nil {
+		e.metrics.RecordSuperAuthorityReorgSignal("unknown_to_engine")
 		e.log.Warn("super authority safe head unknown to engine (reorg signal)",
 			"super_authority_safe", block, "err", err)
 		return e.crossSafeFallback("el-unknown")
@@ -260,6 +261,7 @@ func (e *EngineController) resolveVerifiedAsSafe(block eth.BlockID) eth.L2BlockR
 			"super_authority_safe", br, "err", err)
 		return e.crossSafeFallback("canonicality-lookup-failed")
 	} else if !canonical {
+		e.metrics.RecordSuperAuthorityReorgSignal("non_canonical")
 		e.log.Warn("super authority safe head non-canonical (reorg signal)",
 			"super_authority_safe", br, "canonical", canonicalRef)
 		return e.crossSafeFallback("non-canonical")
@@ -1602,6 +1604,12 @@ func (e *EngineController) FollowSource(eSafeBlockRef, eLocalSafeRef, eFinalized
 
 	// External local safe is found locally but differs: the follower diverged from upstream
 	// and must reorg onto it.
+	e.log.Warn("Follow Source: local safe diverged from upstream",
+		"external_local_safe", eLocalSafeRef,
+		"local_safe", e.localSafeHead,
+		"local_unsafe", e.unsafeHead,
+		"external_safe", eSafeBlockRef,
+		"finalized", eFinalizedRef)
 	if e.originSelectorResetter != nil {
 		// This follower is also a sequencer (the origin-selector resetter is wired only when
 		// sequencing is enabled). A soft unsafe-head update would be clobbered by the next
@@ -1610,12 +1618,24 @@ func (e *EngineController) FollowSource(eSafeBlockRef, eLocalSafeRef, eFinalized
 		// cancels the in-flight build and FCUs onto the upstream chain in one shot (head==safe);
 		// the sequencer then rebuilds from there.
 		logger.Warn("Follow Source: Reorg onto upstream chain")
+		e.metrics.RecordFollowSourceReorg("force_reset")
 		e.forceReset(e.ctx, eLocalSafeRef, eLocalSafeRef, eLocalSafeRef, eSafeBlockRef, eFinalizedRef, false)
+		e.log.Info("Follow Source: reorg onto upstream chain completed",
+			"action", "force_reset",
+			"local_safe", e.localSafeHead,
+			"local_unsafe", e.unsafeHead,
+			"cross_safe", e.SafeL2Head())
 		return
 	}
 
 	// Pure verifier (no local block production): a soft update suffices and may trigger or
 	// retarget EL sync.
+	e.metrics.RecordFollowSourceReorg("soft_update")
 	logger.Warn("Follow Source: Reorg. May Trigger EL sync")
 	followExternalRefs(true)
+	e.log.Info("Follow Source: upstream refs applied",
+		"action", "soft_update",
+		"local_safe", e.localSafeHead,
+		"local_unsafe", e.unsafeHead,
+		"cross_safe", e.SafeL2Head())
 }