-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathmetrics.go
More file actions
346 lines (317 loc) · 11.7 KB
/
Copy pathmetrics.go
File metadata and controls
346 lines (317 loc) · 11.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
package api
import (
"context"
"math"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/forgekeep/nebula-mesh/internal/store"
)
// Result labels for counters whose terminal outcome falls into one of three
// buckets (request succeeded / was refused / failed unexpectedly). Used by
// enrollments, cert renewals, and operator logins. Exported so the Web UI
// package can use the same constants when reporting login outcomes back to
// the API server via RecordLogin.
const (
ResultOK = "ok"
ResultDenied = "denied"
ResultError = "error"
)
// Authentication factor labels for nebula_mgmt_operator_logins_total.
const (
LoginFactorPassword = "password"
LoginFactorTOTP = "totp"
LoginFactorRecovery = "recovery"
LoginFactorOIDC = "oidc"
)
// resultOK / resultDenied / resultError remain as package-local aliases so
// existing internal callers (enrollments, cert renewals) keep their concise
// names without having to switch to the exported spelling everywhere.
const (
resultOK = ResultOK
resultDenied = ResultDenied
resultError = ResultError
)
const (
loginFactorPassword = LoginFactorPassword
loginFactorTOTP = LoginFactorTOTP
loginFactorRecovery = LoginFactorRecovery
loginFactorOIDC = LoginFactorOIDC
)
// Audit action labels: stable identifiers for the audit log entries the API
// writes. Centralized here so the Prometheus counter, the audit log, and any
// future RBAC tooling can share the same vocabulary instead of duplicating
// string literals.
const (
auditHostCreate = "host.create"
auditHostDelete = "host.delete"
auditHostUpdate = "host.update"
auditHostBlock = "host.block"
auditHostUnblock = "host.unblock"
auditHostAuthFailed = "host.auth.failed"
auditHostRotateCertRequested = "host.rotate-cert.requested"
auditHostReenrollRequested = "host.reenroll.requested"
auditHostRekeyCompleted = "host.rekey.completed"
auditHostMobileBundleIssued = "host.mobile_bundle.issued"
auditHostMobileBundleForbidden = "host.mobile_bundle.forbidden"
auditCACreated = "ca.created"
auditCADeleted = "ca.deleted"
auditCARotated = "ca.rotated"
auditNetworkCreate = "network.create"
auditNetworkFirewallUpdate = "network.firewall.update"
auditOperatorCreate = "operator.create"
auditOperatorDisable = "operator.disable"
auditOperatorEnable = "operator.enable"
auditOperatorAPIKeyCreate = "operator.api_key.create"
auditOperatorAPIKeyRevoke = "operator.api_key.revoke"
auditSettingsEnforce2FA = "settings.enforce_2fa"
auditWebhookSubCreate = "webhook_subscription.create"
auditWebhookSubUpdate = "webhook_subscription.update"
auditWebhookSubDelete = "webhook_subscription.delete"
)
// Audit reason codes used in the `details` field of host.auth.failed audit
// entries (ADR 0004 §8 — telemetry). Surfaced as constants so server-side and
// test code share one source of truth.
const (
authReasonUnknownFingerprint = "unknown_fingerprint"
authReasonBadSignature = "bad_signature"
authReasonTimestampSkew = "timestamp_skew"
authReasonReplayedNonce = "replayed_nonce"
authReasonRevoked = "revoked"
authReasonGone = "gone"
)
// caIDFallback is the label value used when a CA signing event happens on
// the legacy single-CA fallback path (no per-host CAID is recorded).
const caIDFallback = "default"
// hostStatusNone is the label value emitted by the hosts gauge when there are
// no enrolled hosts at all — gives dashboards a stable schema to bind to.
const hostStatusNone = "none"
// metrics holds the Prometheus registry and every application metric exposed
// at /metrics. The registry is private to the server instance so test runs
// don't fight over Prometheus's default registerer global state.
type metrics struct {
reg *prometheus.Registry
httpRequests *prometheus.CounterVec
httpDuration *prometheus.HistogramVec
enrollments *prometheus.CounterVec
certRenewals *prometheus.CounterVec
auditEntries *prometheus.CounterVec
operatorLogins *prometheus.CounterVec
caSignatures *prometheus.CounterVec
}
// newMetrics builds a fresh metrics bundle bound to an isolated registry and
// pre-registers the application counters / histograms / collectors. Gauges
// derived from the store (host counts, soonest cert expiry) are exposed via
// a dedicated storeCollector so they always reflect the live state on scrape.
func newMetrics(s store.Store) *metrics {
reg := prometheus.NewRegistry()
m := &metrics{
reg: reg,
httpRequests: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "nebula_mgmt_http_requests_total",
Help: "Total number of HTTP requests served by the management API.",
},
[]string{"method", "route", "status"},
),
httpDuration: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "nebula_mgmt_http_request_duration_seconds",
Help: "Server-side latency for HTTP requests, in seconds.",
Buckets: prometheus.DefBuckets,
},
[]string{"method", "route"},
),
enrollments: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "nebula_mgmt_enrollments_total",
Help: "Enrollment attempts grouped by terminal result.",
},
[]string{"result"},
),
certRenewals: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "nebula_mgmt_cert_renewals_total",
Help: "Auto certificate-renewal attempts grouped by terminal result.",
},
[]string{"result"},
),
auditEntries: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "nebula_mgmt_audit_entries_total",
Help: "Audit log entries written, grouped by action.",
},
[]string{"action"},
),
operatorLogins: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "nebula_mgmt_operator_logins_total",
Help: "Operator login attempts grouped by terminal result and factor.",
},
[]string{"result", "factor"},
),
caSignatures: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "nebula_mgmt_ca_signatures_total",
Help: "Successful certificate signings, grouped by signing CA id.",
},
[]string{"ca_id"},
),
}
reg.MustRegister(m.httpRequests)
reg.MustRegister(m.httpDuration)
reg.MustRegister(m.enrollments)
reg.MustRegister(m.certRenewals)
reg.MustRegister(m.auditEntries)
reg.MustRegister(m.operatorLogins)
reg.MustRegister(m.caSignatures)
// Pre-seed canonical label combinations with zero so every metric shows
// up in the exposition output even before the first event, giving
// dashboards a stable schema and alerts something to threshold on.
results := []string{resultOK, resultDenied, resultError}
for _, result := range results {
m.enrollments.WithLabelValues(result).Add(0)
m.certRenewals.WithLabelValues(result).Add(0)
}
for _, factor := range []string{loginFactorPassword, loginFactorTOTP, loginFactorRecovery, loginFactorOIDC} {
for _, result := range results {
m.operatorLogins.WithLabelValues(result, factor).Add(0)
}
}
m.caSignatures.WithLabelValues(caIDFallback).Add(0)
for _, action := range []string{
auditHostCreate, auditHostDelete, auditHostUpdate, auditHostBlock, auditHostUnblock,
auditCACreated, auditCADeleted, auditCARotated,
auditNetworkCreate, auditNetworkFirewallUpdate,
auditOperatorCreate, auditOperatorDisable, auditOperatorEnable,
auditOperatorAPIKeyCreate, auditOperatorAPIKeyRevoke,
auditSettingsEnforce2FA,
auditWebhookSubCreate, auditWebhookSubUpdate, auditWebhookSubDelete,
} {
m.auditEntries.WithLabelValues(action).Add(0)
}
reg.MustRegister(newStoreCollector(s))
return m
}
// recordEnrollment bumps the enrollments counter. Result is one of
// "ok" / "denied" / "error" — matching the labels documented in the README.
func (m *metrics) recordEnrollment(result string) {
if m == nil {
return
}
m.enrollments.WithLabelValues(result).Inc()
}
func (m *metrics) recordRenewal(result string) {
if m == nil {
return
}
m.certRenewals.WithLabelValues(result).Inc()
}
func (m *metrics) recordAudit(action string) {
if m == nil {
return
}
m.auditEntries.WithLabelValues(action).Inc()
}
func (m *metrics) recordLogin(result, factor string) {
if m == nil {
return
}
m.operatorLogins.WithLabelValues(result, factor).Inc()
}
func (m *metrics) recordSignature(caID string) {
if m == nil {
return
}
if caID == "" {
caID = caIDFallback
}
m.caSignatures.WithLabelValues(caID).Inc()
}
// --- storeCollector -------------------------------------------------------
// storeCollector exposes gauges whose values depend on the current DB state:
//
// - nebula_mgmt_hosts{status, network} — total hosts grouped by status and
// network id.
// - nebula_mgmt_certificates_expiring_seconds — wall-clock seconds until
// the soonest-to-expire enrolled certificate. Negative values indicate the
// soonest cert has already expired. Reports +Inf when no certs exist.
//
// The collector is queried on every Prometheus scrape, so the work is bounded
// by the scrape cadence (typically 15-60s) and the host/cert table sizes.
type storeCollector struct {
store store.Store
hosts *prometheus.Desc
exp *prometheus.Desc
perHost *prometheus.Desc
}
func newStoreCollector(s store.Store) *storeCollector {
return &storeCollector{
store: s,
hosts: prometheus.NewDesc(
"nebula_mgmt_hosts",
"Total hosts grouped by status and network.",
[]string{"status", "network"}, nil,
),
exp: prometheus.NewDesc(
"nebula_mgmt_certificates_expiring_seconds",
"Seconds remaining until the soonest-to-expire enrolled certificate.",
nil, nil,
),
perHost: prometheus.NewDesc(
"nebula_mgmt_cert_expiring_seconds",
"Seconds remaining until the current certificate of each enrolled host expires.",
[]string{"host", "network"}, nil,
),
}
}
func (c *storeCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.hosts
ch <- c.exp
ch <- c.perHost
}
func (c *storeCollector) Collect(ch chan<- prometheus.Metric) {
ctx := context.Background()
// Hosts gauge: count by (status, network). Always emit at least one
// sample so the metric is visible in the exposition format even when
// no hosts are registered yet (gives dashboards a stable schema).
type key struct{ status, network string }
counts := map[key]float64{{status: hostStatusNone, network: ""}: 0}
if hosts, err := c.store.ListHosts(ctx, store.HostFilter{}); err == nil {
for _, h := range hosts {
counts[key{string(h.Status), h.NetworkID}]++
}
}
for k, n := range counts {
ch <- prometheus.MustNewConstMetric(c.hosts, prometheus.GaugeValue, n, k.status, k.network)
}
// Soonest-cert-expiry gauge: wall-clock seconds remaining until the
// soonest-to-expire enrolled certificate. Negative values mean the cert
// has already expired. Reports +Inf when no certs exist so absence is
// distinguishable from "expiring now". Also emits a per-host
// nebula_mgmt_cert_expiring_seconds{host, network} so alerting rules
// can target individual hosts (see issue #41 Prometheus sink).
seconds := math.Inf(1)
if certs, err := c.store.ListEnrolledHostCerts(ctx); err == nil {
hostNetwork := map[string]string{}
if hosts, herr := c.store.ListHosts(ctx, store.HostFilter{}); herr == nil {
for _, h := range hosts {
hostNetwork[h.ID] = h.NetworkID
}
}
soonest := time.Time{}
for _, ci := range certs {
remaining := time.Until(ci.NotAfter).Seconds()
ch <- prometheus.MustNewConstMetric(
c.perHost, prometheus.GaugeValue, remaining,
ci.HostID, hostNetwork[ci.HostID],
)
if soonest.IsZero() || ci.NotAfter.Before(soonest) {
soonest = ci.NotAfter
}
}
if !soonest.IsZero() {
seconds = time.Until(soonest).Seconds()
}
}
ch <- prometheus.MustNewConstMetric(c.exp, prometheus.GaugeValue, seconds)
}