feat(prometheus): add tags to prometheus metrics

server/model/monitor.js

@@ -349,7 +349,7 @@ class Monitor extends BeanModel {
         let previousBeat = null;
         let retries = 0;
 
-        this.prometheus = new Prometheus(this);
+        this.prometheus = new Prometheus(this, await this.getTags());
 
         const beat = async () => {
 

server/prometheus.js

@@ -1,46 +1,22 @@
 const PrometheusClient = require("prom-client");
 const { log } = require("../src/util");
+const { R } = require("redbean-node");
 
-const commonLabels = [
-    "monitor_id",
-    "monitor_name",
-    "monitor_type",
-    "monitor_url",
-    "monitor_hostname",
-    "monitor_port",
-];
-
-const monitorCertDaysRemaining = new PrometheusClient.Gauge({
-    name: "monitor_cert_days_remaining",
-    help: "The number of days remaining until the certificate expires",
-    labelNames: commonLabels
-});
-
-const monitorCertIsValid = new PrometheusClient.Gauge({
-    name: "monitor_cert_is_valid",
-    help: "Is the certificate still valid? (1 = Yes, 0= No)",
-    labelNames: commonLabels
-});
-const monitorResponseTime = new PrometheusClient.Gauge({
-    name: "monitor_response_time",
-    help: "Monitor Response Time (ms)",
-    labelNames: commonLabels
-});
-
-const monitorStatus = new PrometheusClient.Gauge({
-    name: "monitor_status",
-    help: "Monitor Status (1 = UP, 0= DOWN, 2= PENDING, 3= MAINTENANCE)",
-    labelNames: commonLabels
-});
+let monitorCertDaysRemaining = null;
+let monitorCertIsValid = null;
+let monitorResponseTime = null;
+let monitorStatus = null;
 
 class Prometheus {
     monitorLabelValues = {};
 
     /**
      * @param {object} monitor Monitor object to monitor
+     * @param {Array<{name:string,value:?string}>} tags Tags to add to the monitor
      */
-    constructor(monitor) {
+    constructor(monitor, tags) {
         this.monitorLabelValues = {
+            ...this.mapTagsToLabels(tags),
             monitor_id: monitor.id,
             monitor_name: monitor.name,
             monitor_type: monitor.type,
@@ -50,14 +26,108 @@ class Prometheus {
         };
     }
 
+    /**
+     * Initialize Prometheus metrics, and add all available tags as possible labels.
+     * This should be called once at the start of the application.
+     * New tags will NOT be added dynamically, a restart is sadly required to add new tags to the metrics.
+     * Existing tags added to monitors will be updated automatically.
+     * @returns {Promise<void>}
+     */
+    static async init() {
+        // Add all available tags as possible labels,
+        // and use Set to remove possible duplicates (for when multiple tags contain non-ascii characters, and thus are sanitized to the same label)
+        const tags = new Set((await R.findAll("tag")).map((tag) => {
+            return Prometheus.sanitizeForPrometheus(tag.name);
+        }).filter((tagName) => {
+            return tagName !== "";
+        }).sort(this.sortTags));
+
+        const commonLabels = [
+            ...tags,
+            "monitor_id",
+            "monitor_name",
+            "monitor_type",
+            "monitor_url",
+            "monitor_hostname",
+            "monitor_port",
+        ];
+
+        monitorCertDaysRemaining = new PrometheusClient.Gauge({
+            name: "monitor_cert_days_remaining",
+            help: "The number of days remaining until the certificate expires",
+            labelNames: commonLabels
+        });
+
+        monitorCertIsValid = new PrometheusClient.Gauge({
+            name: "monitor_cert_is_valid",
+            help: "Is the certificate still valid? (1 = Yes, 0= No)",
+            labelNames: commonLabels
+        });
+
+        monitorResponseTime = new PrometheusClient.Gauge({
+            name: "monitor_response_time",
+            help: "Monitor Response Time (ms)",
+            labelNames: commonLabels
+        });
+
+        monitorStatus = new PrometheusClient.Gauge({
+            name: "monitor_status",
+            help: "Monitor Status (1 = UP, 0= DOWN, 2= PENDING, 3= MAINTENANCE)",
+            labelNames: commonLabels
+        });
+    }
+
+    /**
+     * Sanitize a string to ensure it can be used as a Prometheus label or value.
+     * See https://github.com/louislam/uptime-kuma/pull/4704#issuecomment-2366524692
+     * @param {string} text The text to sanitize
+     * @returns {string} The sanitized text
+     */
+    static sanitizeForPrometheus(text) {
+        text = text.replace(/[^a-zA-Z0-9_]/g, "");
+        text = text.replace(/^[^a-zA-Z_]+/, "");
+        return text;
+    }
+
+    /**
+     * Map the tags value to valid labels used in Prometheus. Sanitize them in the process.
+     * @param {Array<{name: string, value:?string}>} tags The tags to map
+     * @returns {object} The mapped tags, usable as labels
+     */
+    mapTagsToLabels(tags) {
+        let mappedTags = {};
+        tags.forEach((tag) => {
+            let sanitizedTag = Prometheus.sanitizeForPrometheus(tag.name);
+            if (sanitizedTag === "") {
+                return; // Skip empty tag names
+            }
+
+            if (mappedTags[sanitizedTag] === undefined) {
+                mappedTags[sanitizedTag] = [];
+            }
+
+            let tagValue = Prometheus.sanitizeForPrometheus(tag.value || "");
+            if (tagValue !== "") {
+                mappedTags[sanitizedTag].push(tagValue);
+            }
+
+            mappedTags[sanitizedTag] = mappedTags[sanitizedTag].sort();
+        });
+
+        // Order the tags alphabetically
+        return Object.keys(mappedTags).sort(this.sortTags).reduce((obj, key) => {
+            obj[key] = mappedTags[key];
+            return obj;
+        }, {});
+    }
+
     /**
      * Update the metrics page
      * @param {object} heartbeat Heartbeat details
      * @param {object} tlsInfo TLS details
      * @returns {void}
      */
     update(heartbeat, tlsInfo) {
-
         if (typeof tlsInfo !== "undefined") {
             try {
                 let isValid;
@@ -118,6 +188,27 @@ class Prometheus {
             console.error(e);
         }
     }
+
+    /**
+     * Sort the tags alphabetically, case-insensitive.
+     * @param {string} a The first tag to compare
+     * @param {string} b The second tag to compare
+     * @returns {number} The alphabetical order number
+     */
+    sortTags(a, b) {
+        const aLowerCase = a.toLowerCase();
+        const bLowerCase = b.toLowerCase();
+
+        if (aLowerCase < bLowerCase) {
+            return -1;
+        }
+
+        if (aLowerCase > bLowerCase) {
+            return 1;
+        }
+
+        return 0;
+    }
 }
 
 module.exports = {

server/server.js

@@ -108,6 +108,8 @@ const { apiAuth } = require("./auth");
 const { login } = require("./auth");
 const passwordHash = require("./password-hash");
 
+const { Prometheus } = require("./prometheus");
+
 const hostname = config.hostname;
 
 if (hostname) {
@@ -192,6 +194,9 @@ let needSetup = false;
     server.entryPage = await Settings.get("entryPage");
     await StatusPage.loadDomainMappingList();
 
+    log.debug("server", "Initializing Prometheus");
+    await Prometheus.init();
+
     log.debug("server", "Adding route");
 
     // ***************************