feat: add per-monitor configuration to selectively disable monitors (#50)

* feat: merge e2e-ci into e2e

* add update-e2e-manifests to generate target

* feat: support monitor specific configurations

* update readme

* address comments + make generate

* improvements to logging and handle cases when configmap is deleted

* run make generate

Author: shvbsle

README.md

@@ -36,6 +36,47 @@ It is recommended to install the EKS Node Health Monitoring Agent as an EKS add-
 
 For detailed configuration options and usage documentation, refer to the [Amazon EKS Node Health documentation](https://docs.aws.amazon.com/eks/latest/userguide/node-health.html).
 
+## Configuring Monitors
+
+By default all monitors are enabled. Individual monitors can be disabled via the Helm chart's `nodeAgent.monitors` configuration or by providing a config file at `/etc/nma/config.yaml`.
+
+### Helm Values
+
+```yaml
+nodeAgent:
+  monitors:
+    networking:
+      enabled: false
+    neuron:
+      enabled: false
+```
+
+### Config File Format
+
+The agent reads a YAML config file mounted at `/etc/nma/config.yaml`. Omitted monitors default to enabled.
+
+```yaml
+monitors:
+  kernel-monitor:
+    enabled: true
+  networking:
+    enabled: false
+  storage-monitor:
+    enabled: true
+  nvidia:
+    enabled: true
+  neuron:
+    enabled: false
+  runtime:
+    enabled: true
+```
+
+Valid plugin names: `kernel-monitor`, `networking`, `storage-monitor`, `nvidia`, `neuron`, `runtime`.
+
+When a monitor is disabled:
+- Its health checks are not executed.
+- The corresponding `NodeCondition` (e.g., `NetworkingReady`) is not set on the node, avoiding false-positive healthy status for unmonitored subsystems.
+
 ## Building
 
 ```bash

charts/configuration.schema.json

@@ -47,6 +47,31 @@
                 },
                 "tolerations": {
                     "$ref": "#/definitions/Tolerations"
+                },
+                "monitors": {
+                    "type": "object",
+                    "description": "Per-monitor configuration keyed by plugin name",
+                    "additionalProperties": false,
+                    "properties": {
+                        "kernel-monitor": {
+                            "$ref": "#/definitions/MonitorSettings"
+                        },
+                        "networking": {
+                            "$ref": "#/definitions/MonitorSettings"
+                        },
+                        "storage-monitor": {
+                            "$ref": "#/definitions/MonitorSettings"
+                        },
+                        "nvidia": {
+                            "$ref": "#/definitions/MonitorSettings"
+                        },
+                        "neuron": {
+                            "$ref": "#/definitions/MonitorSettings"
+                        },
+                        "runtime": {
+                            "$ref": "#/definitions/MonitorSettings"
+                        }
+                    }
                 }
             }
         },
@@ -123,6 +148,19 @@
                 "type": "object"
             }
         },
+        "MonitorSettings": {
+            "title": "MonitorSettings",
+            "type": "object",
+            "description": "Per-monitor settings",
+            "additionalProperties": false,
+            "properties": {
+                "enabled": {
+                    "type": "boolean",
+                    "description": "Whether this monitor is enabled",
+                    "default": true
+                }
+            }
+        },
         "StringMap": {
             "title": "StringMap",
             "type": "object",

charts/eks-node-monitoring-agent/README.md

@@ -61,6 +61,7 @@ The following table lists the configurable parameters for this chart and their d
 | nodeAgent.image.pullPolicy | string | `"IfNotPresent"` | Container pull policyfor the eks-node-monitoring-agent |
 | nodeAgent.image.region | string | `"us-west-2"` | ECR repository region for the eks-node-monitoring-agent |
 | nodeAgent.image.tag | string | `"v1.5.2-eksbuild.1"` | Image tag for the eks-node-monitoring-agent |
+| nodeAgent.monitors | object | `{}` | Per-monitor configuration keyed by plugin name. See the main README for details. |
 | nodeAgent.podAnnotations | object | `{}` | Pod annotations applied to the eks-node-monitoring-agent |
 | nodeAgent.podLabels | object | `{}` | Pod labels applied to the eks-node-monitoring-agent |
 | nodeAgent.resources | object | `{"limits":{"cpu":"250m","memory":"200Mi"},"requests":{"cpu":"10m","memory":"30Mi"}}` | Container resources for the eks-node-monitoring-agent |

charts/eks-node-monitoring-agent/templates/daemonset.yaml

@@ -74,7 +74,18 @@ spec:
           volumeMounts:
             - name: host-root
               mountPath: /host
+            {{- if .Values.nodeAgent.monitors }}
+            - name: monitor-config
+              mountPath: /etc/nma
+              readOnly: true
+            {{- end }}
       volumes:
         - name: host-root
           hostPath:
             path: /
+        {{- if .Values.nodeAgent.monitors }}
+        - name: monitor-config
+          configMap:
+            name: {{ include "eks-node-monitoring-agent.fullname" . }}-monitor-config
+            optional: true
+        {{- end }}

charts/eks-node-monitoring-agent/templates/monitor-configmap.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.nodeAgent.monitors }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "eks-node-monitoring-agent.fullname" . }}-monitor-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "eks-node-monitoring-agent.labels" . | nindent 4 }}
+data:
+  config.yaml: |
+    monitors:
+      {{- toYaml .Values.nodeAgent.monitors | nindent 6 }}
+{{- end }}

charts/eks-node-monitoring-agent/values.yaml

@@ -84,6 +84,8 @@ nodeAgent:
     - operator: Exists
   # -- Pod labels applied to the eks-node-monitoring-agent
   podLabels: {}
+  # -- Per-monitor configuration keyed by plugin name. See the main README for details.
+  monitors: {}
   # -- Pod annotations applied to the eks-node-monitoring-agent
   podAnnotations: {}
 

cmd/eks-node-monitoring-agent/main.go

@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
+	"github.com/aws/eks-node-monitoring-agent/api/monitor"
 	"github.com/aws/eks-node-monitoring-agent/api/v1alpha1"
 	"github.com/aws/eks-node-monitoring-agent/pkg/conditions"
 	"github.com/aws/eks-node-monitoring-agent/pkg/config"
@@ -39,6 +40,7 @@ import (
 	_ "github.com/aws/eks-node-monitoring-agent/monitors/neuron"
 	_ "github.com/aws/eks-node-monitoring-agent/monitors/nvidia"
 	_ "github.com/aws/eks-node-monitoring-agent/monitors/storage"
+
 	// Import monitors that require explicit registration (can't use init())
 	"github.com/aws/eks-node-monitoring-agent/monitors/runtime"
 	// Import observer packages to register observers
@@ -185,47 +187,88 @@ func run() error {
 		return err
 	}
 
-	// Get all registered monitors from the global plugin registry
-	allMonitors := registry.GlobalRegistry().AllMonitors()
-	if len(allMonitors) == 0 {
-		logger.Info("no monitors registered - agent will run without monitoring capabilities")
-		return fmt.Errorf("no monitors registered")
+	// Load monitor configuration from ConfigMap mount
+	monitorConfig, configFound, err := config.LoadMonitorConfig(config.DefaultConfigPath)
+	if err != nil {
+		logger.Error(err, "failed to load monitor configuration")
+		return err
+	}
+	if !configFound {
+		logger.Info("monitor config file not found, all monitors will be enabled by default", "path", config.DefaultConfigPath)
 	}
 
-	logger.Info("registered monitors", "count", len(allMonitors))
-	for _, mon := range allMonitors {
-		logger.Info("monitor available", "name", mon.Name())
+	// Filter plugins by configuration and log effective state
+	allPlugins := registry.GlobalRegistry().List()
+	var enabledMonitors []monitor.Monitor
+	var disabledNames []string
+
+	for _, plugin := range allPlugins {
+		enabled := monitorConfig.IsMonitorEnabled(plugin.Name())
+		logger.Info("monitor configuration", "plugin", plugin.Name(), "enabled", enabled)
+		if !enabled {
+			disabledNames = append(disabledNames, plugin.Name())
+			continue
+		}
+		enabledMonitors = append(enabledMonitors, plugin.Monitors()...)
 	}
 
-	// Build condition configs for node exporter
+	if len(disabledNames) > 0 {
+		logger.Info("monitors disabled by configuration", "plugins", disabledNames)
+	}
+
+	if len(enabledMonitors) == 0 {
+		logger.Info("all monitors are disabled by configuration, NMA will not perform any monitoring")
+	} else {
+		logger.Info("enabled monitors", "count", len(enabledMonitors))
+		for _, mon := range enabledMonitors {
+			logger.Info("monitor available", "name", mon.Name())
+		}
+	}
+
+	// Build condition configs for node exporter, only for enabled monitors.
+	// NodeExporter unconditionally sets all provided conditions to ConditionTrue,
+	// so we must exclude conditions for disabled monitors to avoid falsely
+	// reporting health for subsystems that are not being monitored.
 	conditionConfigs := make(map[corev1.NodeConditionType]manager.NodeConditionConfig)
-	conditionConfigs[conditions.KernelReady] = manager.NodeConditionConfig{
-		ReadyReason:  "KernelIsReady",
-		ReadyMessage: "Monitoring for the Kernel system is active",
+	if monitorConfig.IsMonitorEnabled("kernel-monitor") {
+		conditionConfigs[conditions.KernelReady] = manager.NodeConditionConfig{
+			ReadyReason:  "KernelIsReady",
+			ReadyMessage: "Monitoring for the Kernel system is active",
+		}
 	}
-	conditionConfigs[conditions.StorageReady] = manager.NodeConditionConfig{
-		ReadyReason:  "DiskIsReady",
-		ReadyMessage: "Monitoring for the Disk system is active",
+	if monitorConfig.IsMonitorEnabled("storage-monitor") {
+		conditionConfigs[conditions.StorageReady] = manager.NodeConditionConfig{
+			ReadyReason:  "DiskIsReady",
+			ReadyMessage: "Monitoring for the Disk system is active",
+		}
 	}
-	conditionConfigs[conditions.ContainerRuntimeReady] = manager.NodeConditionConfig{
-		ReadyReason:  "ContainerRuntimeIsReady",
-		ReadyMessage: "Monitoring for the ContainerRuntime system is active",
+	if monitorConfig.IsMonitorEnabled("runtime") {
+		conditionConfigs[conditions.ContainerRuntimeReady] = manager.NodeConditionConfig{
+			ReadyReason:  "ContainerRuntimeIsReady",
+			ReadyMessage: "Monitoring for the ContainerRuntime system is active",
+		}
 	}
-	conditionConfigs[conditions.NetworkingReady] = manager.NodeConditionConfig{
-		ReadyReason:  "NetworkingIsReady",
-		ReadyMessage: "Monitoring for the Networking system is active",
+	if monitorConfig.IsMonitorEnabled("networking") {
+		conditionConfigs[conditions.NetworkingReady] = manager.NodeConditionConfig{
+			ReadyReason:  "NetworkingIsReady",
+			ReadyMessage: "Monitoring for the Networking system is active",
+		}
 	}
 
 	switch runtimeContext.AcceleratedHardware() {
 	case config.AcceleratedHardwareNvidia:
-		conditionConfigs[conditions.AcceleratedHardwareReady] = manager.NodeConditionConfig{
-			ReadyReason:  "NvidiaGPUIsReady",
-			ReadyMessage: "Monitoring for the Nvidia GPU system is active",
+		if monitorConfig.IsMonitorEnabled("nvidia") {
+			conditionConfigs[conditions.AcceleratedHardwareReady] = manager.NodeConditionConfig{
+				ReadyReason:  "NvidiaGPUIsReady",
+				ReadyMessage: "Monitoring for the Nvidia GPU system is active",
+			}
 		}
 	case config.AcceleratedHardwareNeuron:
-		conditionConfigs[conditions.AcceleratedHardwareReady] = manager.NodeConditionConfig{
-			ReadyReason:  "NeuronAcceleratedHardwareIsReady",
-			ReadyMessage: "Monitoring for the Neuron AcceleratedHardware system is active",
+		if monitorConfig.IsMonitorEnabled("neuron") {
+			conditionConfigs[conditions.AcceleratedHardwareReady] = manager.NodeConditionConfig{
+				ReadyReason:  "NeuronAcceleratedHardwareIsReady",
+				ReadyMessage: "Monitoring for the Neuron AcceleratedHardware system is active",
+			}
 		}
 	}
 
@@ -244,7 +287,7 @@ func run() error {
 	monitorMgr := manager.NewMonitorManager(hostname, nodeExporter)
 
 	// Register all monitors with the manager
-	for _, mon := range allMonitors {
+	for _, mon := range enabledMonitors {
 		monCtx := log.IntoContext(ctx, logger.WithValues("monitor", mon.Name()))
 		var conditionType corev1.NodeConditionType
 		switch mon.Name() {

pkg/config/monitor.go

@@ -0,0 +1,105 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"slices"
+	"sort"
+	"strings"
+
+	"sigs.k8s.io/yaml"
+)
+
+const DefaultConfigPath = "/etc/nma/config.yaml"
+
+// MonitorSettings holds per-monitor configuration.
+type MonitorSettings struct {
+	Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"`
+}
+
+// IsEnabled returns true if the monitor is enabled.
+func (ms MonitorSettings) IsEnabled() bool {
+	// Defaults to true when Enabled is nil (not explicitly set).
+	// this is to ensure backward compatibility and consistency
+	if ms.Enabled == nil {
+		return true
+	}
+	return *ms.Enabled
+}
+
+// MonitorConfig is the top-level configuration structure.
+type MonitorConfig struct {
+	Monitors map[string]MonitorSettings `yaml:"monitors,omitempty" json:"monitors,omitempty"`
+}
+
+// IsMonitorEnabled checks if a given plugin is enabled.
+// Returns true if the config is nil, the map is nil, or the plugin is not present in the map.
+func (mc *MonitorConfig) IsMonitorEnabled(pluginName string) bool {
+	if mc == nil || mc.Monitors == nil {
+		return true
+	}
+	settings, exists := mc.Monitors[pluginName]
+	if !exists {
+		return true
+	}
+	return settings.IsEnabled()
+}
+
+// KnownPluginNames is the set of valid plugin names for validation.
+var KnownPluginNames = []string{
+	"kernel-monitor",
+	"networking",
+	"storage-monitor",
+	"nvidia",
+	"neuron",
+	"runtime",
+}
+
+// Validate checks that all keys in Monitors are known plugin names.
+func (mc *MonitorConfig) Validate() error {
+	if mc == nil || mc.Monitors == nil {
+		return nil
+	}
+	var unknown []string
+	for name := range mc.Monitors {
+		if !slices.Contains(KnownPluginNames, name) {
+			unknown = append(unknown, name)
+		}
+	}
+	if len(unknown) > 0 {
+		sort.Strings(unknown)
+		return fmt.Errorf("unknown monitor plugin name(s): %s", strings.Join(unknown, ", "))
+	}
+	return nil
+}
+
+// LoadMonitorConfig reads the config file at the given path.
+// Returns a default (all-enabled) config if the file does not exist.
+// Returns an error if the file exists but contains invalid YAML or unknown plugin names.
+// The second return value indicates whether the config file was found on disk.
+func LoadMonitorConfig(path string) (*MonitorConfig, bool, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return &MonitorConfig{}, false, nil
+		}
+		return nil, false, fmt.Errorf("reading monitor config: %w", err)
+	}
+
+	// Empty file is treated as default config (all monitors enabled).
+	if len(data) == 0 {
+		return &MonitorConfig{}, true, nil
+	}
+
+	cfg := &MonitorConfig{}
+	if err := yaml.UnmarshalStrict(data, cfg); err != nil {
+		return nil, false, fmt.Errorf("parsing monitor config: %w", err)
+	}
+
+	if err := cfg.Validate(); err != nil {
+		return nil, false, fmt.Errorf("validating monitor config: %w", err)
+	}
+
+	return cfg, true, nil
+}