[tmpnet] Enable monitoring of local kind cluster

Author: maru-ava

scripts/start_kind_cluster.sh

@@ -8,7 +8,7 @@ set -euo pipefail
 # Enable reuse of the arguments to ginkgo relevant to starting a cluster
 START_CLUSTER_ARGS=()
 for arg in "$@"; do
-  if [[ "${arg}" =~ "--kubeconfig=" || "${arg}" =~ "--kubeconfig-context=" ]]; then
+  if [[ "${arg}" =~ "--kubeconfig=" || "${arg}" =~ "--kubeconfig-context=" || "${arg}" =~ "--start-metrics-collector" || "${arg}" =~ "--start-logs-controller" ]]; then
     START_CLUSTER_ARGS+=("${arg}")
   fi
 done

tests/fixture/bootstrapmonitor/e2e/e2e_test.go

@@ -273,6 +273,7 @@ func newNodeStatefulSet(name string, flags tmpnet.FlagsMap) *appsv1.StatefulSet
 		volumeSize,
 		nodeDataDir,
 		flags,
+		nil, /* labels */
 	)
 
 	// The version annotations key needs to be present to ensure compatibility with json patch replace

tests/fixture/e2e/env.go

@@ -193,7 +193,15 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
 
 	// Once one or more nodes are running it should be safe to wait for promtail to report readiness
 	if flagVars.StartLogsCollector() {
-		require.NoError(tmpnet.WaitForPromtailReadiness(tc.DefaultContext(), tc.Log()))
+		runtimeConfig, err := flagVars.NodeRuntimeConfig()
+		require.NoError(err)
+		if runtimeConfig.Kube != nil {
+			// TODO(marun) Maybe make this configurable to enable the check for a test suite that writes service
+			// discovery configuration for its own metrics endpoint?
+			tc.Log().Warn("skipping check for logs collection readiness since kube nodes won't create have created the required service discovery config")
+		} else {
+			require.NoError(tmpnet.WaitForPromtailReadiness(tc.DefaultContext(), tc.Log()))
+		}
 	}
 
 	if networkCmd == StartNetworkCmd {

tests/fixture/e2e/flags.go

@@ -30,9 +30,7 @@ type FlagVars struct {
 	startNetwork     bool
 	startNetworkVars *flags.StartNetworkVars
 
-	// The collectors configured by these flags run as local processes
-	startMetricsCollector bool
-	startLogsCollector    bool
+	collectorVars *flags.CollectorVars
 
 	checkMetricsCollected bool
 	checkLogsCollected    bool
@@ -88,21 +86,19 @@ func (v *FlagVars) NodeRuntimeConfig() (*tmpnet.NodeRuntimeConfig, error) {
 }
 
 func (v *FlagVars) StartMetricsCollector() bool {
-	return v.startMetricsCollector
+	return v.collectorVars.StartMetricsCollector
 }
 
 func (v *FlagVars) StartLogsCollector() bool {
-	return v.startLogsCollector
+	return v.collectorVars.StartLogsCollector
 }
 
 func (v *FlagVars) CheckMetricsCollected() bool {
-	// TODO(marun) Enable this check for kube in a subsequent PR
-	return v.startNetworkVars.ProcessRuntimeConfigured() && v.checkMetricsCollected
+	return v.checkMetricsCollected
 }
 
 func (v *FlagVars) CheckLogsCollected() bool {
-	// TODO(marun) Enable this check for kube in a subsequent PR
-	return v.startNetworkVars.ProcessRuntimeConfigured() && v.checkLogsCollected
+	return v.checkLogsCollected
 }
 
 func (v *FlagVars) NetworkDir() string {
@@ -144,9 +140,9 @@ func RegisterFlagsWithDefaultOwner(defaultOwner string) *FlagVars {
 
 	vars.startNetworkVars = flags.NewStartNetworkFlagVars(defaultOwner)
 
-	SetMonitoringFlags(
-		&vars.startMetricsCollector,
-		&vars.startLogsCollector,
+	vars.collectorVars = flags.NewCollectorFlagVars()
+
+	SetCheckCollectionFlags(
 		&vars.checkMetricsCollected,
 		&vars.checkLogsCollected,
 	)
@@ -189,19 +185,7 @@ func RegisterFlagsWithDefaultOwner(defaultOwner string) *FlagVars {
 	return &vars
 }
 
-func SetMonitoringFlags(startMetricsCollector, startLogsCollector, checkMetricsCollected, checkLogsCollected *bool) {
-	flag.BoolVar(
-		startMetricsCollector,
-		"start-metrics-collector",
-		cast.ToBool(tmpnet.GetEnvWithDefault("TMPNET_START_METRICS_COLLECTOR", "false")),
-		"[optional] whether to start a local collector of metrics from nodes of the temporary network.",
-	)
-	flag.BoolVar(
-		startLogsCollector,
-		"start-logs-collector",
-		cast.ToBool(tmpnet.GetEnvWithDefault("TMPNET_START_LOGS_COLLECTOR", "false")),
-		"[optional] whether to start a local collector of logs from nodes of the temporary network.",
-	)
+func SetCheckCollectionFlags(checkMetricsCollected *bool, checkLogsCollected *bool) {
 	flag.BoolVar(
 		checkMetricsCollected,
 		"check-metrics-collected",

tests/fixture/tmpnet/README.md

@@ -59,11 +59,13 @@ the following non-test files:
 | Filename                    | Types          | Purpose                                                                |
 |:----------------------------|:---------------|:-----------------------------------------------------------------------|
 | flags/                      |                | Directory defining flags usable with both stdlib flags and spf13/pflag |
+| flags/collector.go          |                | Defines flags configuring collection of logs and metrics               |
 | flags/common.go             |                | Defines type definitions common across other files                     |
 | flags/process_runtime.go    |                | Defines flags configuring the process node runtime                     |
 | flags/runtime.go            |                | Defines flags configuring node runtime                                 |
 | flags/start_network.go      |                | Defines flags configuring network start                                |
 | tmpnetctl/                  |                | Directory containing main entrypoint for tmpnetctl command             |
+| yaml/                       |                | Directory defining kubernetes resources in yaml format                 |
 | check_monitoring.go         |                | Enables checking if logs and metrics were collected                    |
 | defaults.go                 |                | Defines common default configuration                                   |
 | detached_process_default.go |                | Configures detached processes for darwin and linux                     |
@@ -72,6 +74,7 @@ the following non-test files:
 | genesis.go                  |                | Creates test genesis                                                   |
 | kube.go                     |                | Library for Kubernetes interaction                                     |
 | local_network.go            |                | Defines configuration for the default local network                    |
+| monitor_kube.go             |                | Enables collection of logs and metrics from kube pods                  |
 | monitor_processes.go        |                | Enables collection of logs and metrics from local processes            |
 | network.go                  | Network        | Orchestrates and configures temporary networks                         |
 | network_config.go           | Network        | Reads and writes network configuration                                 |

tests/fixture/tmpnet/flags/collector.go

@@ -0,0 +1,48 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package flags
+
+import (
+	"flag"
+
+	"github.com/spf13/cast"
+	"github.com/spf13/pflag"
+
+	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
+)
+
+type CollectorVars struct {
+	StartMetricsCollector bool
+	StartLogsCollector    bool
+}
+
+// NewCollectorFlagVars registers collector flag variables for stdlib flag
+func NewCollectorFlagVars() *CollectorVars {
+	v := &CollectorVars{}
+	v.register(flag.BoolVar)
+	return v
+}
+
+// NewRuntimeConfigFlagSetVars registers collector flag variables for pflag
+func NewCollectorFlagSetVars(flagSet *pflag.FlagSet) *CollectorVars {
+	v := &CollectorVars{}
+	v.register(flagSet.BoolVar)
+	return v
+}
+
+func (v *CollectorVars) register(boolVar varFunc[bool]) {
+	boolVar(
+		&v.StartMetricsCollector,
+		"start-metrics-collector",
+		cast.ToBool(tmpnet.GetEnvWithDefault("TMPNET_START_METRICS_COLLECTOR", "false")),
+		"[optional] whether to start a local collector of metrics from nodes of the temporary network.",
+	)
+
+	boolVar(
+		&v.StartLogsCollector,
+		"start-logs-collector",
+		cast.ToBool(tmpnet.GetEnvWithDefault("TMPNET_START_LOGS_COLLECTOR", "false")),
+		"[optional] whether to start a local collector of logs from nodes of the temporary network.",
+	)
+}

tests/fixture/tmpnet/flags/start_network.go

@@ -59,10 +59,6 @@ func (v *StartNetworkVars) register(stringVar varFunc[string], intVar varFunc[in
 	)
 }
 
-func (v *StartNetworkVars) ProcessRuntimeConfigured() bool {
-	return v.runtimeVars.runtime == processRuntime
-}
-
 func (v *StartNetworkVars) GetNodeCount() (int, error) {
 	if v.nodeCount < 1 {
 		return 0, fmt.Errorf("--node-count must be greater than 0 but got %d", v.nodeCount)

tests/fixture/tmpnet/kube.go

@@ -59,13 +59,33 @@ func NewNodeStatefulSet(
 	volumeSize string,
 	volumeMountPath string,
 	flags FlagsMap,
+	labels map[string]string,
 ) *appsv1.StatefulSet {
 	objectMeta := metav1.ObjectMeta{}
 	if generateName {
 		objectMeta.GenerateName = name + "-"
 	} else {
 		objectMeta.Name = name
 	}
+
+	podAnnotations := map[string]string{
+		"prometheus.io/scrape": "true",
+		"prometheus.io/path":   "/ext/metrics",
+		"promtail/collect":     "true",
+	}
+
+	podLabels := map[string]string{
+		"app": name,
+	}
+	for label, value := range labels {
+		// These labels may contain values invalid for use in labels. Set them as annotations instead.
+		if label == "gh_repo" || label == "gh_workflow" {
+			podAnnotations[label] = value
+			continue
+		}
+		podLabels[label] = value
+	}
+
 	return &appsv1.StatefulSet{
 		ObjectMeta: objectMeta,
 		Spec: appsv1.StatefulSetSpec{
@@ -95,9 +115,8 @@ func NewNodeStatefulSet(
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						"app": name,
-					},
+					Labels:      podLabels,
+					Annotations: podAnnotations,
 				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{

tests/fixture/tmpnet/kube_runtime.go

@@ -262,6 +262,7 @@ func (p *KubeRuntime) Start(ctx context.Context) error {
 		fmt.Sprintf("%dGi", runtimeConfig.VolumeSizeGB),
 		volumeMountPath,
 		flags,
+		p.node.getMonitoringLabels(),
 	)
 
 	_, err = clientset.AppsV1().StatefulSets(runtimeConfig.Namespace).Create(