Add allowlist to secret filtering

charts/k8s-monitoring/charts/feature-pod-logs/README.md

@@ -82,6 +82,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| secretFilter.allowlist | list | `[]` | List of regular expressions to allowlist matching secrets. |
 | secretFilter.enabled | bool | `false` | Enable secret filtering. |
 | secretFilter.includeGeneric | bool | `false` | Include the generic API key rule. |
 | secretFilter.partialMask | int | `0` | Show the first N characters of the secret. |

charts/k8s-monitoring/charts/feature-pod-logs/schema-mods/types-and-enums.json

@@ -1,5 +1,6 @@
 {
   "properties": {
-    "gatherMethod": { "enum": ["volumes", "filelog", "kubernetesApi"] }
+    "gatherMethod": { "enum": ["volumes", "filelog", "kubernetesApi"]},
+    "secretFilter": { "properties": { "allowlist":  { "type": "array", "items": {"type": "string" }}}}
   }
 }

charts/k8s-monitoring/charts/feature-pod-logs/templates/_common_log_processing.alloy.tpl

@@ -87,6 +87,13 @@ loki.process "pod_logs" {
 loki.secretfilter "pod_logs" {
   include_generic = {{ .Values.secretFilter.includeGeneric }}
   partial_mask = {{ .Values.secretFilter.partialMask }}
+{{- if .Values.secretFilter.allowlist }}
+  allowlist = [
+  {{- range $value := .Values.secretFilter.allowlist }}
+    {{ $value | quote }},
+  {{- end }}
+  ]
+{{- end }}
 {{- end }}
   forward_to = argument.logs_destinations.value
 }

charts/k8s-monitoring/charts/feature-pod-logs/tests/__snapshot__/default_test.yaml.snap

@@ -713,6 +713,191 @@ should allow using the secret filter:
           forward_to = argument.logs_destinations.value
         }
       }
+should allow using the secret filter with an allowlist:
+  1: |
+    |-
+      declare "pod_logs" {
+        argument "logs_destinations" {
+          comment = "Must be a list of log destinations where collected logs should be forwarded to"
+        }
+
+        discovery.relabel "filtered_pods" {
+          targets = discovery.kubernetes.pods.targets
+          rule {
+            source_labels = ["__meta_kubernetes_namespace"]
+            action = "replace"
+            target_label = "namespace"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_pod_name"]
+            action = "replace"
+            target_label = "pod"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_pod_container_name"]
+            action = "replace"
+            target_label = "container"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
+            separator = "/"
+            action = "replace"
+            replacement = "$1"
+            target_label = "job"
+          }
+
+          // set the container runtime as a label
+          rule {
+            action = "replace"
+            source_labels = ["__meta_kubernetes_pod_container_id"]
+            regex = "^(\\S+):\\/\\/.+$"
+            replacement = "$1"
+            target_label = "tmp_container_runtime"
+          }
+
+          // make all labels on the pod available to the pipeline as labels,
+          // they are omitted before write to loki via stage.label_keep unless explicitly set
+          rule {
+            action = "labelmap"
+            regex = "__meta_kubernetes_pod_label_(.+)"
+          }
+
+          // make all annotations on the pod available to the pipeline as labels,
+          // they are omitted before write to loki via stage.label_keep unless explicitly set
+          rule {
+            action = "labelmap"
+            regex = "__meta_kubernetes_pod_annotation_(.+)"
+          }
+
+          // explicitly set service_name. if not set, loki will automatically try to populate a default.
+          // see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
+          //
+          // choose the first value found from the following ordered list:
+          // - pod.annotation[resource.opentelemetry.io/service.name]
+          // - pod.label[app.kubernetes.io/name]
+          // - k8s.pod.name
+          // - k8s.container.name
+          rule {
+            action = "replace"
+            source_labels = [
+              "__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name",
+              "__meta_kubernetes_pod_label_app_kubernetes_io_name",
+              "__meta_kubernetes_pod_container_name",
+            ]
+            separator = ";"
+            regex = "^(?:;*)?([^;]+).*$"
+            replacement = "$1"
+            target_label = "service_name"
+          }
+
+          // set resource attributes
+          rule {
+            action = "labelmap"
+            regex = "__meta_kubernetes_pod_annotation_resource_opentelemetry_io_(.+)"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_job"]
+            regex = "(.+)"
+            target_label = "job"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+            regex = "(.+)"
+            target_label = "app_kubernetes_io_name"
+          }
+        }
+
+        discovery.kubernetes "pods" {
+          role = "pod"
+          selectors {
+            role = "pod"
+            field = "spec.nodeName=" + sys.env("HOSTNAME")
+          }
+        }
+
+        discovery.relabel "filtered_pods_with_paths" {
+          targets = discovery.relabel.filtered_pods.output
+
+          rule {
+            source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
+            separator = "/"
+            action = "replace"
+            replacement = "/var/log/pods/*$1/*.log"
+            target_label = "__path__"
+          }
+        }
+
+        local.file_match "pod_logs" {
+          path_targets = discovery.relabel.filtered_pods_with_paths.output
+        }
+
+        loki.source.file "pod_logs" {
+          targets    = local.file_match.pod_logs.targets
+          forward_to = [loki.process.pod_logs.receiver]
+        }
+
+        loki.process "pod_logs" {
+          stage.match {
+            selector = "{tmp_container_runtime=~\"containerd|cri-o\"}"
+            // the cri processing stage extracts the following k/v pairs: log, stream, time, flags
+            stage.cri {}
+
+            // Set the extract flags and stream values as labels
+            stage.labels {
+              values = {
+                flags  = "",
+                stream  = "",
+              }
+            }
+          }
+
+          stage.match {
+            selector = "{tmp_container_runtime=\"docker\"}"
+            // the docker processing stage extracts the following k/v pairs: log, stream, time
+            stage.docker {}
+
+            // Set the extract stream value as a label
+            stage.labels {
+              values = {
+                stream  = "",
+              }
+            }
+          }
+
+          // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster,
+          // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary
+          // container runtime label as it is no longer needed.
+          stage.label_drop {
+            values = [
+              "filename",
+              "tmp_container_runtime",
+            ]
+          }
+          stage.structured_metadata {
+            values = {
+              "k8s_pod_name" = "k8s_pod_name",
+              "pod" = "pod",
+            }
+          }
+
+          // Only keep the labels that are defined in the `keepLabels` list.
+          stage.label_keep {
+            values = ["app_kubernetes_io_name","container","instance","job","level","namespace","service_name","service_namespace","deployment_environment","deployment_environment_name","k8s_namespace_name","k8s_deployment_name","k8s_statefulset_name","k8s_daemonset_name","k8s_cronjob_name","k8s_job_name","k8s_node_name"]
+          }
+
+          forward_to = [loki.secretfilter.pod_logs.receiver]
+        }
+
+        loki.secretfilter "pod_logs" {
+          include_generic = false
+          partial_mask = 3
+          allowlist = [
+            ".*secret.*",
+            ".*token.*",
+          ]
+          forward_to = argument.logs_destinations.value
+        }
+      }
 should render the default configuration:
   1: |
     |-

charts/k8s-monitoring/charts/feature-pod-logs/tests/default_test.yaml

@@ -58,3 +58,17 @@ tests:
           of: ConfigMap
       - matchSnapshot:
           path: data["module.alloy"]
+  - it: should allow using the secret filter with an allowlist
+    set:
+      deployAsConfigMap: true
+      secretFilter:
+        enabled: true
+        partialMask: 3
+        allowlist:
+          - ".*secret.*"
+          - ".*token.*"
+    asserts:
+      - isKind:
+          of: ConfigMap
+      - matchSnapshot:
+          path: data["module.alloy"]

charts/k8s-monitoring/charts/feature-pod-logs/values.schema.json

@@ -78,6 +78,12 @@
         "secretFilter": {
             "type": "object",
             "properties": {
+                "allowlist": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
                 "enabled": {
                     "type": "boolean"
                 },

charts/k8s-monitoring/charts/feature-pod-logs/values.yaml

@@ -89,6 +89,10 @@ secretFilter:
   # @section -- Secret Filtering
   partialMask: 0
 
+  # -- List of regular expressions to allowlist matching secrets.
+  # @section -- Secret Filtering
+  allowlist: []
+
 # -- Stage blocks to be added to the loki.process component for pod logs.
 # ([docs](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#blocks))
 # This value is templated so that you can refer to other values from this file.