feat: added support for batching to loki write (#1754)

* added support for batching to loki write

* removed new test

* removed destination test

Author: Jayclifford345

charts/k8s-monitoring/destinations/loki-values.yaml

@@ -76,6 +76,14 @@ maxBackoffPeriod: 5m
 # @section -- General
 maxBackoffRetries: 10
 
+# -- Maximum batch size of logs to accumulate before sending.
+# @section -- General
+batchSize: ""
+
+# -- Maximum amount of time to wait before sending a batch.
+# @section -- General
+batchWait: ""
+
 auth:
   # -- The type of authentication to do.
   # Options are "none" (default), "basic", "bearerToken", "oauth2".

charts/k8s-monitoring/docs/destinations/loki.md

@@ -53,6 +53,8 @@ This defines the options for defining a destination for logs that use the Loki p
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| batchSize | string | `""` | Maximum batch size of logs to accumulate before sending. |
+| batchWait | string | `""` | Maximum amount of time to wait before sending a batch. |
 | clusterLabels | list | `["cluster","k8s.cluster.name"]` | Labels to be set with the cluster name as the value. |
 | extraHeaders | object | `{}` | Extra headers to be set when sending data. All values are treated as strings and automatically quoted. |
 | extraHeadersFrom | object | `{}` | Extra headers to be set when sending data through a dynamic reference. All values are treated as raw strings and not quoted. |

charts/k8s-monitoring/schema-mods/definitions/loki-destination.schema.json

@@ -86,6 +86,12 @@
                 }
             }
         },
+        "batchSize": {
+            "type": "string"
+        },
+        "batchWait": {
+            "type": "string"
+        },
         "clusterLabels": {
             "type": "array",
             "items": {

charts/k8s-monitoring/templates/destinations/_destination_loki.tpl

@@ -18,6 +18,12 @@ loki.write {{ include "helper.alloy_name" .name | quote }} {
 {{- else }}
     url = {{ .url | quote }} 
 {{- end }}
+{{- if .batchSize }}
+    batch_size = {{ .batchSize | quote }}
+{{- end }}
+{{- if .batchWait }}
+    batch_wait = {{ .batchWait | quote }}
+{{- end }}
 {{- if eq (include "secrets.usesSecret" (dict "object" . "key" "tenantId")) "true" }}
     tenant_id = {{ include "secrets.read" (dict "object" . "key" "tenantId" "nonsensitive" true) }}
 {{- end }}

charts/k8s-monitoring/tests/integration/pod-logs/default/.rendered/output.yaml

@@ -6,6 +6,8 @@ destinations:
   - name: localLoki
     type: loki
     url: http://loki.loki.svc:3100/loki/api/v1/push
+    batchSize: "1MiB"
+    batchWait: "1s"
     tenantId: "1"
     auth:
       type: basic

charts/k8s-monitoring/tests/integration/pod-logs/default/values.yaml

@@ -1356,6 +1356,12 @@
                         }
                     }
                 },
+                "batchSize": {
+                    "type": "string"
+                },
+                "batchWait": {
+                    "type": "string"
+                },
                 "clusterLabels": {
                     "type": "array",
                     "items": {