[splunkhecexporter] Copy payload before sending to avoid corruption on reuse (#34507)

atoulme · web-flow · commit 52ac35ec4b80 · 2024-08-08T17:11:32.000-07:00
**Description:** This bug is a manifestation of golang/go#51907. Under high load, the pool of buffers used to send requests is reused enough that the same buffer is used concurrently to process data and be sent as request body. The fix is to copy the payload into a new byte array before sending it. **Link to tracking Issue:** Fixes #34357 This change is markedly bad for performance but I don't see any way to avoid it. ``` goos: darwin goarch: arm64 pkg: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter │ before.txt │ after.txt │ │ sec/op │ sec/op vs base │ _pushLogData_10_10_1024-10 85.52µ ± ∞ ¹ 107.67µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_8K-10 76.02µ ± ∞ ¹ 149.42µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_2M-10 74.12µ ± ∞ ¹ 108.30µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_200_2M-10 1.519m ± ∞ ¹ 2.519m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_2M-10 14.23m ± ∞ ¹ 18.18m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_5M-10 13.87m ± ∞ ¹ 17.96m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_1024-10 578.1µ ± ∞ ¹ 947.0µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_8K-10 134.7µ ± ∞ ¹ 229.3µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_2M-10 134.3µ ± ∞ ¹ 193.0µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_200_2M-10 3.043m ± ∞ ¹ 3.950m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_2M-10 31.01m ± ∞ ¹ 35.64m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_5M-10 31.23m ± ∞ ¹ 36.80m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024-10 159.8µ ± ∞ ¹ 176.7µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K-10 162.2µ ± ∞ ¹ 188.0µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M-10 328.0µ ± ∞ ¹ 520.0µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M-10 3.282m ± ∞ ¹ 3.725m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M-10 31.02m ± ∞ ¹ 37.29m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M-10 31.39m ± ∞ ¹ 37.30m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024-10 248.3µ ± ∞ ¹ 321.8µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K-10 246.9µ ± ∞ ¹ 316.1µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M-10 414.4µ ± ∞ ¹ 609.7µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M-10 4.918m ± ∞ ¹ 6.390m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M-10 46.97m ± ∞ ¹ 66.91m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M-10 47.10m ± ∞ ¹ 60.84m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024_MultiMetric-10 298.3µ ± ∞ ¹ 546.6µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K_MultiMetric-10 303.6µ ± ∞ ¹ 567.6µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M_MultiMetric-10 299.4µ ± ∞ ¹ 612.3µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M_MultiMetric-10 5.933m ± ∞ ¹ 8.737m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M_MultiMetric-10 57.02m ± ∞ ¹ 81.24m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M_MultiMetric-10 57.12m ± ∞ ¹ 77.35m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024_MultiMetric-10 383.8µ ± ∞ ¹ 625.9µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K_MultiMetric-10 399.6µ ± ∞ ¹ 570.0µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M_MultiMetric-10 392.9µ ± ∞ ¹ 644.6µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M_MultiMetric-10 7.560m ± ∞ ¹ 12.370m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M_MultiMetric-10 72.72m ± ∞ ¹ 100.91m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M_MultiMetric-10 81.60m ± ∞ ¹ 88.94m ± ∞ ¹ ~ (p=1.000 n=1) ² ConsumeLogsRejected-10 772.2µ ± ∞ ¹ 1041.0µ ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 1.930m 2.723m +41.10% ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 │ before.txt │ after.txt │ │ B/op │ B/op vs base │ _pushLogData_10_10_1024-10 73.12Ki ± ∞ ¹ 73.11Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_8K-10 63.79Ki ± ∞ ¹ 63.75Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_2M-10 63.19Ki ± ∞ ¹ 63.17Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_200_2M-10 1.254Mi ± ∞ ¹ 1.254Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_2M-10 12.59Mi ± ∞ ¹ 12.73Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_5M-10 12.65Mi ± ∞ ¹ 12.71Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_1024-10 2.411Mi ± ∞ ¹ 2.415Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_8K-10 65.22Ki ± ∞ ¹ 63.94Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_2M-10 65.26Ki ± ∞ ¹ 64.83Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_200_2M-10 1.253Mi ± ∞ ¹ 1.259Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_2M-10 12.54Mi ± ∞ ¹ 12.54Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_5M-10 12.54Mi ± ∞ ¹ 12.55Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024-10 81.56Ki ± ∞ ¹ 81.49Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K-10 88.50Ki ± ∞ ¹ 88.49Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M-10 2.081Mi ± ∞ ¹ 2.081Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M-10 3.563Mi ± ∞ ¹ 3.564Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M-10 19.76Mi ± ∞ ¹ 19.79Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M-10 25.76Mi ± ∞ ¹ 25.78Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024-10 83.96Ki ± ∞ ¹ 82.30Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K-10 89.35Ki ± ∞ ¹ 89.29Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M-10 2.081Mi ± ∞ ¹ 2.083Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M-10 3.564Mi ± ∞ ¹ 3.564Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M-10 17.69Mi ± ∞ ¹ 17.71Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M-10 20.70Mi ± ∞ ¹ 20.70Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024_MultiMetric-10 150.6Ki ± ∞ ¹ 150.5Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K_MultiMetric-10 150.5Ki ± ∞ ¹ 150.5Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M_MultiMetric-10 150.7Ki ± ∞ ¹ 150.5Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M_MultiMetric-10 2.936Mi ± ∞ ¹ 2.938Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M_MultiMetric-10 29.39Mi ± ∞ ¹ 29.45Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M_MultiMetric-10 29.38Mi ± ∞ ¹ 29.47Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024_MultiMetric-10 151.0Ki ± ∞ ¹ 150.8Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K_MultiMetric-10 151.0Ki ± ∞ ¹ 151.2Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M_MultiMetric-10 151.0Ki ± ∞ ¹ 151.3Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M_MultiMetric-10 2.936Mi ± ∞ ¹ 2.939Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M_MultiMetric-10 29.25Mi ± ∞ ¹ 29.27Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M_MultiMetric-10 29.26Mi ± ∞ ¹ 29.27Mi ± ∞ ¹ ~ (p=1.000 n=1) ² ConsumeLogsRejected-10 641.6Ki ± ∞ ¹ 640.9Ki ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 1.231Mi 1.230Mi -0.04% ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 │ before.txt │ after.txt │ │ allocs/op │ allocs/op vs base │ _pushLogData_10_10_1024-10 821.0 ± ∞ ¹ 821.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_8K-10 716.0 ± ∞ ¹ 716.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_2M-10 709.0 ± ∞ ¹ 709.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_200_2M-10 14.02k ± ∞ ¹ 14.02k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushLogData_100_200_2M-10 140.0k ± ∞ ¹ 140.0k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushLogData_100_200_5M-10 140.0k ± ∞ ¹ 140.0k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushLogData_compressed_10_10_1024-10 782.0 ± ∞ ¹ 783.0 ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushLogData_compressed_10_10_8K-10 709.0 ± ∞ ¹ 709.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_2M-10 709.0 ± ∞ ¹ 709.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_200_2M-10 14.02k ± ∞ ¹ 14.02k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushLogData_compressed_100_200_2M-10 140.0k ± ∞ ¹ 140.0k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushLogData_compressed_100_200_5M-10 140.0k ± ∞ ¹ 140.0k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_10_10_1024-10 1.410k ± ∞ ¹ 1.410k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K-10 1.410k ± ∞ ¹ 1.410k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M-10 1.417k ± ∞ ¹ 1.417k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M-10 28.02k ± ∞ ¹ 28.02k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_100_200_2M-10 280.1k ± ∞ ¹ 280.1k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_100_200_5M-10 280.1k ± ∞ ¹ 280.0k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_compressed_10_10_1024-10 1.410k ± ∞ ¹ 1.410k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K-10 1.410k ± ∞ ¹ 1.410k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M-10 1.418k ± ∞ ¹ 1.418k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M-10 28.02k ± ∞ ¹ 28.02k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_compressed_100_200_2M-10 280.0k ± ∞ ¹ 280.0k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_compressed_100_200_5M-10 280.0k ± ∞ ¹ 280.0k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_10_10_1024_MultiMetric-10 1.835k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K_MultiMetric-10 1.835k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M_MultiMetric-10 1.835k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M_MultiMetric-10 36.07k ± ∞ ¹ 36.07k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_100_200_2M_MultiMetric-10 360.4k ± ∞ ¹ 360.4k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_100_200_5M_MultiMetric-10 360.4k ± ∞ ¹ 360.4k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_compressed_10_10_1024_MultiMetric-10 1.835k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K_MultiMetric-10 1.835k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M_MultiMetric-10 1.835k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M_MultiMetric-10 36.07k ± ∞ ¹ 36.07k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_compressed_100_200_2M_MultiMetric-10 360.4k ± ∞ ¹ 360.4k ± ∞ ¹ ~ (p=1.000 n=1) ³ _pushMetricData_compressed_100_200_5M_MultiMetric-10 360.4k ± ∞ ¹ 360.4k ± ∞ ¹ ~ (p=1.000 n=1) ³ ConsumeLogsRejected-10 7.015k ± ∞ ¹ 7.014k ± ∞ ¹ ~ (p=1.000 n=1) ³ geomean 11.64k 11.64k +0.00% ¹ need >= 6 samples for confidence interval at level 0.95 ² all samples are equal ³ need >= 4 samples to detect a difference at alpha level 0.05 ```
diff --git a/.chloggen/safe_bytes.yaml b/.chloggen/safe_bytes.yaml
@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: splunkhecexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Copy the bytes to be placed in the request body to avoid corruption on reuse
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34357]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  This bug is a manifestation of https://github.com/golang/go/issues/51907.
+  Under high load, the pool of buffers used to send requests is reused enough
+  that the same buffer is used concurrently to process data and be sent as request body.
+  The fix is to copy the payload into a new byte array before sending it.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []
diff --git a/exporter/splunkhecexporter/buffer.go b/exporter/splunkhecexporter/buffer.go
@@ -22,6 +22,7 @@ type buffer interface {
 	Reset()
 	Len() int
 	Empty() bool
+	Bytes() []byte
 }
 
 type cancellableBytesWriter struct {
@@ -59,6 +60,10 @@ func (c *cancellableBytesWriter) Empty() bool {
 	return c.innerWriter.Len() == 0
 }
 
+func (c *cancellableBytesWriter) Bytes() []byte {
+	return c.innerWriter.Bytes()
+}
+
 type cancellableGzipWriter struct {
 	innerBuffer *bytes.Buffer
 	innerWriter *gzip.Writer
@@ -125,6 +130,10 @@ func (c *cancellableGzipWriter) Empty() bool {
 	return c.rawLen == 0
 }
 
+func (c *cancellableGzipWriter) Bytes() []byte {
+	return c.innerBuffer.Bytes()
+}
+
 // bufferPool is a pool of buffer objects.
 type bufferPool struct {
 	pool *sync.Pool
diff --git a/exporter/splunkhecexporter/hec_worker.go b/exporter/splunkhecexporter/hec_worker.go
@@ -4,6 +4,7 @@
 package splunkhecexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter"
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
@@ -27,7 +28,11 @@ type defaultHecWorker struct {
 }
 
 func (hec *defaultHecWorker) send(ctx context.Context, buf buffer, headers map[string]string) error {
-	req, err := http.NewRequestWithContext(ctx, "POST", hec.url.String(), buf)
+	// We copy the bytes to a new buffer to avoid corruption. This is a workaround to avoid hitting https://github.com/golang/go/issues/51907.
+	nb := make([]byte, buf.Len())
+	copy(nb, buf.Bytes())
+	bodyBuf := bytes.NewReader(nb)
+	req, err := http.NewRequestWithContext(ctx, "POST", hec.url.String(), bodyBuf)
 	if err != nil {
 		return consumererror.NewPermanent(err)
 	}
