Merge #2191: RUM-8042 Add pending batches to batch deleted metric

Co-authored-by: maxep <[email protected]>

Author: dd-mergequeue[bot]

DatadogCore/Sources/Core/Storage/Files/Directory.swift

@@ -129,6 +129,13 @@ internal struct Directory: DirectoryProtocol {
         return File(url: fileURL)
     }
 
+    /// Returns count of files in this directory.
+    func filesCount() throws -> Int {
+        return try FileManager.default
+            .contentsOfDirectory(at: url, includingPropertiesForKeys: [.isRegularFileKey])
+            .count
+    }
+
     /// Returns all files of this directory.
     func files() throws -> [File] {
         return try FileManager.default

DatadogCore/Sources/Core/Storage/FilesOrchestrator.swift

@@ -120,6 +120,7 @@ internal class FilesOrchestrator: FilesOrchestratorType {
         lastWritableFileObjectsCount = 1
         lastWritableFileApproximatedSize = writeSize
         lastWritableFileLastWriteDate = dateProvider.now
+
         return newFile
     }
 
@@ -175,8 +176,7 @@ internal class FilesOrchestrator: FilesOrchestratorType {
     func getReadableFiles(excludingFilesNamed excludedFileNames: Set<String> = [], limit: Int = .max) -> [ReadableFile] {
         do {
             let filesFromOldest = try directory.files()
-                .map { (file: $0, fileCreationDate: fileCreationDateFrom(fileName: $0.name)) }
-                .compactMap { try deleteFileIfItsObsolete(file: $0.file, fileCreationDate: $0.fileCreationDate) }
+                .compactMap { try deleteFileIfItsObsolete(file: $0, fileCreationDate: fileCreationDateFrom(fileName: $0.name)) }
                 .sorted(by: { $0.fileCreationDate < $1.fileCreationDate })
 
             if ignoreFilesAgeWhenReading {
@@ -185,12 +185,11 @@ internal class FilesOrchestrator: FilesOrchestratorType {
                     .map { $0.file }
             }
 
-            let filtered = filesFromOldest
+            return filesFromOldest
                 .filter {
-                    let fileAge = dateProvider.now.timeIntervalSince($0.fileCreationDate)
-                    return excludedFileNames.contains($0.file.name) == false && fileAge >= performance.minFileAgeForRead
+                    !excludedFileNames.contains($0.file.name) &&
+                    dateProvider.now.timeIntervalSince($0.fileCreationDate) >= performance.minFileAgeForRead
                 }
-            return filtered
                 .prefix(limit)
                 .map { $0.file }
         } catch {
@@ -278,7 +277,8 @@ internal class FilesOrchestrator: FilesOrchestratorType {
                 BatchDeletedMetric.batchAgeKey: batchAge.toMilliseconds,
                 BatchDeletedMetric.batchRemovalReasonKey: deletionReason.toString(),
                 BatchDeletedMetric.inBackgroundKey: false,
-                BatchDeletedMetric.backgroundTasksEnabled: metricsData.backgroundTasksEnabled
+                BatchDeletedMetric.backgroundTasksEnabled: metricsData.backgroundTasksEnabled,
+                BatchDeletedMetric.pendingBatches: try? directory.filesCount()
             ],
             sampleRate: BatchDeletedMetric.sampleRate
         )

DatadogCore/Sources/SDKMetrics/BatchMetrics.swift

@@ -59,6 +59,9 @@ internal enum BatchDeletedMetric {
     /// If the background tasks were enabled.
     static let backgroundTasksEnabled = "background_tasks_enabled"
 
+    /// Count of pending batches left on track.
+    static let pendingBatches = "pending_batches"
+
     /// Allowed values for `batchRemovalReasonKey`.
     enum RemovalReason {
         /// The batch was delivered to Intake and deleted upon receiving given `responseCode`.

DatadogCore/Tests/Datadog/Core/Persistence/FilesOrchestrator+MetricsTests.swift

@@ -49,17 +49,22 @@ class FilesOrchestrator_MetricsTests: XCTestCase {
     func testWhenReadableFileIsDeleted_itSendsBatchDeletedMetric() throws {
         // Given
         let orchestrator = createOrchestrator()
-        let file = try XCTUnwrap(orchestrator.getWritableFile(writeSize: 1) as? ReadableFile)
         let expectedBatchAge = storage.minFileAgeForRead + 1
 
-        // When:
-        // - wait and delete the file
+        // When: create 1 batch file
+        _ = try orchestrator.getWritableFile(writeSize: 1)
+
+        // When: wait and create 2nd batch file
+        dateProvider.advance(bySeconds: expectedBatchAge)
+        let file = try XCTUnwrap(orchestrator.getWritableFile(writeSize: 1) as? ReadableFile)
+
+        // When: wait and delete one
         dateProvider.advance(bySeconds: expectedBatchAge)
         orchestrator.delete(readableFile: file, deletionReason: .intakeCode(responseCode: 202))
 
         // Then
         let metric = try XCTUnwrap(telemetry.messages.firstMetric(named: "Batch Deleted"))
-        DDAssertReflectionEqual(metric.attributes, [
+        DDAssertJSONEqual(metric.attributes, [
             "metric_type": "batch deleted",
             "track": "track name",
             "consent": "consent value",
@@ -72,6 +77,7 @@ class FilesOrchestrator_MetricsTests: XCTestCase {
             "background_tasks_enabled": false,
             "batch_age": expectedBatchAge.toMilliseconds,
             "batch_removal_reason": "intake-code-202",
+            "pending_batches": 1
         ])
         XCTAssertEqual(metric.sampleRate, BatchDeletedMetric.sampleRate)
     }
@@ -90,7 +96,7 @@ class FilesOrchestrator_MetricsTests: XCTestCase {
 
         // Then
         let metric = try XCTUnwrap(telemetry.messages.firstMetric(named: "Batch Deleted"))
-        DDAssertReflectionEqual(metric.attributes, [
+        DDAssertJSONEqual(metric.attributes, [
             "metric_type": "batch deleted",
             "track": "track name",
             "consent": "consent value",
@@ -103,6 +109,7 @@ class FilesOrchestrator_MetricsTests: XCTestCase {
             "background_tasks_enabled": false,
             "batch_age": (storage.maxFileAgeForRead + 1).toMilliseconds,
             "batch_removal_reason": "obsolete",
+            "pending_batches": 0
         ])
         XCTAssertEqual(metric.sampleRate, BatchDeletedMetric.sampleRate)
     }
@@ -124,7 +131,7 @@ class FilesOrchestrator_MetricsTests: XCTestCase {
 
         // Then
         let metric = try XCTUnwrap(telemetry.messages.firstMetric(named: "Batch Deleted"))
-        DDAssertReflectionEqual(metric.attributes, [
+        DDAssertJSONEqual(metric.attributes, [
             "metric_type": "batch deleted",
             "track": "track name",
             "consent": "consent value",
@@ -137,6 +144,7 @@ class FilesOrchestrator_MetricsTests: XCTestCase {
             "background_tasks_enabled": false,
             "batch_age": expectedBatchAge.toMilliseconds,
             "batch_removal_reason": "purged",
+            "pending_batches": 0
         ])
         XCTAssertEqual(metric.sampleRate, BatchDeletedMetric.sampleRate)
     }