Added file size storage limits

Examples/apps/DestinationsExample/DestinationsExample/AppDelegate.swift

@@ -86,7 +86,7 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
 
     // This functionality is needed to forward deep link attribution data with AppsFlyer
     // Report Push Notification attribution data for re-engagements
-    func application(_ application: UIApplication, didReceiveRemoteNotification userInfo: [String : Any], fetchCompletionHandler completionHandler: @escaping (UIBackgroundFetchResult) -> Void) {
+    func application(_ application: UIApplication, didReceiveRemoteNotification userInfo: [AnyHashable : Any], fetchCompletionHandler completionHandler: @escaping (UIBackgroundFetchResult) -> Void) {
         // this enables remote notifications for various destinations (appsflyer)
         analytics?.receivedRemoteNotification(userInfo: userInfo)
     }

Sources/Segment/Plugins/Platforms/iOS/iOSDelegation.swift

@@ -16,14 +16,14 @@ import UIKit
 public protocol RemoteNotifications: Plugin {
     func registeredForRemoteNotifications(deviceToken: Data)
     func failedToRegisterForRemoteNotification(error: Error?)
-    func receivedRemoteNotification(userInfo: [String: Any])
+    func receivedRemoteNotification(userInfo: [AnyHashable: Any])
     func handleAction(identifier: String, userInfo: [String: Any])
 }
 
 extension RemoteNotifications {
     public func registeredForRemoteNotifications(deviceToken: Data) {}
     public func failedToRegisterForRemoteNotification(error: Error?) {}
-    public func receivedRemoteNotification(userInfo: [String: Any]) {}
+    public func receivedRemoteNotification(userInfo: [AnyHashable: Any]) {}
     public func handleAction(identifier: String, userInfo: [String: Any]) {}
 }
 
@@ -46,7 +46,7 @@ extension Analytics {
         }
     }
 
-    public func receivedRemoteNotification(userInfo: [String: Any]) {
+    public func receivedRemoteNotification(userInfo: [AnyHashable: Any]) {
         apply { plugin in
             if let p = plugin as? RemoteNotifications {
                 p.receivedRemoteNotification(userInfo: userInfo)

Sources/Segment/Plugins/Platforms/watchOS/watchOSDelegation.swift

@@ -15,13 +15,13 @@ import WatchKit
 public protocol RemoteNotifications: Plugin {
     func registeredForRemoteNotifications(deviceToken: Data)
     func failedToRegisterForRemoteNotification(error: Error?)
-    func receivedRemoteNotification(userInfo: [String: Any])
+    func receivedRemoteNotification(userInfo: [AnyHashable: Any])
 }
 
 extension RemoteNotifications {
     public func registeredForRemoteNotifications(deviceToken: Data) {}
     public func failedToRegisterForRemoteNotification(error: Error?) {}
-    public func receivedRemoteNotification(userInfo: [String: Any]) {}
+    public func receivedRemoteNotification(userInfo: [AnyHashable: Any]) {}
 }
 
 extension Analytics {
@@ -43,7 +43,7 @@ extension Analytics {
         }
     }
 
-    public func receivedRemoteNotification(userInfo: [String: Any]) {
+    public func receivedRemoteNotification(userInfo: [AnyHashable: Any]) {
         apply { plugin in
             if let p = plugin as? RemoteNotifications {
                 p.receivedRemoteNotification(userInfo: userInfo)

Sources/Segment/Plugins/SegmentDestination.swift

@@ -37,7 +37,6 @@ public class SegmentDestination: DestinationPlugin {
     private var httpClient: HTTPClient?
     private var uploads = [UploadTaskInfo]()
     private var storage: Storage?
-    private var maxPayloadSize = 500000 // Max 500kb
 
     private var apiKey: String? = nil
     private var apiHost: String? = nil
@@ -139,21 +138,19 @@ public class SegmentDestination: DestinationPlugin {
             for url in data {
                 analytics.log(message: "Processing Batch:\n\(url.lastPathComponent)")
 
-                if isPayloadSizeAcceptable(url: url) {
-                    let uploadTask = httpClient.startBatchUpload(writeKey: analytics.configuration.values.writeKey, batch: url) { (result) in
-                        switch result {
+                let uploadTask = httpClient.startBatchUpload(writeKey: analytics.configuration.values.writeKey, batch: url) { (result) in
+                    switch result {
                         case .success(_):
                             storage.remove(file: url)
                         default:
                             analytics.logFlush()
-                        }
-
-                        analytics.log(message: "Processed: \(url.lastPathComponent)")
-                    }
-                    // we have a legit upload in progress now, so add it to our list.
-                    if let upload = uploadTask {
-                        add(uploadTask: UploadTaskInfo(url: url, task: upload))
                     }
+
+                    analytics.log(message: "Processed: \(url.lastPathComponent)")
+                }
+                // we have a legit upload in progress now, so add it to our list.
+                if let upload = uploadTask {
+                    add(uploadTask: UploadTaskInfo(url: url, task: upload))
                 }
             }
         } else {
@@ -190,30 +187,4 @@ extension SegmentDestination {
     internal func add(uploadTask: UploadTaskInfo) {
         uploads.append(uploadTask)
     }
-
-    internal func isPayloadSizeAcceptable(url: URL) -> Bool {
-        var result = true
-        var fileSizeTotal: Int64 = 0
-
-        // Make sure we're under the max payload size.
-        do {
-            let attributes = try FileManager.default.attributesOfItem(atPath: url.path)
-            guard let fileSize = attributes[FileAttributeKey.size] as? Int64 else {
-                analytics?.log(message: "File size could not be read")
-                // none of the logic beyond here will work if we can't get the
-                // filesize so assume everything is good and hope for the best.
-                return true
-            }
-            fileSizeTotal += fileSize
-        } catch {
-            analytics?.log(message: "Could not read file attributes")
-        }
-
-        if fileSizeTotal >= maxPayloadSize {
-            analytics?.log(message: "Batch file is too large to be sent")
-            result = false
-        }
-        return result
-    }
-
 }

Sources/Segment/Utilities/Storage.swift

@@ -13,6 +13,7 @@ internal class Storage: Subscriber {
     let writeKey: String
     let syncQueue = DispatchQueue(label: "storage.segment.com")
     let userDefaults: UserDefaults?
+    static let MAXFILESIZE = 475000     // Server accepts max 500k per batch
 
     init(store: Store, writeKey: String) {
         self.store = store
@@ -44,14 +45,8 @@ extension Storage {
         switch key {
         case .events:
             if let event = value as? RawEvent {
-                // this is synchronized against finish(file:) down below.
-                var currentFile = 0
-                syncQueue.sync {
-                    let index: Int = userDefaults?.integer(forKey: key.rawValue) ?? 0
-                    userDefaults?.set(index, forKey: key.rawValue)
-                    currentFile = index
-                }
-                self.storeEvent(toFile: self.eventsFile(index: currentFile), event: event)
+                let eventStoreFile = currentFile(key)
+                self.storeEvent(toFile: eventStoreFile, event: event)
             }
             break
         default:
@@ -137,6 +132,16 @@ extension Storage {
         }
         return result
     }
+
+    func currentFile(_ key: Storage.Constants) -> URL {
+        var currentFile = 0
+        syncQueue.sync {
+            let index: Int = userDefaults?.integer(forKey: key.rawValue) ?? 0
+            userDefaults?.set(index, forKey: key.rawValue)
+            currentFile = index
+        }
+        return self.eventsFile(index: currentFile)
+    }
 }
 
 // MARK: - State Subscriptions
@@ -212,19 +217,34 @@ extension Storage {
 // MARK: - Event Storage
 
 extension Storage {
+
     func storeEvent(toFile file: URL, event: RawEvent) {
+
+        var storeFile = file
+
         let fm = FileManager.default
         var newFile = false
-        if fm.fileExists(atPath: file.path) == false {
-            start(file: file)
+        if fm.fileExists(atPath: storeFile.path) == false {
+            start(file: storeFile)
+            newFile = true
+        }
+
+        // Verify file size isn't too large
+        if let attributes = try? fm.attributesOfItem(atPath: storeFile.path),
+           let fileSize = attributes[FileAttributeKey.size] as? UInt64,
+           fileSize >= Storage.MAXFILESIZE {
+            finish(file: storeFile)
+            // Set the new file path
+            storeFile = currentFile(.events)
+            start(file: storeFile)
             newFile = true
         }
 
         syncQueue.sync {
             do {
                 let jsonString = event.toString()
                 if let jsonData = jsonString.data(using: .utf8) {
-                    let handle = try FileHandle(forWritingTo: file)
+                    let handle = try FileHandle(forWritingTo: storeFile)
                     handle.seekToEndOfFile()
                     // prepare for the next entry
                     if newFile == false {
@@ -237,7 +257,7 @@ extension Storage {
                     assert(false, "Storage: Unable to convert event to json!")
                 }
             } catch {
-                assert(false, "Storage: failed to write event to \(file), error: \(error)")
+                assert(false, "Storage: failed to write event to \(storeFile), error: \(error)")
             }
         }
     }