amplitude · justin-fiedler · Jul 19, 2023 · Jun 30, 2023 · Jul 2, 2023 · Jul 2, 2023
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: macos-12
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Set Xcode 14
         run: |
           sudo xcode-select -switch /Applications/Xcode_14.1.app

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -27,7 +27,7 @@ jobs:
     needs: [authorize]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Set Xcode 14
         run: |
           sudo xcode-select -switch /Applications/Xcode_14.1.app

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: macos-12
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Set Xcode 14
         run: |
           sudo xcode-select -switch /Applications/Xcode_14.1.app

diff --git a/Amplitude-Swift.xcodeproj/project.pbxproj b/Amplitude-Swift.xcodeproj/project.pbxproj
@@ -22,10 +22,16 @@
 
 /* Begin PBXBuildFile section */
 		8EDEC14255F82E24CEE00B36 /* AmplitudeSessionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDEC0630C3B587334275D9B /* AmplitudeSessionTests.swift */; };
+		8EDEC2E0CC80DF79F5463ACC /* RemnantDataMigrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDEC6A9899998F823C278F7 /* RemnantDataMigrationTests.swift */; };
 		8EDEC4EE0DE1C89889F451B5 /* QueueTimeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDEC4F83BFAA664749FAEF0 /* QueueTimeTests.swift */; };
 		8EDEC972AEB33E4528F7FEEB /* StoragePrefixMigrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDEC9B98272069D70D08EA4 /* StoragePrefixMigrationTests.swift */; };
 		8EDECD602E181B3E2E85D4DF /* StoragePrefixMigration.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDECC2335BCF4C2EC3A6206 /* StoragePrefixMigration.swift */; };
+		8EDEC8F8DD2CDCD6568512F8 /* RemnantDataMigration.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDEC19F9FBC98A0D4E5A513 /* RemnantDataMigration.swift */; };
 		8EDECFCCF4219767F26210D6 /* Sessions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDEC2B8B38E04CDB51F0E83 /* Sessions.swift */; };
+		BA0359CA2A51585D007C383B /* legacy_v3.sqlite in Resources */ = {isa = PBXBuildFile; fileRef = BA0359C92A51585D007C383B /* legacy_v3.sqlite */; };
+		BA0639F62A4DD491000F1CEE /* LegacyDatabaseStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA0639F52A4DD491000F1CEE /* LegacyDatabaseStorage.swift */; };
+		BA994B9A2A4F48DE00D0913F /* LegacyDatabaseStorageTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA994B992A4F48DE00D0913F /* LegacyDatabaseStorageTests.swift */; };
+		BA994B9D2A4F4FCB00D0913F /* legacy_v4.sqlite in Resources */ = {isa = PBXBuildFile; fileRef = BA994B9B2A4F4B7500D0913F /* legacy_v4.sqlite */; };
 		BA9BEA4B299FB43B00BC0F7C /* IdentifyInterceptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA9BEA4A299FB43B00BC0F7C /* IdentifyInterceptor.swift */; };
 		BA9BEA4D299FB4BB00BC0F7C /* IdentifyInterceptorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA9BEA4C299FB4BB00BC0F7C /* IdentifyInterceptorTests.swift */; };
 		OBJ_100 /* Mediator.swift in Sources */ = {isa = PBXBuildFile; fileRef = OBJ_21 /* Mediator.swift */; };
@@ -104,10 +110,16 @@
 
 /* Begin PBXFileReference section */
 		8EDEC0630C3B587334275D9B /* AmplitudeSessionTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AmplitudeSessionTests.swift; sourceTree = "<group>"; };
+		8EDEC19F9FBC98A0D4E5A513 /* RemnantDataMigration.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RemnantDataMigration.swift; sourceTree = "<group>"; };
 		8EDEC2B8B38E04CDB51F0E83 /* Sessions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Sessions.swift; sourceTree = "<group>"; };
 		8EDEC4F83BFAA664749FAEF0 /* QueueTimeTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = QueueTimeTests.swift; sourceTree = "<group>"; };
 		8EDEC9B98272069D70D08EA4 /* StoragePrefixMigrationTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = StoragePrefixMigrationTests.swift; sourceTree = "<group>"; };
 		8EDECC2335BCF4C2EC3A6206 /* StoragePrefixMigration.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = StoragePrefixMigration.swift; sourceTree = "<group>"; };
+		8EDEC6A9899998F823C278F7 /* RemnantDataMigrationTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RemnantDataMigrationTests.swift; sourceTree = "<group>"; };
+		BA0359C92A51585D007C383B /* legacy_v3.sqlite */ = {isa = PBXFileReference; lastKnownFileType = file; path = legacy_v3.sqlite; sourceTree = "<group>"; };
+		BA0639F52A4DD491000F1CEE /* LegacyDatabaseStorage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LegacyDatabaseStorage.swift; sourceTree = "<group>"; };
+		BA994B992A4F48DE00D0913F /* LegacyDatabaseStorageTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LegacyDatabaseStorageTests.swift; sourceTree = "<group>"; };
+		BA994B9B2A4F4B7500D0913F /* legacy_v4.sqlite */ = {isa = PBXFileReference; lastKnownFileType = file; path = legacy_v4.sqlite; sourceTree = "<group>"; };
 		BA9BEA4A299FB43B00BC0F7C /* IdentifyInterceptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IdentifyInterceptor.swift; sourceTree = "<group>"; };
 		BA9BEA4C299FB4BB00BC0F7C /* IdentifyInterceptorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IdentifyInterceptorTests.swift; sourceTree = "<group>"; };
 		OBJ_10 /* ConsoleLogger.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConsoleLogger.swift; sourceTree = "<group>"; };
@@ -196,6 +208,8 @@
 			isa = PBXGroup;
 			children = (
 				8EDECC2335BCF4C2EC3A6206 /* StoragePrefixMigration.swift */,
+				BA0639F52A4DD491000F1CEE /* LegacyDatabaseStorage.swift */,
+				8EDEC19F9FBC98A0D4E5A513 /* RemnantDataMigration.swift */,
 			);
 			path = Migration;
 			sourceTree = "<group>";
@@ -204,6 +218,10 @@
 			isa = PBXGroup;
 			children = (
 				8EDEC9B98272069D70D08EA4 /* StoragePrefixMigrationTests.swift */,
+				BA0359C92A51585D007C383B /* legacy_v3.sqlite */,
+				BA994B9B2A4F4B7500D0913F /* legacy_v4.sqlite */,
+				BA994B992A4F48DE00D0913F /* LegacyDatabaseStorageTests.swift */,
+				8EDEC6A9899998F823C278F7 /* RemnantDataMigrationTests.swift */,
 			);
 			path = Migration;
 			sourceTree = "<group>";
@@ -430,6 +448,7 @@
 			buildPhases = (
 				OBJ_141 /* Sources */,
 				OBJ_159 /* Frameworks */,
+				BA994B9C2A4F4FBE00D0913F /* Resources */,
 			);
 			buildRules = (
 			);
@@ -484,6 +503,18 @@
 		};
 /* End PBXProject section */
 
+/* Begin PBXResourcesBuildPhase section */
+		BA994B9C2A4F4FBE00D0913F /* Resources */ = {
+			isa = PBXResourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				BA994B9D2A4F4FCB00D0913F /* legacy_v4.sqlite in Resources */,
+				BA0359CA2A51585D007C383B /* legacy_v3.sqlite in Resources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXResourcesBuildPhase section */
+
 /* Begin PBXSourcesBuildPhase section */
 		OBJ_130 /* Sources */ = {
 			isa = PBXSourcesBuildPhase;
@@ -518,6 +549,8 @@
 				8EDEC4EE0DE1C89889F451B5 /* QueueTimeTests.swift in Sources */,
 				8EDEC14255F82E24CEE00B36 /* AmplitudeSessionTests.swift in Sources */,
 				8EDEC972AEB33E4528F7FEEB /* StoragePrefixMigrationTests.swift in Sources */,
+				BA994B9A2A4F48DE00D0913F /* LegacyDatabaseStorageTests.swift in Sources */,
+				8EDEC2E0CC80DF79F5463ACC /* RemnantDataMigrationTests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -551,6 +584,7 @@
 				OBJ_111 /* InMemoryStorage.swift in Sources */,
 				OBJ_112 /* PersistentStorage.swift in Sources */,
 				BA9BEA4B299FB43B00BC0F7C /* IdentifyInterceptor.swift in Sources */,
+				BA0639F62A4DD491000F1CEE /* LegacyDatabaseStorage.swift in Sources */,
 				OBJ_113 /* Timeline.swift in Sources */,
 				OBJ_114 /* TrackingOptions.swift in Sources */,
 				OBJ_115 /* Types.swift in Sources */,
@@ -564,6 +598,7 @@
 				OBJ_124 /* UrlExtension.swift in Sources */,
 				8EDECFCCF4219767F26210D6 /* Sessions.swift in Sources */,
 				8EDECD602E181B3E2E85D4DF /* StoragePrefixMigration.swift in Sources */,
+				8EDEC8F8DD2CDCD6568512F8 /* RemnantDataMigration.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

diff --git a/Sources/Amplitude/Amplitude.swift b/Sources/Amplitude/Amplitude.swift
@@ -25,6 +25,10 @@ public class Amplitude {
         return self.configuration.storageProvider
     }()
 
+    lazy var identifyStorage: any Storage = {
+        return self.configuration.identifyStorageProvider
+    }()
+
     lazy var timeline: Timeline = {
         return Timeline()
     }()
@@ -47,6 +51,10 @@ public class Amplitude {
 
         migrateApiKeyStorages()
 
+        if configuration.migrateLegacyData {
+            RemnantDataMigration(self).execute()
+        }
+
         if let deviceId: String? = configuration.storageProvider.read(key: .DEVICE_ID) {
             state.deviceId = deviceId
         }

diff --git a/Sources/Amplitude/Configuration.swift b/Sources/Amplitude/Configuration.swift
@@ -32,6 +32,7 @@ public class Configuration {
     public var minTimeBetweenSessionsMillis: Int
     public var trackingSessionEvents: Bool?
     public var identifyBatchIntervalMillis: Int
+    public let migrateLegacyData: Bool
 
     public init(
         apiKey: String,
@@ -57,7 +58,8 @@ public class Configuration {
         flushEventsOnClose: Bool = true,
         minTimeBetweenSessionsMillis: Int = Constants.Configuration.MIN_TIME_BETWEEN_SESSIONS_MILLIS,
         trackingSessionEvents: Bool = true,
-        identifyBatchIntervalMillis: Int = Constants.Configuration.IDENTIFY_BATCH_INTERVAL_MILLIS
+        identifyBatchIntervalMillis: Int = Constants.Configuration.IDENTIFY_BATCH_INTERVAL_MILLIS,
+        migrateLegacyData: Bool = true
     ) {
         self.apiKey = apiKey
         self.flushQueueSize = flushQueueSize
@@ -85,6 +87,7 @@ public class Configuration {
         self.minTimeBetweenSessionsMillis = minTimeBetweenSessionsMillis
         self.trackingSessionEvents = trackingSessionEvents
         self.identifyBatchIntervalMillis = identifyBatchIntervalMillis
+        self.migrateLegacyData = migrateLegacyData
         // Logging is OFF by default
         self.loggerProvider.logLevel = logLevel.rawValue
     }

diff --git a/Sources/Amplitude/Migration/LegacyDatabaseStorage.swift b/Sources/Amplitude/Migration/LegacyDatabaseStorage.swift
@@ -0,0 +1,207 @@
+import Foundation
+import SQLite3
+
+class LegacyDatabaseStorage {
+    private static let DATABASE_NAME = "com.amplitude.database"
+    private static let EVENT_TABLE_NAME = "events"
+    private static let IDENTIFY_TABLE_NAME = "identifys"
+    private static let INTERCEPTED_IDENTIFY_TABLE_NAME = "intercepted_identifys"
+    private static let STORE_TABLE_NAME = "store"
+    private static let LONG_STORE_TABLE_NAME = "long_store"
+    private static let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self)
+
+    private static var instances: [String: LegacyDatabaseStorage] = [:]
+    private static let instanceQueue = DispatchQueue(label: "legacyDatabaseStorage.amplitude.com")
+
+    let databasePath: String
+    let logger: (any Logger)?
+
+    public static func getStorage(_ instanceName: String, _ logger: (any Logger)?) -> LegacyDatabaseStorage {
+        instanceQueue.sync {
+            var normalizedInstanceName = instanceName.lowercased()
+            if normalizedInstanceName == Constants.Configuration.DEFAULT_INSTANCE {
+                normalizedInstanceName = ""
+            }
+            if let storage = instances[normalizedInstanceName] {
+                return storage
+            }
+            let storage = LegacyDatabaseStorage(getDatabasePath(normalizedInstanceName).path, logger)
+            instances[normalizedInstanceName] = storage
+            return storage
+        }
+    }
+
+    static func getDatabasePath(_ instanceName: String) -> URL {
+        #if os(tvOS)
+        let searchPathDirectory = FileManager.SearchPathDirectory.cachesDirectory
+        #else
+        let searchPathDirectory = FileManager.SearchPathDirectory.libraryDirectory
+        #endif
+
+        let urls = FileManager.default.urls(for: searchPathDirectory, in: .userDomainMask)
+        var databaseUrl = urls[0]
+
+        var databaseName = DATABASE_NAME
+        if instanceName != "" {
+            databaseName += "_\(instanceName)"
+        }
+        databaseUrl.appendPathComponent(databaseName)
+        return databaseUrl
+    }
+
+    public init(_ databasePath: String, _ logger: (any Logger)?) {
+        self.databasePath = databasePath
+        self.logger = logger
+    }
+
+    func getValue(_ key: String) -> String? {
+        getValueFromTable(LegacyDatabaseStorage.STORE_TABLE_NAME, key) as? String
+    }
+
+    func getLongValue(_ key: String) -> Int64? {
+        getValueFromTable(LegacyDatabaseStorage.LONG_STORE_TABLE_NAME, key) as? Int64
+    }
+
+    private func getValueFromTable(_ table: String, _ key: String) -> Any? {
+        let query = "SELECT key, value FROM \(table) WHERE key = ?;"
+        return executeQuery(query) { stmt in
+            let bindResult = sqlite3_bind_text(stmt, 1, key, -1, LegacyDatabaseStorage.SQLITE_TRANSIENT)
+            if bindResult != SQLITE_OK {
+                logger?.error(message: "bind query parameter failed with result: \(bindResult)")
+                return
+            }
+
+            let stepResult = sqlite3_step(stmt)
+            if stepResult != SQLITE_ROW {
+                logger?.error(message: "execute query '\(query)' failed with result: \(stepResult)")
+                return
+            }
+
+            if sqlite3_column_type(stmt, 1) != SQLITE_NULL {
+                if table == LegacyDatabaseStorage.STORE_TABLE_NAME {
+                    guard let rawValue = sqlite3_column_text(stmt, 1) else {
+                        return
+                    }
+                    return String(cString: rawValue)
+                } else {
+                    return sqlite3_column_int64(stmt, 1)
+                }
+            } else {
+                return
+            }
+        }
+    }
+
+    func removeValue(_ key: String) {
+        removeValueFromTable(LegacyDatabaseStorage.STORE_TABLE_NAME, key)
+    }
+
+    func removeLongValue(_ key: String) {
+        removeValueFromTable(LegacyDatabaseStorage.LONG_STORE_TABLE_NAME, key)
+    }
+
+    private func removeValueFromTable(_ table: String, _ key: String) {
+        let query = "DELETE FROM \(table) WHERE key = ?;"
+        _ = executeQuery(query) { stmt in
+            let bindResult = sqlite3_bind_text(stmt, 1, key, -1, LegacyDatabaseStorage.SQLITE_TRANSIENT)
+            if bindResult != SQLITE_OK {
+                logger?.error(message: "bind query parameter failed with result: \(bindResult)")
+                return
+            }
+
+            let stepResult = sqlite3_step(stmt)
+            if stepResult != SQLITE_DONE {
+                logger?.error(message: "execute query '\(query)' failed with result: \(stepResult)")
+                return
+            }
+        }
+    }
+
+    func removeEvent(_ rowId: Int64) {
+        removeEventFromTable(LegacyDatabaseStorage.EVENT_TABLE_NAME, rowId)
+    }
+
+    func removeIdentify(_ rowId: Int64) {
+        removeEventFromTable(LegacyDatabaseStorage.IDENTIFY_TABLE_NAME, rowId)
+    }
+
+    func removeInterceptedIdentify(_ rowId: Int64) {
+        removeEventFromTable(LegacyDatabaseStorage.INTERCEPTED_IDENTIFY_TABLE_NAME, rowId)
+    }
+
+    private func removeEventFromTable(_ table: String, _ rowId: Int64) {
+        let query = "DELETE FROM \(table) WHERE id = ?;"
+        _ = executeQuery(query) { stmt in
+            let bindResult = sqlite3_bind_int64(stmt, 1, rowId)
+            if bindResult != SQLITE_OK {
+                logger?.error(message: "bind query parameter failed with result: \(bindResult)")
+                return
+            }
+
+            let stepResult = sqlite3_step(stmt)
+            if stepResult != SQLITE_DONE {
+                logger?.error(message: "execute query '\(query)' failed with result: \(stepResult)")
+                return
+            }
+        }
+    }
+
+    func readEvents() -> [[String: Any]] {
+        readEventsFromTable(LegacyDatabaseStorage.EVENT_TABLE_NAME)
+    }
+
+    func readIdentifies() -> [[String: Any]] {
+        readEventsFromTable(LegacyDatabaseStorage.IDENTIFY_TABLE_NAME)
+    }
+
+    func readInterceptedIdentifies() -> [[String: Any]] {
+        readEventsFromTable(LegacyDatabaseStorage.INTERCEPTED_IDENTIFY_TABLE_NAME)
+    }
+
+    private func readEventsFromTable(_ table: String) -> [[String: Any]] {
+        let query = "SELECT id, event FROM \(table) ORDER BY id;"
+        return executeQuery(query) { stmt in
+            var events: [[String: Any]] = []
+            while sqlite3_step(stmt) == SQLITE_ROW {
+                let rowId = sqlite3_column_int64(stmt, 0)
+                let rawEventData = sqlite3_column_text(stmt, 1)
+                if rawEventData != nil {
+                    let eventData = String(cString: rawEventData!).data(using: .utf8)
+                    if eventData != nil && eventData!.count > 0 {
+                        let event = try? JSONSerialization.jsonObject(with: eventData!, options: []) as? [String: Any]
+                        if var event {
+                            event["$rowId"] = rowId
+                            events.append(event)
+                        }
+                    }
+                }
+            }
+            return events
+        } ?? []
+    }
+
+    private func executeQuery<T>(_ query: String, _ block: (_ stmt: OpaquePointer) -> T) -> T? {
+        if !FileManager.default.fileExists(atPath: databasePath) {
+            return nil
+        }
+
+        var db: OpaquePointer?
+        let openResult = sqlite3_open(databasePath, &db)
+        if openResult != SQLITE_OK {
+            logger?.error(message: "open database failed with result: \(openResult)")
+            sqlite3_close(db)
+            return nil
+        }
+
+        var stmt: OpaquePointer?
+        let prepareResult = sqlite3_prepare(db, query, -1, &stmt, nil)
+        if prepareResult != SQLITE_OK {
+            logger?.error(message: "prepare query failed with result: \(prepareResult)")
+            return nil
+        }
+        let value = block(stmt!)
+        sqlite3_finalize(stmt)
+        sqlite3_close(db)
+        return value
+    }
+}