Prepare Kotlin update, add schema options

CHANGELOG.md

@@ -2,7 +2,11 @@
 
 ## unreleased
 
-- Add sync progress information through `SyncStatusData.downloadProgress`.
+* Add sync progress information through `SyncStatusData.downloadProgress`.
+* Add `trackPreviousValues` option on `Table` which sets `CrudEntry.previousValues` to previous values on updates.
+* Add `trackMetadata` option on `Table` which adds a `_metadata` column that can be used for updates.
+  The configured metadata is available through `CrudEntry.metadata`.
+* Add `ignoreEmptyUpdates` option which skips creating CRUD entries for updates that don't change any values.
 
 # 1.0.0
 

Package.resolved

@@ -1,14 +1,5 @@
 {
   "pins" : [
-    {
-      "identity" : "powersync-kotlin",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/powersync-ja/powersync-kotlin.git",
-      "state" : {
-        "revision" : "ccd2e595195c59d570eb93a878ad6a5cfca72ada",
-        "version" : "1.0.1+SWIFT.0"
-      }
-    },
     {
       "identity" : "powersync-sqlite-core-swift",
       "kind" : "remoteSourceControl",

Package.swift

@@ -4,6 +4,35 @@
 import PackageDescription
 let packageName = "PowerSync"
 
+// Set this to the absolute path of your Kotlin SDK checkout if you want to use a local Kotlin
+// build. Also see docs/LocalBuild.md for details
+let localKotlinSdkOverride: String? = nil
+
+// Our target and dependency setup is different when a local Kotlin SDK is used. Without the local
+// SDK, we have no package dependency on Kotlin and download the XCFramework from Kotlin releases as
+// a binary target.
+// With a local SDK, we point to a `Package.swift` within the Kotlin SDK containing a target pointing
+// towards a local framework build
+var conditionalDependencies: [Package.Dependency] = []
+var conditionalTargets: [Target] = []
+var kotlinTargetDependency = Target.Dependency.target(name: "PowerSyncKotlin")
+
+if let kotlinSdkPath = localKotlinSdkOverride {
+    // We can't depend on local XCFrameworks outside of this project's root, so there's a Package.swift
+    // in the PowerSyncKotlin project pointing towards a local build.
+    conditionalDependencies.append(.package(path: "\(kotlinSdkPath)/PowerSyncKotlin"))
+
+    kotlinTargetDependency = .product(name: "PowerSyncKotlin", package: "PowerSyncKotlin")
+} else {
+    // Not using a local build, so download from releases
+    conditionalTargets.append(.binaryTarget(
+        name: "PowerSyncKotlin",
+        // TODO: Use GitHub release once https://github.com/powersync-ja/powersync-kotlin/releases/tag/untagged-fde4386dec502ec27067 is published
+        url: "https://fsn1.your-objectstorage.com/simon-public/powersync.zip",
+        checksum: "b6770dc22ae31315adc599e653fea99614226312fe861dbd8764e922a5a83b09"
+    ))
+}
+
 let package = Package(
     name: packageName,
     platforms: [
@@ -17,21 +46,20 @@ let package = Package(
             targets: ["PowerSync"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/powersync-ja/powersync-kotlin.git", "1.0.1+SWIFT.0"..<"1.1.0+SWIFT.0"),
         .package(url: "https://github.com/powersync-ja/powersync-sqlite-core-swift.git", "0.3.14"..<"0.4.0")
-    ],
+    ] + conditionalDependencies,
     targets: [
         // Targets are the basic building blocks of a package, defining a module or a test suite.
         // Targets can depend on other targets in this package and products from dependencies.
         .target(
             name: packageName,
             dependencies: [
-                .product(name: "PowerSyncKotlin", package: "powersync-kotlin"),
+                kotlinTargetDependency,
                 .product(name: "PowerSyncSQLiteCore", package: "powersync-sqlite-core-swift")
             ]),
         .testTarget(
             name: "PowerSyncTests",
             dependencies: ["PowerSync"]
         ),
-    ]
+    ] + conditionalTargets
 )

Sources/PowerSync/Kotlin/KotlinAdapter.swift

@@ -23,13 +23,25 @@ enum KotlinAdapter {
 
     struct Table {
         static func toKotlin(_ table: TableProtocol) -> PowerSyncKotlin.Table {
-            PowerSyncKotlin.Table(
+            let trackPreviousKotlin: PowerSyncKotlin.TrackPreviousValuesOptions? = if let track = table.trackPreviousValues {
+                PowerSyncKotlin.TrackPreviousValuesOptions(
+                    columnFilter: track.columnFilter,
+                    onlyWhenChanged: track.onlyWhenChanged
+                )
+            } else {
+                nil
+            }
+
+            return PowerSyncKotlin.Table(
                 name: table.name,
                 columns: table.columns.map { Column.toKotlin($0) },
                 indexes: table.indexes.map { Index.toKotlin($0) },
                 localOnly: table.localOnly,
                 insertOnly: table.insertOnly,
-                viewNameOverride: table.viewNameOverride
+                viewNameOverride: table.viewNameOverride,
+                trackMetadata: table.trackMetadata,
+                trackPreviousValues: trackPreviousKotlin,
+                ignoreEmptyUpdates: table.ignoreEmptyUpdates
             )
         }
     }

Sources/PowerSync/Kotlin/db/KotlinCrudEntry.swift

@@ -28,9 +28,17 @@ struct KotlinCrudEntry : CrudEntry {
         entry.transactionId?.int64Value
     }
 
+    var metadata: String? {
+        entry.metadata
+    }
+
     var opData: [String : String?]? {
         /// Kotlin represents this as Map<String, String?>, but this is
         /// converted to [String: Any] by SKIEE
         entry.opData?.mapValues { $0 as? String }
     }
+
+    var previousValues: [String : String?]? {
+        entry.previousValues?.mapValues { $0 as? String }
+    }
 }

Sources/PowerSync/Protocol/Schema/Table.swift

@@ -26,6 +26,46 @@ public protocol TableProtocol {
     ///
     var viewNameOverride: String? { get }
     var viewName: String { get }
+
+    /// Whether to add a hidden `_metadata` column that will ne abled for updates to
+    /// attach custom information about writes.
+    ///
+    /// When the `_metadata` column is written to for inserts or updates, its value will not be
+    /// part of ``CrudEntry/opData``. Instead, it is reported as ``CrudEntry/metadata``,
+    /// allowing ``PowerSyncBackendConnector``s to handle these updates specially.
+    var trackMetadata: Bool { get }
+
+    /// When set to a non-`nil` value, track old values of columns for ``CrudEntry/previousValues``.
+    ///
+    /// See ``TrackPreviousValuesOptions`` for details
+    var trackPreviousValues: TrackPreviousValuesOptions? { get }
+
+    /// Whether an `UPDATE` statement that doesn't change any values should be ignored entirely when
+    /// creating CRUD entries.
+    ///
+    /// This is disabled by default, meaning that an `UPDATE` on a row that doesn't change values would
+    /// create a ``CrudEntry`` with an empty ``CrudEntry/opData`` and ``UpdateType/patch``.
+    var ignoreEmptyUpdates: Bool { get }
+}
+
+/// Options to include old values in ``CrudEntry/previousValues`` for update statements.
+///
+/// These options are enabled by passing them to a non-local ``Table`` constructor.
+public struct TrackPreviousValuesOptions {
+    /// A filter of column names for which updates should be tracked.
+    ///
+    /// When set to a non-`nil` value, columns not included in this list will not appear in
+    /// ``CrudEntry/previousValues``. By default, all columns are included.
+    public let columnFilter: [String]?;
+
+    /// Whether to only include old values when they were changed by an update, instead of always including
+    /// all old values.
+    public let onlyWhenChanged: Bool;
+
+    public init(columnFilter: [String]? = nil, onlyWhenChanged: Bool = false) {
+        self.columnFilter = columnFilter
+        self.onlyWhenChanged = onlyWhenChanged
+    }
 }
 
 private let MAX_AMOUNT_OF_COLUMNS = 63
@@ -40,6 +80,9 @@ public struct Table: TableProtocol {
     public let localOnly: Bool
     public let insertOnly: Bool
     public let viewNameOverride: String?
+    public let trackMetadata: Bool
+    public let trackPreviousValues: TrackPreviousValuesOptions?
+    public let ignoreEmptyUpdates: Bool
 
     public var viewName: String {
         viewNameOverride ?? name
@@ -60,14 +103,20 @@ public struct Table: TableProtocol {
         indexes: [Index] = [],
         localOnly: Bool = false,
         insertOnly: Bool = false,
-        viewNameOverride: String? = nil
+        viewNameOverride: String? = nil,
+        trackMetadata: Bool = false,
+        trackPreviousValues: TrackPreviousValuesOptions? = nil,
+        ignoreEmptyUpdates: Bool = false
     ) {
         self.name = name
         self.columns = columns
         self.indexes = indexes
         self.localOnly = localOnly
         self.insertOnly = insertOnly
         self.viewNameOverride = viewNameOverride
+        self.trackMetadata = trackMetadata
+        self.trackPreviousValues = trackPreviousValues
+        self.ignoreEmptyUpdates = ignoreEmptyUpdates
     }
 
     private func hasInvalidSqliteCharacters(_ string: String) -> Bool {
@@ -82,11 +131,20 @@ public struct Table: TableProtocol {
         if columns.count > MAX_AMOUNT_OF_COLUMNS {
             throw TableError.tooManyColumns(tableName: name, count: columns.count)
         }
-
+        
         if let viewNameOverride = viewNameOverride,
            hasInvalidSqliteCharacters(viewNameOverride) {
             throw TableError.invalidViewName(viewName: viewNameOverride)
         }
+
+        if localOnly {
+            if trackPreviousValues != nil {
+                throw TableError.trackPreviousForLocalTable(tableName: name)
+            }
+            if trackMetadata {
+                throw TableError.metadataForLocalTable(tableName: name)
+            }
+        }
 
         var columnNames = Set<String>(["id"])
 
@@ -156,4 +214,8 @@ public enum TableError: Error {
     case duplicateIndex(tableName: String, indexName: String)
     case invalidIndexName(tableName: String, indexName: String)
     case columnNotFound(tableName: String, columnName: String, indexName: String)
+    /// Local-only tables can't enable ``Table/trackMetadata`` because no updates are tracked for those tables at all.
+    case metadataForLocalTable(tableName: String)
+    /// Local-only tables can't enable ``Table/trackPreviousValues`` because no updates are tracked for those tables at all.
+    case trackPreviousForLocalTable(tableName: String)
 }

Sources/PowerSync/Protocol/db/CrudEntry.swift

@@ -44,6 +44,20 @@ public protocol CrudEntry {
     /// The transaction ID associated with the entry, if any.
     var transactionId: Int64? { get }
 
+    /// User-defined metadata that can be attached to writes.
+    ///
+    /// This is the value the `_metadata` column had when the write to the database was made,
+    /// allowing backend connectors to e.g. identify a write and tear it specially.
+    ///
+    /// Note that the `_metadata` column and this field are only available when ``Table/trackMetadata``
+    /// is enabled.
+    var metadata: String? { get }
+
     /// The operation data associated with the entry, represented as a dictionary of column names to their values.
     var opData: [String: String?]? { get }
+
+    /// Previous values before this change.
+    ///
+    /// These values can be tracked for `UPDATE` statements when ``Table/trackPreviousValues`` is enabled.
+    var previousValues: [String: String?]? { get }
 }

Tests/PowerSyncTests/CrudTests.swift

@@ -33,6 +33,95 @@ final class CrudTests: XCTestCase {
         database = nil
         try await super.tearDown()
     }
+
+    func testTrackMetadata() async throws {
+        try await database.updateSchema(schema: Schema(tables: [
+            Table(name: "lists", columns: [.text("name")], trackMetadata: true)
+        ]))
+
+        try await database.execute("INSERT INTO lists (id, name, _metadata) VALUES (uuid(), 'test', 'so meta')")
+        guard let batch = try await database.getNextCrudTransaction() else {
+            return XCTFail("Should have batch after insert")
+        }
+
+        XCTAssertEqual(batch.crud[0].metadata, "so meta")
+    }
+
+    func testTrackPreviousValues() async throws {
+        try await database.updateSchema(schema: Schema(tables: [
+            Table(
+                name: "lists",
+                columns: [.text("name"), .text("content")],
+                trackPreviousValues: TrackPreviousValuesOptions()
+            )
+        ]))
+
+        try await database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), 'entry', 'content')")
+        try await database.execute("DELETE FROM ps_crud")
+        try await database.execute("UPDATE lists SET name = 'new name'")
+
+        guard let batch = try await database.getNextCrudTransaction() else {
+            return XCTFail("Should have batch after update")
+        }
+
+        XCTAssertEqual(batch.crud[0].previousValues, ["name": "entry", "content": "content"])
+    }
+
+    func testTrackPreviousValuesWithFilter() async throws {
+        try await database.updateSchema(schema: Schema(tables: [
+            Table(
+                name: "lists",
+                columns: [.text("name"), .text("content")],
+                trackPreviousValues: TrackPreviousValuesOptions(
+                    columnFilter: ["name"]
+                )
+            )
+        ]))
+
+        try await database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), 'entry', 'content')")
+        try await database.execute("DELETE FROM ps_crud")
+        try await database.execute("UPDATE lists SET name = 'new name'")
+
+        guard let batch = try await database.getNextCrudTransaction() else {
+            return XCTFail("Should have batch after update")
+        }
+
+        XCTAssertEqual(batch.crud[0].previousValues, ["name": "entry"])
+    }
+
+    func testTrackPreviousValuesOnlyWhenChanged() async throws {
+        try await database.updateSchema(schema: Schema(tables: [
+            Table(
+                name: "lists",
+                columns: [.text("name"), .text("content")],
+                trackPreviousValues: TrackPreviousValuesOptions(
+                    onlyWhenChanged: true
+                )
+            )
+        ]))
+
+        try await database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), 'entry', 'content')")
+        try await database.execute("DELETE FROM ps_crud")
+        try await database.execute("UPDATE lists SET name = 'new name'")
+
+        guard let batch = try await database.getNextCrudTransaction() else {
+            return XCTFail("Should have batch after update")
+        }
+
+        XCTAssertEqual(batch.crud[0].previousValues, ["name": "entry"])
+    }
+
+    func testIgnoreEmptyUpdate() async throws {
+        try await database.updateSchema(schema: Schema(tables: [
+            Table(name: "lists", columns: [.text("name")], ignoreEmptyUpdates: true)
+        ]))
+        try await database.execute("INSERT INTO lists (id, name) VALUES (uuid(), 'test')")
+        try await database.execute("DELETE FROM ps_crud")
+        try await database.execute("UPDATE lists SET name = 'test'") // Same value!
+
+        let batch = try await database.getNextCrudTransaction()
+        XCTAssertNil(batch)
+    }
 
     func testCrudBatch() async throws {
         // Create some items

Tests/PowerSyncTests/Schema/TableTests.swift

@@ -190,6 +190,35 @@ final class TableTests: XCTestCase {
         }
     }
 
+    func testInvalidLocalOnlyTrackMetadata() {
+        let table = Table(name: "test", columns: [Column.text("name")], localOnly: true, trackMetadata: true)
+
+        XCTAssertThrowsError(try table.validate()) { error in
+            guard case TableError.metadataForLocalTable(let tableName) = error else {
+                XCTFail("Expected metadataForLocalTable error")
+                return
+            }
+            XCTAssertEqual(tableName, "test")
+        }
+    }
+
+    func testInvalidLocalOnlyTrackPrevious() {
+        let table = Table(
+            name: "test_prev",
+            columns: [Column.text("name")],
+            localOnly: true,
+            trackPreviousValues: TrackPreviousValuesOptions()
+        )
+
+        XCTAssertThrowsError(try table.validate()) { error in
+            guard case TableError.trackPreviousForLocalTable(let tableName) = error else {
+                XCTFail("Expected trackPreviousForLocalTable error")
+                return
+            }
+            XCTAssertEqual(tableName, "test_prev")
+        }
+    }
+
     func testValidTableValidation() throws {
         let table = Table(
             name: "users",