diff --git a/CHANGELOG.md b/CHANGELOG.md index aadcdd4a..852d5da0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## unreleased + +* Add `trackPreviousValues` option on `Table` which sets `CrudEntry.oldData` to previous values on updates. +* Add `trackMetadata` option on `Table` which adds a `_metadata` column that can be used for updates. + The configured metadata is available through `CrudEntry.metadata`. +* Add `ignoreEmptyUpdates` option which skips creating CRUD entries for updates that don't change any values. + ## 1.0.1 * [Internal] Version bump for broken Swift release pipeline diff --git a/core/src/commonIntegrationTest/kotlin/com/powersync/CrudTest.kt b/core/src/commonIntegrationTest/kotlin/com/powersync/CrudTest.kt new file mode 100644 index 00000000..a2a1ed91 --- /dev/null +++ b/core/src/commonIntegrationTest/kotlin/com/powersync/CrudTest.kt @@ -0,0 +1,93 @@ +package com.powersync + +import com.powersync.db.schema.Column +import com.powersync.db.schema.Schema +import com.powersync.db.schema.Table +import com.powersync.db.schema.TrackPreviousValuesOptions +import com.powersync.testutils.databaseTest +import io.kotest.matchers.shouldBe +import kotlin.test.Test + +class CrudTest { + @Test + fun includeMetadata() = + databaseTest { + database.updateSchema(Schema(Table("lists", listOf(Column.text("name")), trackMetadata = true))) + + database.execute("INSERT INTO lists (id, name, _metadata) VALUES (uuid(), ?, ?)", listOf("entry", "so meta")) + val batch = database.getNextCrudTransaction() + batch!!.crud[0].metadata shouldBe "so meta" + } + + @Test + fun includeOldValues() = + databaseTest { + database.updateSchema( + Schema( + Table("lists", listOf(Column.text("name"), Column.text("content")), trackPreviousValues = TrackPreviousValuesOptions()), + ), + ) + + database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content")) + database.execute("DELETE FROM ps_crud") + database.execute("UPDATE lists SET name = ?", listOf("new name")) + + val batch = database.getNextCrudTransaction() + batch!!.crud[0].oldData shouldBe mapOf("name" to "entry", "content" to "content") + } + + @Test + fun includeOldValuesWithFilter() = + databaseTest { + database.updateSchema( + Schema( + Table( + "lists", + listOf(Column.text("name"), Column.text("content")), + trackPreviousValues = TrackPreviousValuesOptions(columnFilter = listOf("name")), + ), + ), + ) + + database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content")) + database.execute("DELETE FROM ps_crud") + database.execute("UPDATE lists SET name = ?, content = ?", listOf("new name", "new content")) + + val batch = database.getNextCrudTransaction() + batch!!.crud[0].oldData shouldBe mapOf("name" to "entry") + } + + @Test + fun includeOldValuesWhenChanged() = + databaseTest { + database.updateSchema( + Schema( + Table( + "lists", + listOf(Column.text("name"), Column.text("content")), + trackPreviousValues = TrackPreviousValuesOptions(onlyWhenChanged = true), + ), + ), + ) + + database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content")) + database.execute("DELETE FROM ps_crud") + database.execute("UPDATE lists SET name = ?", listOf("new name")) + + val batch = database.getNextCrudTransaction() + batch!!.crud[0].oldData shouldBe mapOf("name" to "entry") + } + + @Test + fun ignoreEmptyUpdate() = + databaseTest { + database.updateSchema(Schema(Table("lists", listOf(Column.text("name"), Column.text("content")), ignoreEmptyUpdates = true))) + + database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content")) + database.execute("DELETE FROM ps_crud") + database.execute("UPDATE lists SET name = ?", listOf("entry")) + + val batch = database.getNextCrudTransaction() + batch shouldBe null + } +} diff --git a/core/src/commonMain/kotlin/com/powersync/db/crud/CrudEntry.kt b/core/src/commonMain/kotlin/com/powersync/db/crud/CrudEntry.kt index 7acfb0a9..73d0da84 100644 --- a/core/src/commonMain/kotlin/com/powersync/db/crud/CrudEntry.kt +++ b/core/src/commonMain/kotlin/com/powersync/db/crud/CrudEntry.kt @@ -1,5 +1,7 @@ package com.powersync.db.crud +import com.powersync.PowerSyncDatabase +import com.powersync.db.schema.Table import com.powersync.utils.JsonUtil import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonObject @@ -37,6 +39,15 @@ public data class CrudEntry( * This may change in the future. */ val transactionId: Int?, + /** + * User-defined metadata that can be attached to writes. + * + * This is the value the `_metadata` column had when the write to the database was made, + * allowing backend connectors to e.g. identify a write and treat it specially. + * + * Note that the `_metadata` column is only available when [Table.trackMetadata] is enabled. + */ + val metadata: String? = null, /** * Data associated with the change. * @@ -47,6 +58,13 @@ public data class CrudEntry( * For DELETE, this is null. */ val opData: Map?, + /** + * Previous values before this change. + * + * These values can be tracked for `UPDATE` statements when [Table.trackPreviousValues] is + * enabled. + */ + val oldData: Map? = null, ) { public companion object { public fun fromRow(row: CrudRow): CrudEntry { @@ -61,6 +79,11 @@ public data class CrudEntry( }, table = data["type"]!!.jsonPrimitive.content, transactionId = row.txId, + metadata = data["metadata"]?.jsonPrimitive?.content, + oldData = + data["old"]?.jsonObject?.mapValues { (_, value) -> + value.jsonPrimitive.contentOrNull + }, ) } } diff --git a/core/src/commonMain/kotlin/com/powersync/db/schema/Table.kt b/core/src/commonMain/kotlin/com/powersync/db/schema/Table.kt index 6314cd37..3e121834 100644 --- a/core/src/commonMain/kotlin/com/powersync/db/schema/Table.kt +++ b/core/src/commonMain/kotlin/com/powersync/db/schema/Table.kt @@ -1,14 +1,18 @@ package com.powersync.db.schema +import com.powersync.db.crud.CrudEntry import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonArray private const val MAX_AMOUNT_OF_COLUMNS = 1999 /** * A single table in the schema. */ -public data class Table constructor( +public data class Table( /** * The synced table name, matching sync rules. */ @@ -33,6 +37,22 @@ public data class Table constructor( * Override the name for the view */ private val viewNameOverride: String? = null, + /** + * Whether to add a hidden `_metadata` column that will be enabled for updates to attach custom + * information about writes that will be reported through [CrudEntry.metadata]. + */ + val trackMetadata: Boolean = false, + /** + * When set to a non-null value, track old values of columns for [CrudEntry.oldData]. + * + * See [TrackPreviousValuesOptions] for details. + */ + val trackPreviousValues: TrackPreviousValuesOptions? = null, + /** + * Whether an `UPDATE` statement that doesn't change any values should be ignored when creating + * CRUD entries. + */ + val ignoreEmptyUpdates: Boolean = false, ) { init { /** @@ -81,6 +101,9 @@ public data class Table constructor( name: String, columns: List, viewName: String? = null, + ignoreEmptyUpdates: Boolean = false, + trackMetadata: Boolean = false, + trackPreviousValues: TrackPreviousValuesOptions? = null, ): Table = Table( name, @@ -89,6 +112,9 @@ public data class Table constructor( localOnly = false, insertOnly = true, viewNameOverride = viewName, + ignoreEmptyUpdates = ignoreEmptyUpdates, + trackMetadata = trackMetadata, + trackPreviousValues = trackPreviousValues, ) } @@ -135,6 +161,13 @@ public data class Table constructor( throw AssertionError("Invalid characters in view name: $viewNameOverride") } + check(!localOnly || !trackMetadata) { + "Can't track metadata for local-only tables." + } + check(!localOnly || trackPreviousValues == null) { + "Can't track old values for local-only tables." + } + val columnNames = mutableSetOf("id") for (column in columns) { when { @@ -185,6 +218,26 @@ public data class Table constructor( get() = viewNameOverride ?: name } +/** + * Options to include old values in [CrudEntry.oldData] for update statements. + * + * These options are enabled by passing them to a non-local [Table] constructor. + */ +public data class TrackPreviousValuesOptions( + /** + * A filter of column names for which updates should be tracked. + * + * When set to a non-null value, columns not included in this list will not appear in + * [CrudEntry.oldData]. By default, all columns are included. + */ + val columnFilter: List? = null, + /** + * Whether to only include old values when they were changed by an update, instead of always + * including all old values, + */ + val onlyWhenChanged: Boolean = false, +) + @Serializable internal data class SerializableTable( var name: String, @@ -196,16 +249,39 @@ internal data class SerializableTable( val insertOnly: Boolean = false, @SerialName("view_name") val viewName: String? = null, + @SerialName("ignore_empty_update") + val ignoreEmptyUpdate: Boolean = false, + @SerialName("include_metadata") + val includeMetadata: Boolean = false, + @SerialName("include_old") + val includeOld: JsonElement = JsonPrimitive(false), + @SerialName("include_old_only_when_changed") + val includeOldOnlyWhenChanged: Boolean = false, ) internal fun Table.toSerializable(): SerializableTable = with(this) { SerializableTable( - name, - columns.map { it.toSerializable() }, - indexes.map { it.toSerializable() }, - localOnly, - insertOnly, - viewName, + name = name, + columns = columns.map { it.toSerializable() }, + indexes = indexes.map { it.toSerializable() }, + localOnly = localOnly, + insertOnly = insertOnly, + viewName = viewName, + ignoreEmptyUpdate = ignoreEmptyUpdates, + includeMetadata = trackMetadata, + includeOld = + trackPreviousValues?.let { + if (it.columnFilter != null) { + buildJsonArray { + for (column in it.columnFilter) { + add(JsonPrimitive(column)) + } + } + } else { + JsonPrimitive(true) + } + } ?: JsonPrimitive(false), + includeOldOnlyWhenChanged = trackPreviousValues?.onlyWhenChanged ?: false, ) } diff --git a/core/src/commonTest/kotlin/com/powersync/db/schema/TableTest.kt b/core/src/commonTest/kotlin/com/powersync/db/schema/TableTest.kt index c2040f57..839be340 100644 --- a/core/src/commonTest/kotlin/com/powersync/db/schema/TableTest.kt +++ b/core/src/commonTest/kotlin/com/powersync/db/schema/TableTest.kt @@ -1,5 +1,14 @@ package com.powersync.db.schema +import com.powersync.utils.JsonUtil +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.boolean +import kotlinx.serialization.json.encodeToJsonElement +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.serializer import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith @@ -180,4 +189,44 @@ class TableTest { assertEquals(exception.message, "users: id column is automatically added, custom id columns are not supported") } + + @Test + fun testValidationLocalOnlyWithMetadata() { + val table = Table("foo", listOf(Column.text("bar")), localOnly = true, trackMetadata = true) + + val exception = shouldThrow { table.validate() } + exception.message shouldBe "Can't track metadata for local-only tables." + } + + @Test + fun testValidationLocalOnlyWithIncludeOld() { + val table = Table("foo", listOf(Column.text("bar")), localOnly = true, trackPreviousValues = TrackPreviousValuesOptions()) + + val exception = shouldThrow { table.validate() } + exception.message shouldBe "Can't track old values for local-only tables." + } + + @Test + fun handlesOptions() { + fun serialize(table: Table): JsonObject = + JsonUtil.json.encodeToJsonElement(serializer(), table.toSerializable()) as JsonObject + + serialize(Table("foo", emptyList(), trackMetadata = true))["include_metadata"]!!.jsonPrimitive.boolean shouldBe true + serialize(Table("foo", emptyList(), ignoreEmptyUpdates = true))["ignore_empty_update"]!!.jsonPrimitive.boolean shouldBe true + + serialize(Table("foo", emptyList(), trackPreviousValues = TrackPreviousValuesOptions())).let { + it["include_old"]!!.jsonPrimitive.boolean shouldBe true + it["include_old_only_when_changed"]!!.jsonPrimitive.boolean shouldBe false + } + + serialize(Table("foo", emptyList(), trackPreviousValues = TrackPreviousValuesOptions(columnFilter = listOf("foo", "bar")))).let { + it["include_old"]!!.jsonArray.map { e -> e.jsonPrimitive.content } shouldBe listOf("foo", "bar") + it["include_old_only_when_changed"]!!.jsonPrimitive.boolean shouldBe false + } + + serialize(Table("foo", emptyList(), trackPreviousValues = TrackPreviousValuesOptions(onlyWhenChanged = true))).let { + it["include_old"]!!.jsonPrimitive.boolean shouldBe true + it["include_old_only_when_changed"]!!.jsonPrimitive.boolean shouldBe true + } + } }