Skip to content

Commit

Permalink
source-postgres-batch: Test keyless and full-refresh captures
Browse files Browse the repository at this point in the history 
Includes a test which explicitly sets `keyless_row_id` to on and
off for a keyless full-refresh table in order to demonstrate the
difference in behavior.
  • Loading branch information
@willdonnelly
willdonnelly committed Feb 25, 2025
1 parent bfb4e20 commit 60f5377
Show file tree
Hide file tree
Showing 9 changed files with 617 additions and 0 deletions.
42 changes: 42 additions & 0 deletions source-postgres-batch/.snapshots/TestFeatureFlagKeylessRowID-Disabled-Capture
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# ================================
# Collection "acmeCo/test/test_featureflagkeylessrowid_disabled_906545": 34 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Additional row 5","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Additional row 6","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 7","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Initial row 4 (updated)","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Additional row 6 (updated)","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"New row A","value":20}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"test_featureflagkeylessrowid_disabled_906545":{"DocumentCount":9,"LastPolled":"<TIMESTAMP>"}}}

69 changes: 69 additions & 0 deletions source-postgres-batch/.snapshots/TestFeatureFlagKeylessRowID-Disabled-Discovery
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
Binding 0:
{
"resource_config_json": {
"name": "test_featureflagkeylessrowid_disabled_906545",
"schema": "test",
"table": "featureflagkeylessrowid_disabled_906545"
},
"resource_path": [
"test_featureflagkeylessrowid_disabled_906545"
],
"collection": {
"name": "acmeCo/test/test_featureflagkeylessrowid_disabled_906545",
"read_schema_json": {
"type": "object",
"required": [
"_meta"
],
"properties": {
"_meta": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata",
"properties": {
"polled": {
"type": "string",
"format": "date-time",
"title": "Polled Timestamp",
"description": "The time at which the update query which produced this document as executed."
},
"index": {
"type": "integer",
"title": "Result Index",
"description": "The index of this document within the query execution which produced it."
},
"row_id": {
"type": "integer",
"title": "Row ID",
"description": "Row ID of the Document"
},
"op": {
"type": "string",
"enum": [
"c",
"u",
"d"
],
"title": "Change Operation",
"description": "Operation type (c: Create / u: Update / d: Delete)",
"default": "u"
}
},
"type": "object",
"required": [
"polled",
"index",
"row_id"
]
}
},
"x-infer-schema": true
},
"key": [
"/_meta/polled",
"/_meta/index"
],
"projections": null
},
"state_key": "test_featureflagkeylessrowid_disabled_906545"
}

43 changes: 43 additions & 0 deletions source-postgres-batch/.snapshots/TestFeatureFlagKeylessRowID-Enabled-Capture
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# ================================
# Collection "acmeCo/test/test_featureflagkeylessrowid_enabled_905157": 35 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0,"op":"c"},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1,"op":"c"},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2,"op":"c"},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3,"op":"c"},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4,"op":"c"},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0,"op":"u"},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1,"op":"u"},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2,"op":"u"},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3,"op":"u"},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4,"op":"u"},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5,"op":"c"},"data":"Additional row 5","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6,"op":"c"},"data":"Additional row 6","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7,"op":"c"},"data":"Additional row 7","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8,"op":"c"},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9,"op":"c"},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0,"op":"u"},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1,"op":"u"},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2,"op":"u"},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3,"op":"u"},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4,"op":"u"},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5,"op":"u"},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6,"op":"u"},"data":"Initial row 4 (updated)","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7,"op":"u"},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8,"op":"u"},"data":"Additional row 6 (updated)","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9,"op":"u"},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0,"op":"u"},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1,"op":"u"},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2,"op":"u"},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3,"op":"u"},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4,"op":"u"},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5,"op":"u"},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6,"op":"u"},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7,"op":"u"},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8,"op":"u"},"data":"New row A","value":20}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9,"op":"d"}}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"test_featureflagkeylessrowid_enabled_905157":{"DocumentCount":9,"LastPolled":"<TIMESTAMP>"}}}

68 changes: 68 additions & 0 deletions source-postgres-batch/.snapshots/TestFeatureFlagKeylessRowID-Enabled-Discovery
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
Binding 0:
{
"resource_config_json": {
"name": "test_featureflagkeylessrowid_enabled_905157",
"schema": "test",
"table": "featureflagkeylessrowid_enabled_905157"
},
"resource_path": [
"test_featureflagkeylessrowid_enabled_905157"
],
"collection": {
"name": "acmeCo/test/test_featureflagkeylessrowid_enabled_905157",
"read_schema_json": {
"type": "object",
"required": [
"_meta"
],
"properties": {
"_meta": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata",
"properties": {
"polled": {
"type": "string",
"format": "date-time",
"title": "Polled Timestamp",
"description": "The time at which the update query which produced this document as executed."
},
"index": {
"type": "integer",
"title": "Result Index",
"description": "The index of this document within the query execution which produced it."
},
"row_id": {
"type": "integer",
"title": "Row ID",
"description": "Row ID of the Document"
},
"op": {
"type": "string",
"enum": [
"c",
"u",
"d"
],
"title": "Change Operation",
"description": "Operation type (c: Create / u: Update / d: Delete)",
"default": "u"
}
},
"type": "object",
"required": [
"polled",
"index",
"row_id"
]
}
},
"x-infer-schema": true
},
"key": [
"/_meta/row_id"
],
"projections": null
},
"state_key": "test_featureflagkeylessrowid_enabled_905157"
}

25 changes: 25 additions & 0 deletions source-postgres-batch/.snapshots/TestKeylessCapture-Capture
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# ================================
# Collection "acmeCo/test/test_keylesscapture_315710": 17 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","txid":999999,"value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","txid":999999,"value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","txid":999999,"value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Initial row 3","txid":999999,"value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 4","txid":999999,"value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":5},"data":"Additional row 5","txid":999999,"value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":6},"data":"Additional row 6","txid":999999,"value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":7},"data":"Additional row 7","txid":999999,"value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":8},"data":"Additional row 8","txid":999999,"value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":9},"data":"Additional row 9","txid":999999,"value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":10},"data":"Initial row 3 (updated)","txid":999999,"value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":11},"data":"Initial row 4 (updated)","txid":999999,"value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":12},"data":"Additional row 5 (updated)","txid":999999,"value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":13},"data":"Additional row 6 (updated)","txid":999999,"value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":14},"data":"Additional row 7 (updated)","txid":999999,"value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":15},"data":"Reinserted row 4","txid":999999,"value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":16},"data":"Reinserted row 6","txid":999999,"value":6}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"test_keylesscapture_315710":{"CursorNames":["txid"],"CursorValues":[999999],"DocumentCount":17,"LastPolled":"<TIMESTAMP>"}}}

72 changes: 72 additions & 0 deletions source-postgres-batch/.snapshots/TestKeylessCapture-Discovery
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
Binding 0:
{
"resource_config_json": {
"name": "test_keylesscapture_315710",
"schema": "test",
"table": "keylesscapture_315710",
"cursor": [
"txid"
]
},
"resource_path": [
"test_keylesscapture_315710"
],
"collection": {
"name": "acmeCo/test/test_keylesscapture_315710",
"read_schema_json": {
"type": "object",
"required": [
"_meta"
],
"properties": {
"_meta": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata",
"properties": {
"polled": {
"type": "string",
"format": "date-time",
"title": "Polled Timestamp",
"description": "The time at which the update query which produced this document as executed."
},
"index": {
"type": "integer",
"title": "Result Index",
"description": "The index of this document within the query execution which produced it."
},
"row_id": {
"type": "integer",
"title": "Row ID",
"description": "Row ID of the Document"
},
"op": {
"type": "string",
"enum": [
"c",
"u",
"d"
],
"title": "Change Operation",
"description": "Operation type (c: Create / u: Update / d: Delete)",
"default": "u"
}
},
"type": "object",
"required": [
"polled",
"index",
"row_id"
]
}
},
"x-infer-schema": true
},
"key": [
"/_meta/polled",
"/_meta/index"
],
"projections": null
},
"state_key": "test_keylesscapture_315710"
}

43 changes: 43 additions & 0 deletions source-postgres-batch/.snapshots/TestKeylessFullRefreshCapture-Capture
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# ================================
# Collection "acmeCo/test/test_keylessfullrefreshcapture_941246": 35 Documents
# ================================
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Initial row 3","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Initial row 4","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Additional row 5","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Additional row 6","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 7","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Initial row 4 (updated)","value":4}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"Additional row 6 (updated)","value":6}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":0,"row_id":0},"data":"Initial row 0","value":0}
{"_meta":{"polled":"<TIMESTAMP>","index":1,"row_id":1},"data":"Initial row 1","value":1}
{"_meta":{"polled":"<TIMESTAMP>","index":2,"row_id":2},"data":"Initial row 2","value":2}
{"_meta":{"polled":"<TIMESTAMP>","index":3,"row_id":3},"data":"Additional row 8","value":8}
{"_meta":{"polled":"<TIMESTAMP>","index":4,"row_id":4},"data":"Additional row 9","value":9}
{"_meta":{"polled":"<TIMESTAMP>","index":5,"row_id":5},"data":"Initial row 3 (updated)","value":3}
{"_meta":{"polled":"<TIMESTAMP>","index":6,"row_id":6},"data":"Additional row 5 (updated)","value":5}
{"_meta":{"polled":"<TIMESTAMP>","index":7,"row_id":7},"data":"Additional row 7 (updated)","value":7}
{"_meta":{"polled":"<TIMESTAMP>","index":8,"row_id":8},"data":"New row A","value":20}
{"_meta":{"polled":"<TIMESTAMP>","index":9,"row_id":9},"data":"New row B","value":21}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"test_keylessfullrefreshcapture_941246":{"DocumentCount":10,"LastPolled":"<TIMESTAMP>"}}}

Loading

0 comments on commit 60f5377

Please sign in to comment.