Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

source-oracle: use rs_id and ssn for keyless tables #2453

Merged
merged 1 commit into from
Feb 27, 2025
Merged

source-oracle: use rs_id and ssn for keyless tables #2453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions source-oracle-batch/config.pdb.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
user: c##flow_test_logminer
password_sops: ENC[AES256_GCM,data:8ZzAMqo8P8BDb+Xexv4K/g==,iv:P3WBCG658USMguqW10CCPBiIo20TsatG3+DU9z7f/Bo=,tag:/01GPk1BLhzj2yrxRNWz6A==,type:str]
user: C##FLOW_TEST_LOGMINER
password_sops: ENC[AES256_GCM,data:Sj9wns0ttyR1Q5Xraic1pQ==,iv:vsK2V5iqUanuKIQKOwerxbbVc5IWqwgOukY2PwFa6W8=,tag:n9vzMroNXRcl1eS0GbSe2A==,type:str]
address: estuarytest.sub06191706070.testvcn.oraclevcn.com:1521
database: MYPDB.sub06191706070.testvcn.oraclevcn.com
advanced:
Expand All @@ -18,8 +18,8 @@ sops:
azure_kv: []
hc_vault: []
age: []
lastmodified: "2024-12-19T09:52:09Z"
mac: ENC[AES256_GCM,data:ajqErfrqO2tfiVIlXnwAJN1eWe06VtUrn8OsCEu3Nl6h1PjboJRq2KsLHs1vIq0+LEQoiwHbUBaIZIq0d7yjeQyg0eMMYKWnZSeIRSZq3JRgbFT5zW8VLnluvAbuXRW+o3XYHhCoKmnUhd9HSHa5HjkxoSXruDEdsdUWcUOIv6w=,iv:yPIRnne6Ghh0KX01xayc/r/x1NYrh3w4t/A8iocqiqs=,tag:iVHXGD6Wjhn4KSHWEzQHOA==,type:str]
lastmodified: "2025-02-26T10:38:28Z"
mac: ENC[AES256_GCM,data:XWxaAmL+FzKvhHV9Rr5PrLIK2JIr73lFMCvy1IDA1NZFnModmBZ5RcIXSBDmaLIP0PlCEKL4xOKSPqAgX2otTQHsp/hzHR/Z/gJFjbI1bL7620rZTV+Ba6w1c8ADtgyy0bLDiGv3ZKHpGdeZag/21KdKavN7tUdFxl83DZcF078=,iv:RyXRPw3L3vhKfLS2yGHY+V/7M+ExlZAAZcopQsOL07w=,tag:qKT6JJ7z+z35JuUCmup8eg==,type:str]
pgp: []
encrypted_suffix: _sops
version: 3.8.1
6 changes: 3 additions & 3 deletions source-oracle/.snapshots/TestAllTypes-backfill
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# ================================
# Collection "acmeCo/test/flow_test_logminer/t18110541": 1 Documents
# Collection "acmeCo/test/c__flow_test_logminer/t18110541": 1 Documents
# ================================
{"DATEONLY":"2022-01-01T00:00:00","DATETIME":"1998-01-01T13:00:00","DEFAULTNUM":"123456789123456.123456789","DOUBLE_PRECISION":"123456789.123457","FLOAT_126":"123456789.123457","INTEG":"18446744073709551615","INTERVAL_DAY":"+01 02:03:04.567000","INTERVAL_YEAR":"+1234-05","NUM":"123456789.123457","NUM15":123456789123456,"NUM19":"1234567891234567891","NVCHAR2":"nvarchar2 value with unicode characters ❤️ \\ 🔥️')","R":"dGVzdGluZyByYXcgdmFsdWU=","REAL_NUM":"123456789.123457","SINGLE_NCHAR":"a","SMALL_INT":"123456789","TS":"2022-01-01T13:00:00.000000","TS_LOCAL_TZ":"2022-01-01T11:00:00.000000","TS_LOCAL_TZ_NINE":"2022-01-01T11:00:00.000000000","TS_NINE":"2022-01-01T13:00:00.123456789","TS_TZ":"2022-01-01T12:00:00.000000Z","TS_TZ_NINE":"2022-01-01T12:00:00.123456789Z","VCAHR2":"varchar2 value","VCHAR":"varchar value","_meta":{"op":"c","source":{"schema":"FLOW_TEST_LOGMINER","snapshot":true,"table":"T18110541","row_id":"AAAAAAAAAAAAAAAAAA"}}}
{"DATEONLY":"2022-01-01T00:00:00","DATETIME":"1998-01-01T13:00:00","DEFAULTNUM":"123456789123456.123456789","DOUBLE_PRECISION":"123456789.123457","FLOAT_126":"123456789.123457","INTEG":"18446744073709551615","INTERVAL_DAY":"+01 02:03:04.567000","INTERVAL_YEAR":"+1234-05","NUM":"123456789.123457","NUM15":123456789123456,"NUM19":"1234567891234567891","NVCHAR2":"nvarchar2 value with unicode characters ❤️ \\ 🔥️')","R":"dGVzdGluZyByYXcgdmFsdWU=","REAL_NUM":"123456789.123457","SINGLE_NCHAR":"a","SMALL_INT":"123456789","TS":"2022-01-01T13:00:00.000000","TS_LOCAL_TZ":"2022-01-01T11:00:00.000000","TS_LOCAL_TZ_NINE":"2022-01-01T11:00:00.000000000","TS_NINE":"2022-01-01T13:00:00.123456789","TS_TZ":"2022-01-01T12:00:00.000000Z","TS_TZ_NINE":"2022-01-01T12:00:00.123456789Z","VCAHR2":"varchar2 value","VCHAR":"varchar value","_meta":{"op":"c","source":{"schema":"C##FLOW_TEST_LOGMINER","snapshot":true,"table":"T18110541","row_id":"AAAAAAAAAAAAAAAAAA","rs_id":"111111111111111111","ssn":111}}}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"FLOW_TEST_LOGMINER%2FT18110541":{"backfilled":1,"mode":"Active"}},"cursor":"11111111"}
{"bindingStateV1":{"C%23%23FLOW_TEST_LOGMINER%2FT18110541":{"backfilled":1,"mode":"Active"}},"cursor":"11111111"}

27 changes: 19 additions & 8 deletions source-oracle/.snapshots/TestAllTypes-discover
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
Binding 0:
{
"recommended_name": "flow_test_logminer/t18110541",
"recommended_name": "c__flow_test_logminer/t18110541",
"resource_config_json": {
"mode": "Without Primary Key",
"namespace": "FLOW_TEST_LOGMINER",
"namespace": "C##FLOW_TEST_LOGMINER",
"stream": "T18110541"
},
"document_schema_json": {
"$defs": {
"FLOW_TEST_LOGMINERT18110541": {
"C##FLOW_TEST_LOGMINERT18110541": {
"type": "object",
"$anchor": "FLOW_TEST_LOGMINERT18110541",
"$anchor": "C##FLOW_TEST_LOGMINERT18110541",
"properties": {
"DATEONLY": {
"description": "(source type: DATE)",
Expand Down Expand Up @@ -229,7 +229,7 @@ Binding 0:
],
"properties": {
"before": {
"$ref": "#FLOW_TEST_LOGMINERT18110541",
"$ref": "#C##FLOW_TEST_LOGMINERT18110541",
"description": "Record state immediately before this change was applied.",
"reduce": {
"strategy": "firstWriteWins"
Expand Down Expand Up @@ -269,13 +269,23 @@ Binding 0:
"row_id": {
"type": "string",
"description": "ROWID of the document"
},
"rs_id": {
"type": "string",
"description": "Record Set ID of the logical change"
},
"ssn": {
"type": "integer",
"description": "SQL sequence number of the logical change"
}
},
"type": "object",
"required": [
"schema",
"table",
"row_id"
"row_id",
"rs_id",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are there any concerns about breakage to preexisting users when adding these as required? I think the answer is no (very few users at present and at least one seems to have automatic discovery disabled anyway), but just wanted to make sure this has been considered and we're confident no compatibility strategies are required.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah no real customers are using this seriously yet. Launchmetrics has been the only one who tried it but ended up not fitting their requirements and they are waiting for the batch connector

"ssn"
]
}
},
Expand All @@ -286,12 +296,13 @@ Binding 0:
}
},
{
"$ref": "#FLOW_TEST_LOGMINERT18110541"
"$ref": "#C##FLOW_TEST_LOGMINERT18110541"
}
]
},
"key": [
"/_meta/source/row_id"
"/_meta/source/rs_id",
"/_meta/source/ssn"
]
}

6 changes: 3 additions & 3 deletions source-oracle/.snapshots/TestAllTypes-replication
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# ================================
# Collection "acmeCo/test/flow_test_logminer/t18110541": 1 Documents
# Collection "acmeCo/test/c__flow_test_logminer/t18110541": 1 Documents
# ================================
{"DATEONLY":"2022-01-01T00:00:00","DATETIME":"1998-01-01T13:00:00","DEFAULTNUM":"123456789123456.123456789","DOUBLE_PRECISION":"123456789.123457","FLOAT_126":"123456789.123457","INTEG":"18446744073709551615","INTERVAL_DAY":"+01 02:03:04.567000","INTERVAL_YEAR":"+1234-05","NUM":"123456789.123457","NUM15":123456789123456,"NUM19":"1234567891234567891","NVCHAR2":"nvarchar2 value with unicode characters ❤️ \\ 🔥️')","R":"dGVzdGluZyByYXcgdmFsdWU=","REAL_NUM":"123456789.123457","SINGLE_NCHAR":"a","SMALL_INT":"123456789","TS":"2022-01-01T13:00:00","TS_LOCAL_TZ":"2022-01-01T11:00:00","TS_LOCAL_TZ_NINE":"2022-01-01T11:00:00","TS_NINE":"2022-01-01T13:00:00.123456789","TS_TZ":"2022-01-01T12:00:00Z","TS_TZ_NINE":"2022-01-01T12:00:00.123456789Z","VCAHR2":"varchar2 value","VCHAR":"varchar value","_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"FLOW_TEST_LOGMINER","table":"T18110541","scn":11111111,"row_id":"AAAAAAAAAAAAAAAAAA"}}}
{"DATEONLY":"2022-01-01T00:00:00","DATETIME":"1998-01-01T13:00:00","DEFAULTNUM":"123456789123456.123456789","DOUBLE_PRECISION":"123456789.123457","FLOAT_126":"123456789.123457","INTEG":"18446744073709551615","INTERVAL_DAY":"+01 02:03:04.567000","INTERVAL_YEAR":"+1234-05","NUM":"123456789.123457","NUM15":123456789123456,"NUM19":"1234567891234567891","NVCHAR2":"nvarchar2 value with unicode characters ❤️ \\ 🔥️')","R":"dGVzdGluZyByYXcgdmFsdWU=","REAL_NUM":"123456789.123457","SINGLE_NCHAR":"a","SMALL_INT":"123456789","TS":"2022-01-01T13:00:00","TS_LOCAL_TZ":"2022-01-01T11:00:00","TS_LOCAL_TZ_NINE":"2022-01-01T11:00:00","TS_NINE":"2022-01-01T13:00:00.123456789","TS_TZ":"2022-01-01T12:00:00Z","TS_TZ_NINE":"2022-01-01T12:00:00.123456789Z","VCAHR2":"varchar2 value","VCHAR":"varchar value","_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"C##FLOW_TEST_LOGMINER","table":"T18110541","scn":11111111,"row_id":"AAAAAAAAAAAAAAAAAA","rs_id":"111111111111111111","ssn":111}}}
# ================================
# Final State Checkpoint
# ================================
{"bindingStateV1":{"FLOW_TEST_LOGMINER%2FT18110541":{"backfilled":1,"mode":"Active"}},"cursor":"11111111"}
{"bindingStateV1":{"C%23%23FLOW_TEST_LOGMINER%2FT18110541":{"backfilled":1,"mode":"Active"}},"cursor":"11111111"}

9 changes: 0 additions & 9 deletions source-oracle/.snapshots/TestAllTypesPDB-backfill
View file
Open in desktop

This file was deleted.

Loading
Loading