-
Notifications
You must be signed in to change notification settings - Fork 18
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
source-mysql: Handle inconsistent column name case
Apparently column names in MySQL are always case-insensitive, which means that it's possible to issue a DDL query naming a column with different capitalization from what we've got in our metadata, and that ought to work. This is solved by using a case-sensitive lookup from the column name in the DDL alteration query to an index in our metadata, and then "canonicalizing" the name to match what's in our metadata so that the rest of the metadata update logic works reliably. This fixes #2421
- Loading branch information
1 parent
004ffff
commit 639e679
Showing
5 changed files
with
200 additions
and
7 deletions.
There are no files selected for viewing
132 changes: 132 additions & 0 deletions
132
source-mysql/.snapshots/TestAlterTable_RenameColumnCaseInsensitive-discover
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| Binding 0: | ||
| { | ||
| "recommended_name": "test/altertable_renamecolumncaseinsensitive_41432361", | ||
| "resource_config_json": { | ||
| "namespace": "test", | ||
| "stream": "AlterTable_RenameColumnCaseInsensitive_41432361" | ||
| }, | ||
| "document_schema_json": { | ||
| "$defs": { | ||
| "TestAlterTable_RenameColumnCaseInsensitive_41432361": { | ||
| "type": "object", | ||
| "required": [ | ||
| "id" | ||
| ], | ||
| "$anchor": "TestAlterTable_RenameColumnCaseInsensitive_41432361", | ||
| "properties": { | ||
| "id": { | ||
| "type": "integer", | ||
| "description": "(source type: non-nullable int)" | ||
| }, | ||
| "new_data": { | ||
| "description": "(source type: text)", | ||
| "type": [ | ||
| "string", | ||
| "null" | ||
| ] | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "allOf": [ | ||
| { | ||
| "if": { | ||
| "properties": { | ||
| "_meta": { | ||
| "properties": { | ||
| "op": { | ||
| "const": "d" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "then": { | ||
| "reduce": { | ||
| "delete": true, | ||
| "strategy": "merge" | ||
| } | ||
| }, | ||
| "else": { | ||
| "reduce": { | ||
| "strategy": "merge" | ||
| } | ||
| }, | ||
| "required": [ | ||
| "_meta" | ||
| ], | ||
| "properties": { | ||
| "_meta": { | ||
| "type": "object", | ||
| "required": [ | ||
| "op", | ||
| "source" | ||
| ], | ||
| "properties": { | ||
| "before": { | ||
| "$ref": "#TestAlterTable_RenameColumnCaseInsensitive_41432361", | ||
| "description": "Record state immediately before this change was applied.", | ||
| "reduce": { | ||
| "strategy": "firstWriteWins" | ||
| } | ||
| }, | ||
| "op": { | ||
| "enum": [ | ||
| "c", | ||
| "d", | ||
| "u" | ||
| ], | ||
| "description": "Change operation type: 'c' Create/Insert, 'u' Update, 'd' Delete." | ||
| }, | ||
| "source": { | ||
| "$id": "https://github.com/estuary/connectors/source-mysql/mysql-source-info", | ||
| "properties": { | ||
| "ts_ms": { | ||
| "type": "integer", | ||
| "description": "Unix timestamp (in millis) at which this event was recorded by the database." | ||
| }, | ||
| "schema": { | ||
| "type": "string", | ||
| "description": "Database schema (namespace) of the event." | ||
| }, | ||
| "snapshot": { | ||
| "type": "boolean", | ||
| "description": "Snapshot is true if the record was produced from an initial table backfill and unset if produced from the replication log." | ||
| }, | ||
| "table": { | ||
| "type": "string", | ||
| "description": "Database table of the event." | ||
| }, | ||
| "cursor": { | ||
| "type": "string", | ||
| "description": "Cursor value representing the current position in the binlog." | ||
| }, | ||
| "txid": { | ||
| "type": "string", | ||
| "description": "The global transaction identifier associated with a change by MySQL. Only set if GTIDs are enabled." | ||
| } | ||
| }, | ||
| "type": "object", | ||
| "required": [ | ||
| "schema", | ||
| "table", | ||
| "cursor" | ||
| ] | ||
| } | ||
| }, | ||
| "reduce": { | ||
| "strategy": "merge" | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "$ref": "#TestAlterTable_RenameColumnCaseInsensitive_41432361" | ||
| } | ||
| ] | ||
| }, | ||
| "key": [ | ||
| "/id" | ||
| ] | ||
| } | ||
|
|
10 changes: 10 additions & 0 deletions
10
source-mysql/.snapshots/TestAlterTable_RenameColumnCaseInsensitive-init
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| # ================================ | ||
| # Collection "acmeCo/test/test/altertable_renamecolumncaseinsensitive_41432361": 2 Documents | ||
| # ================================ | ||
| {"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"AlterTable_RenameColumnCaseInsensitive_41432361","cursor":"backfill:0"}},"dataColumn":"aaa","id":1} | ||
| {"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"AlterTable_RenameColumnCaseInsensitive_41432361","cursor":"backfill:1"}},"dataColumn":"bbb","id":2} | ||
| # ================================ | ||
| # Final State Checkpoint | ||
| # ================================ | ||
| {"bindingStateV1":{"test%2FAlterTable_RenameColumnCaseInsensitive_41432361":{"backfilled":2,"key_columns":["id"],"metadata":{"charset":"utf8mb4","schema":{"columns":["id","dataColumn"],"types":{"dataColumn":{"charset":"utf8mb4","type":"text"},"id":{"type":"int"}}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} | ||
|
|
10 changes: 10 additions & 0 deletions
10
source-mysql/.snapshots/TestAlterTable_RenameColumnCaseInsensitive-renamed
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| # ================================ | ||
| # Collection "acmeCo/test/test/altertable_renamecolumncaseinsensitive_41432361": 2 Documents | ||
| # ================================ | ||
| {"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_RenameColumnCaseInsensitive_41432361","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"dataColumn":"ccc","id":3} | ||
| {"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_RenameColumnCaseInsensitive_41432361","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"id":4,"new_data":"ddd"} | ||
| # ================================ | ||
| # Final State Checkpoint | ||
| # ================================ | ||
| {"bindingStateV1":{"test%2FAlterTable_RenameColumnCaseInsensitive_41432361":{"backfilled":2,"key_columns":["id"],"metadata":{"charset":"utf8mb4","schema":{"columns":["id","new_data"],"types":{"id":{"type":"int"},"new_data":{"charset":"utf8mb4","type":"text"}}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters