diff --git a/source-bigquery-batch/.snapshots/TestAsyncCapture-Capture b/source-bigquery-batch/.snapshots/TestAsyncCapture-Capture deleted file mode 100644 index 532db1cf4c..0000000000 --- a/source-bigquery-batch/.snapshots/TestAsyncCapture-Capture +++ /dev/null @@ -1,40 +0,0 @@ -# ================================ -# Collection "acmeCo/test/asynccapture_817595": 32 Documents -# ================================ -{"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1} -{"_meta":{"polled":"","index":999},"data":"Value for row 4","id":4} -{"_meta":{"polled":"","index":999},"data":"Value for row 5","id":5} -{"_meta":{"polled":"","index":999},"data":"Value for row 8","id":8} -{"_meta":{"polled":"","index":999},"data":"Value for row 9","id":9} -{"_meta":{"polled":"","index":999},"data":"Value for row 10","id":10} -{"_meta":{"polled":"","index":999},"data":"Value for row 12","id":12} -{"_meta":{"polled":"","index":999},"data":"Value for row 16","id":16} -{"_meta":{"polled":"","index":999},"data":"Value for row 18","id":18} -{"_meta":{"polled":"","index":999},"data":"Value for row 19","id":19} -{"_meta":{"polled":"","index":999},"data":"Value for row 20","id":20} -{"_meta":{"polled":"","index":999},"data":"Value for row 21","id":21} -{"_meta":{"polled":"","index":999},"data":"Value for row 23","id":23} -{"_meta":{"polled":"","index":999},"data":"Value for row 24","id":24} -{"_meta":{"polled":"","index":999},"data":"Value for row 27","id":27} -{"_meta":{"polled":"","index":999},"data":"Value for row 29","id":29} -{"_meta":{"polled":"","index":999},"data":"Value for row 31","id":31} -{"_meta":{"polled":"","index":999},"data":"Value for row 35","id":35} -{"_meta":{"polled":"","index":999},"data":"Value for row 36","id":36} -{"_meta":{"polled":"","index":999},"data":"Value for row 37","id":37} -{"_meta":{"polled":"","index":999},"data":"Value for row 38","id":38} -{"_meta":{"polled":"","index":999},"data":"Value for row 39","id":39} -{"_meta":{"polled":"","index":999},"data":"Value for row 40","id":40} -{"_meta":{"polled":"","index":999},"data":"Value for row 41","id":41} -{"_meta":{"polled":"","index":999},"data":"Value for row 42","id":42} -{"_meta":{"polled":"","index":999},"data":"Value for row 43","id":43} -{"_meta":{"polled":"","index":999},"data":"Value for row 44","id":44} -{"_meta":{"polled":"","index":999},"data":"Value for row 45","id":45} -{"_meta":{"polled":"","index":999},"data":"Value for row 46","id":46} -{"_meta":{"polled":"","index":999},"data":"Value for row 47","id":47} -{"_meta":{"polled":"","index":999},"data":"Value for row 48","id":48} -{"_meta":{"polled":"","index":999},"data":"Value for row 49","id":49} -# ================================ -# Final State Checkpoint -# ================================ -{"bindingStateV1":{"asynccapture_817595":{"CursorNames":["id"],"CursorValues":[49],"LastPolled":""}}} - diff --git a/source-bigquery-batch/.snapshots/TestAsyncCapture-Discovery b/source-bigquery-batch/.snapshots/TestAsyncCapture-Discovery deleted file mode 100644 index 1e3c083cc8..0000000000 --- a/source-bigquery-batch/.snapshots/TestAsyncCapture-Discovery +++ /dev/null @@ -1,57 +0,0 @@ -Binding 0: -{ - "resource_config_json": { - "name": "asynccapture_817595", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`asynccapture_817595`;\n{{- else -}}\n SELECT * FROM `testdata`.`asynccapture_817595`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", - "cursor": [ - "id" - ] - }, - "resource_path": [ - "asynccapture_817595" - ], - "collection": { - "name": "acmeCo/test/asynccapture_817595", - "read_schema_json": { - "type": "object", - "required": [ - "_meta", - "id" - ], - "properties": { - "_meta": { - "$schema": "http://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/estuary/connectors/source-bigquery-batch/document-metadata", - "properties": { - "polled": { - "type": "string", - "format": "date-time", - "title": "Polled Timestamp", - "description": "The time at which the update query which produced this document as executed." - }, - "index": { - "type": "integer", - "title": "Result Index", - "description": "The index of this document within the query execution which produced it." - } - }, - "type": "object", - "required": [ - "polled", - "index" - ] - }, - "id": { - "type": "integer" - } - }, - "x-infer-schema": true - }, - "key": [ - "/id" - ], - "projections": null - }, - "state_key": "asynccapture_817595" - } - diff --git a/source-bigquery-batch/.snapshots/TestBinaryTypes-Discovery b/source-bigquery-batch/.snapshots/TestBinaryTypes-Discovery index 0f7ea519a0..64410793c1 100644 --- a/source-bigquery-batch/.snapshots/TestBinaryTypes-Discovery +++ b/source-bigquery-batch/.snapshots/TestBinaryTypes-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "binarytypes_537491", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`binarytypes_537491`;\n{{- else -}}\n SELECT * FROM `testdata`.`binarytypes_537491`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "binarytypes_537491", "cursor": [ "id" ] diff --git a/source-bigquery-batch/.snapshots/TestCaptureWithDatetimeCursor-Discovery b/source-bigquery-batch/.snapshots/TestCaptureWithDatetimeCursor-Discovery index 772e4b318a..c04a2060b4 100644 --- a/source-bigquery-batch/.snapshots/TestCaptureWithDatetimeCursor-Discovery +++ b/source-bigquery-batch/.snapshots/TestCaptureWithDatetimeCursor-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "capturewithdatetimecursor_877736", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`capturewithdatetimecursor_877736`;\n{{- else -}}\n SELECT * FROM `testdata`.`capturewithdatetimecursor_877736`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "capturewithdatetimecursor_877736", "cursor": [ "updated_at" ] diff --git a/source-bigquery-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery b/source-bigquery-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery index e3ba01ed6f..58d00e01d5 100644 --- a/source-bigquery-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery +++ b/source-bigquery-batch/.snapshots/TestCaptureWithEmptyPoll-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "capturewithemptypoll_890703", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`capturewithemptypoll_890703`;\n{{- else -}}\n SELECT * FROM `testdata`.`capturewithemptypoll_890703`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "capturewithemptypoll_890703", "cursor": [ "updated_at" ] diff --git a/source-bigquery-batch/.snapshots/TestCaptureWithModifications-Discovery b/source-bigquery-batch/.snapshots/TestCaptureWithModifications-Discovery index 0da0f03510..d5e972743b 100644 --- a/source-bigquery-batch/.snapshots/TestCaptureWithModifications-Discovery +++ b/source-bigquery-batch/.snapshots/TestCaptureWithModifications-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "capturewithmodifications_786099", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`capturewithmodifications_786099`;\n{{- else -}}\n SELECT * FROM `testdata`.`capturewithmodifications_786099`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "capturewithmodifications_786099", "cursor": [ "updated_at" ] diff --git a/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Capture b/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Capture1 similarity index 69% rename from source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Capture rename to source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Capture1 index bb075a592c..033e59308c 100644 --- a/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Capture +++ b/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Capture1 @@ -1,15 +1,13 @@ # ================================ -# Collection "acmeCo/test/capturewithnullcursor_662607": 7 Documents +# Collection "acmeCo/test/capturewithnullcursor_662607": 5 Documents # ================================ -{"_meta":{"polled":"","index":999},"data":"Value with NULL cursor","id":0,"sort_col":null} -{"_meta":{"polled":"","index":999},"data":"Third NULL cursor","id":4,"sort_col":null} {"_meta":{"polled":"","index":999},"data":"Another NULL cursor","id":2,"sort_col":null} +{"_meta":{"polled":"","index":999},"data":"Third NULL cursor","id":4,"sort_col":null} +{"_meta":{"polled":"","index":999},"data":"Value with NULL cursor","id":0,"sort_col":null} {"_meta":{"polled":"","index":999},"data":"Value with cursor 10","id":1,"sort_col":10} {"_meta":{"polled":"","index":999},"data":"Value with cursor 20","id":3,"sort_col":20} -{"_meta":{"polled":"","index":999},"data":"Value with cursor 25","id":7,"sort_col":25} -{"_meta":{"polled":"","index":999},"data":"Final value cursor 30","id":9,"sort_col":30} # ================================ # Final State Checkpoint # ================================ -{"bindingStateV1":{"capturewithnullcursor_662607":{"CursorNames":["sort_col"],"CursorValues":[30],"LastPolled":""}}} +{"bindingStateV1":{"capturewithnullcursor_662607":{"CursorNames":["sort_col"],"CursorValues":[20],"LastPolled":""}}} diff --git a/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Capture2 b/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Capture2 new file mode 100644 index 0000000000..eff1455dc6 --- /dev/null +++ b/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Capture2 @@ -0,0 +1,10 @@ +# ================================ +# Collection "acmeCo/test/capturewithnullcursor_662607": 2 Documents +# ================================ +{"_meta":{"polled":"","index":999},"data":"Final value cursor 30","id":9,"sort_col":30} +{"_meta":{"polled":"","index":999},"data":"Value with cursor 25","id":7,"sort_col":25} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"capturewithnullcursor_662607":{"CursorNames":["sort_col"],"CursorValues":[30],"LastPolled":""}}} + diff --git a/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Discovery b/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Discovery index 77b2ec7de7..3cacc6ed0f 100644 --- a/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Discovery +++ b/source-bigquery-batch/.snapshots/TestCaptureWithNullCursor-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "capturewithnullcursor_662607", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`capturewithnullcursor_662607`;\n{{- else -}}\n SELECT * FROM `testdata`.`capturewithnullcursor_662607`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "capturewithnullcursor_662607", "cursor": [ "sort_col" ] diff --git a/source-bigquery-batch/.snapshots/TestCaptureWithTwoColumnCursor-Discovery b/source-bigquery-batch/.snapshots/TestCaptureWithTwoColumnCursor-Discovery index 6636c0561c..0ffa4330f0 100644 --- a/source-bigquery-batch/.snapshots/TestCaptureWithTwoColumnCursor-Discovery +++ b/source-bigquery-batch/.snapshots/TestCaptureWithTwoColumnCursor-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "capturewithtwocolumncursor_321285", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`capturewithtwocolumncursor_321285`;\n{{- else -}}\n SELECT * FROM `testdata`.`capturewithtwocolumncursor_321285`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "capturewithtwocolumncursor_321285", "cursor": [ "col1", "col2" diff --git a/source-bigquery-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Discovery b/source-bigquery-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Discovery index 6244355b0d..cb2c8da87a 100644 --- a/source-bigquery-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Discovery +++ b/source-bigquery-batch/.snapshots/TestCaptureWithUpdatedAtCursor-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "capturewithupdatedatcursor_792371", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`capturewithupdatedatcursor_792371`;\n{{- else -}}\n SELECT * FROM `testdata`.`capturewithupdatedatcursor_792371`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "capturewithupdatedatcursor_792371", "cursor": [ "updated_at" ] diff --git a/source-bigquery-batch/.snapshots/TestCompositeTypes-Discovery b/source-bigquery-batch/.snapshots/TestCompositeTypes-Discovery index 5e0c456794..c639196250 100644 --- a/source-bigquery-batch/.snapshots/TestCompositeTypes-Discovery +++ b/source-bigquery-batch/.snapshots/TestCompositeTypes-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "compositetypes_483529", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`compositetypes_483529`;\n{{- else -}}\n SELECT * FROM `testdata`.`compositetypes_483529`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "compositetypes_483529", "cursor": [ "id" ] diff --git a/source-bigquery-batch/.snapshots/TestFullRefresh-Capture1 b/source-bigquery-batch/.snapshots/TestFullRefresh-Capture1 new file mode 100644 index 0000000000..660f21e432 --- /dev/null +++ b/source-bigquery-batch/.snapshots/TestFullRefresh-Capture1 @@ -0,0 +1,12 @@ +# ================================ +# Collection "acmeCo/test/fullrefresh_902536": 4 Documents +# ================================ +{"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0} +{"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1} +{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2} +{"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"fullrefresh_902536":{"CursorNames":null,"CursorValues":null,"LastPolled":""}}} + diff --git a/source-bigquery-batch/.snapshots/TestFullRefresh-Capture b/source-bigquery-batch/.snapshots/TestFullRefresh-Capture2 similarity index 69% rename from source-bigquery-batch/.snapshots/TestFullRefresh-Capture rename to source-bigquery-batch/.snapshots/TestFullRefresh-Capture2 index 7244b08bf7..c4faa2476b 100644 --- a/source-bigquery-batch/.snapshots/TestFullRefresh-Capture +++ b/source-bigquery-batch/.snapshots/TestFullRefresh-Capture2 @@ -1,18 +1,14 @@ # ================================ -# Collection "acmeCo/test/fullrefresh_902536": 12 Documents +# Collection "acmeCo/test/fullrefresh_902536": 8 Documents # ================================ -{"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3} {"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0} -{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2} {"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1} +{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2} {"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3} {"_meta":{"polled":"","index":999},"data":"Value for row 4","id":4} -{"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1} -{"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2} {"_meta":{"polled":"","index":999},"data":"Value for row 5","id":5} -{"_meta":{"polled":"","index":999},"data":"Value for row 7","id":7} -{"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0} {"_meta":{"polled":"","index":999},"data":"Value for row 6","id":6} +{"_meta":{"polled":"","index":999},"data":"Value for row 7","id":7} # ================================ # Final State Checkpoint # ================================ diff --git a/source-bigquery-batch/.snapshots/TestFullRefresh-Discovery b/source-bigquery-batch/.snapshots/TestFullRefresh-Discovery index a5157d8f24..51b8e6eebe 100644 --- a/source-bigquery-batch/.snapshots/TestFullRefresh-Discovery +++ b/source-bigquery-batch/.snapshots/TestFullRefresh-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "fullrefresh_902536", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`fullrefresh_902536`;\n{{- else -}}\n SELECT * FROM `testdata`.`fullrefresh_902536`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}" + "schema": "testdata", + "table": "fullrefresh_902536" }, "resource_path": [ "fullrefresh_902536" diff --git a/source-bigquery-batch/.snapshots/TestIntegerTypes-Discovery b/source-bigquery-batch/.snapshots/TestIntegerTypes-Discovery index 5520f893c7..2de58074d5 100644 --- a/source-bigquery-batch/.snapshots/TestIntegerTypes-Discovery +++ b/source-bigquery-batch/.snapshots/TestIntegerTypes-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "integertypes_795898", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`integertypes_795898`;\n{{- else -}}\n SELECT * FROM `testdata`.`integertypes_795898`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "integertypes_795898", "cursor": [ "id" ] diff --git a/source-bigquery-batch/.snapshots/TestJSONType-Discovery b/source-bigquery-batch/.snapshots/TestJSONType-Discovery index 170f02e0e6..aa38f09f59 100644 --- a/source-bigquery-batch/.snapshots/TestJSONType-Discovery +++ b/source-bigquery-batch/.snapshots/TestJSONType-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "jsontype_519262", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`jsontype_519262`;\n{{- else -}}\n SELECT * FROM `testdata`.`jsontype_519262`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "jsontype_519262", "cursor": [ "id" ] diff --git a/source-bigquery-batch/.snapshots/TestNumericTypes-Discovery b/source-bigquery-batch/.snapshots/TestNumericTypes-Discovery index 2d489145c6..dcaa9b2d8b 100644 --- a/source-bigquery-batch/.snapshots/TestNumericTypes-Discovery +++ b/source-bigquery-batch/.snapshots/TestNumericTypes-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "numerictypes_559424", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`numerictypes_559424`;\n{{- else -}}\n SELECT * FROM `testdata`.`numerictypes_559424`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "numerictypes_559424", "cursor": [ "id" ] diff --git a/source-bigquery-batch/.snapshots/TestSimpleCapture-Capture b/source-bigquery-batch/.snapshots/TestSimpleCapture-Capture index f50b203ae7..9999ea6c69 100644 --- a/source-bigquery-batch/.snapshots/TestSimpleCapture-Capture +++ b/source-bigquery-batch/.snapshots/TestSimpleCapture-Capture @@ -1,7 +1,6 @@ # ================================ -# Collection "acmeCo/test/simplecapture_140272": 20 Documents +# Collection "acmeCo/test/simplecapture_140272": 8 Documents # ================================ -{"_meta":{"polled":"","index":999},"data":"Value for row 0","id":0} {"_meta":{"polled":"","index":999},"data":"Value for row 1","id":1} {"_meta":{"polled":"","index":999},"data":"Value for row 2","id":2} {"_meta":{"polled":"","index":999},"data":"Value for row 3","id":3} @@ -10,19 +9,8 @@ {"_meta":{"polled":"","index":999},"data":"Value for row 6","id":6} {"_meta":{"polled":"","index":999},"data":"Value for row 7","id":7} {"_meta":{"polled":"","index":999},"data":"Value for row 8","id":8} -{"_meta":{"polled":"","index":999},"data":"Value for row 9","id":9} -{"_meta":{"polled":"","index":999},"data":"Value for row 10","id":10} -{"_meta":{"polled":"","index":999},"data":"Value for row 11","id":11} -{"_meta":{"polled":"","index":999},"data":"Value for row 12","id":12} -{"_meta":{"polled":"","index":999},"data":"Value for row 13","id":13} -{"_meta":{"polled":"","index":999},"data":"Value for row 14","id":14} -{"_meta":{"polled":"","index":999},"data":"Value for row 15","id":15} -{"_meta":{"polled":"","index":999},"data":"Value for row 16","id":16} -{"_meta":{"polled":"","index":999},"data":"Value for row 17","id":17} -{"_meta":{"polled":"","index":999},"data":"Value for row 18","id":18} -{"_meta":{"polled":"","index":999},"data":"Value for row 19","id":19} # ================================ # Final State Checkpoint # ================================ -{"bindingStateV1":{"simplecapture_140272":{"CursorNames":["id"],"CursorValues":[19],"LastPolled":""}}} +{"bindingStateV1":{"simplecapture_140272":{"CursorNames":["id"],"CursorValues":[8],"LastPolled":""}}} diff --git a/source-bigquery-batch/.snapshots/TestSimpleCapture-Discovery b/source-bigquery-batch/.snapshots/TestSimpleCapture-Discovery index 71e679ae5c..52a9c52966 100644 --- a/source-bigquery-batch/.snapshots/TestSimpleCapture-Discovery +++ b/source-bigquery-batch/.snapshots/TestSimpleCapture-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "simplecapture_140272", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`simplecapture_140272`;\n{{- else -}}\n SELECT * FROM `testdata`.`simplecapture_140272`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "simplecapture_140272", "cursor": [ "id" ] diff --git a/source-bigquery-batch/.snapshots/TestStringTypes-Discovery b/source-bigquery-batch/.snapshots/TestStringTypes-Discovery index 2dd48f45cd..c3e109de52 100644 --- a/source-bigquery-batch/.snapshots/TestStringTypes-Discovery +++ b/source-bigquery-batch/.snapshots/TestStringTypes-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "stringtypes_339419", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`stringtypes_339419`;\n{{- else -}}\n SELECT * FROM `testdata`.`stringtypes_339419`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "stringtypes_339419", "cursor": [ "id" ] diff --git a/source-bigquery-batch/.snapshots/TestTemporalTypes-Discovery b/source-bigquery-batch/.snapshots/TestTemporalTypes-Discovery index 159f681894..7e67e0b464 100644 --- a/source-bigquery-batch/.snapshots/TestTemporalTypes-Discovery +++ b/source-bigquery-batch/.snapshots/TestTemporalTypes-Discovery @@ -2,7 +2,8 @@ Binding 0: { "resource_config_json": { "name": "temporaltypes_137023", - "template": "{{/* Default query template which adapts to cursor field selection */}}\n{{- if not .CursorFields -}}\n SELECT * FROM `testdata`.`temporaltypes_137023`;\n{{- else -}}\n SELECT * FROM `testdata`.`temporaltypes_137023`\n {{- if not .IsFirstQuery -}}\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = @p{{$j}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e @p{{$i}}\n\t{{- end -}})\n {{- end}}\n ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- end}}", + "schema": "testdata", + "table": "temporaltypes_137023", "cursor": [ "id" ] diff --git a/source-bigquery-batch/main_test.go b/source-bigquery-batch/main_test.go index 0a911f290f..15dcf887f4 100644 --- a/source-bigquery-batch/main_test.go +++ b/source-bigquery-batch/main_test.go @@ -13,7 +13,6 @@ import ( "os" "regexp" "strings" - "sync/atomic" "testing" "text/template" "time" @@ -146,9 +145,17 @@ func createTestTable(ctx context.Context, t testing.TB, client *bigquery.Client, t.Helper() require.NoError(t, executeSetupQuery(ctx, t, client, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName))) require.NoError(t, executeSetupQuery(ctx, t, client, fmt.Sprintf("CREATE TABLE %s%s", tableName, tableDef))) - t.Cleanup(func() { - require.NoError(t, executeSetupQuery(ctx, t, client, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName))) - }) + // A note on table cleanup: + // + // Typically in SQL database tests we drop tables at the end of each test. But we + // can't reliably do that in BigQuery. + // + // BigQuery has a rate limit of 5 table update operations per 10 seconds per table. + // DML operations (such as test data inserts) count against this limit but aren't + // subject to it (weird policy, but okay), which means that in general we can't + // reliably drop a table immediately after running a test. So we don't bother, + // because we _can_ reliably drop+recreate it at the start of a test, and that + // is sufficient for our purposes. } func summarizeBindings(t testing.TB, bindings []*pf.CaptureSpec_Binding) string { @@ -316,43 +323,6 @@ func TestSimpleCapture(t *testing.T) { }) } -// TestAsyncCapture performs a capture with periodic restarts, in parallel with a batch of inserts. -func TestAsyncCapture(t *testing.T) { - var ctx, cs, control = context.Background(), testCaptureSpec(t), testBigQueryClient(t) - var tableName, uniqueID = testTableName(t, uniqueTableID(t)) - createTestTable(ctx, t, control, tableName, "(id INTEGER PRIMARY KEY NOT ENFORCED, data STRING)") - - cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) - setCursorColumns(t, cs.Bindings[0], "id") - t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - - t.Run("Capture", func(t *testing.T) { - // Spawn a worker thread which will insert 50 rows of data in 5 batches. - // BigQuery insert latency is highly variable on test time-scales, so we signal - // when the insertions are all done so the capturing code knows when to stop. - var insertsDone atomic.Bool - go func() { - for batch := 0; batch < 5; batch++ { - var args [][]any - for i := batch * 10; i < (batch+1)*10; i++ { - args = append(args, []any{i, fmt.Sprintf("Value for row %d", i)}) - } - require.NoError(t, parallelSetupQueries(ctx, t, control, fmt.Sprintf("INSERT INTO %s VALUES (@p0, @p1)", tableName), args)) - } - time.Sleep(5 * time.Second) - insertsDone.Store(true) - }() - - // Run the capture over and over for 5 seconds each time until all inserts have finished. - for !insertsDone.Load() { - var captureCtx, cancelCapture = context.WithCancel(ctx) - time.AfterFunc(5*time.Second, cancelCapture) - cs.Capture(captureCtx, t, nil) - } - cupaloy.SnapshotT(t, cs.Summary()) - }) -} - // TestIntegerTypes exercises discovery and capture of the integer types // INT, SMALLINT, INTEGER, BIGINT, TINYINT, and BYTEINT. In BigQuery these // types are all aliases for each other, but it's worth being thorough. @@ -702,25 +672,27 @@ func TestFullRefresh(t *testing.T) { cs.Bindings = discoverBindings(ctx, t, cs, regexp.MustCompile(uniqueID)) t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - t.Run("Capture", func(t *testing.T) { - setShutdownAfterQuery(t, true) + setShutdownAfterQuery(t, true) - require.NoError(t, parallelSetupQueries(ctx, t, control, - fmt.Sprintf("INSERT INTO %s VALUES (@p0, @p1)", tableName), [][]any{ - {0, "Value for row 0"}, {1, "Value for row 1"}, - {2, "Value for row 2"}, {3, "Value for row 3"}, - })) - cs.Capture(ctx, t, nil) + // Sorting is required for test stability because full-refresh captures are unordered. + // The multiset of row-captures will be the same across all runs, so this works. + cs.Validator = &st.SortedCaptureValidator{} - require.NoError(t, parallelSetupQueries(ctx, t, control, - fmt.Sprintf("INSERT INTO %s VALUES (@p0, @p1)", tableName), [][]any{ - {4, "Value for row 4"}, {5, "Value for row 5"}, - {6, "Value for row 6"}, {7, "Value for row 7"}, - })) - cs.Capture(ctx, t, nil) + require.NoError(t, parallelSetupQueries(ctx, t, control, + fmt.Sprintf("INSERT INTO %s VALUES (@p0, @p1)", tableName), [][]any{ + {0, "Value for row 0"}, {1, "Value for row 1"}, + {2, "Value for row 2"}, {3, "Value for row 3"}, + })) + cs.Capture(ctx, t, nil) + t.Run("Capture1", func(t *testing.T) { cupaloy.SnapshotT(t, cs.Summary()); cs.Reset() }) - cupaloy.SnapshotT(t, cs.Summary()) - }) + require.NoError(t, parallelSetupQueries(ctx, t, control, + fmt.Sprintf("INSERT INTO %s VALUES (@p0, @p1)", tableName), [][]any{ + {4, "Value for row 4"}, {5, "Value for row 5"}, + {6, "Value for row 6"}, {7, "Value for row 7"}, + })) + cs.Capture(ctx, t, nil) + t.Run("Capture2", func(t *testing.T) { cupaloy.SnapshotT(t, cs.Summary()); cs.Reset() }) } // TestCaptureWithUpdatedAtCursor exercises the use-case of a capture using a @@ -963,35 +935,36 @@ func TestCaptureWithNullCursor(t *testing.T) { setCursorColumns(t, cs.Bindings[0], "sort_col") t.Run("Discovery", func(t *testing.T) { cupaloy.SnapshotT(t, summarizeBindings(t, cs.Bindings)) }) - t.Run("Capture", func(t *testing.T) { - setShutdownAfterQuery(t, true) - - // First batch with mix of NULL and non-NULL cursor values - require.NoError(t, parallelSetupQueries(ctx, t, control, - fmt.Sprintf("INSERT INTO %s VALUES (@p0, @p1, @p2)", tableName), - [][]any{ - {0, "Value with NULL cursor", bigquery.NullInt64{}}, - {1, "Value with cursor 10", 10}, - {2, "Another NULL cursor", bigquery.NullInt64{}}, - {3, "Value with cursor 20", 20}, - {4, "Third NULL cursor", bigquery.NullInt64{}}, - })) - cs.Capture(ctx, t, nil) + setShutdownAfterQuery(t, true) - // Second batch testing NULL handling after initial cursor - require.NoError(t, parallelSetupQueries(ctx, t, control, - fmt.Sprintf("INSERT INTO %s VALUES (@p0, @p1, @p2)", tableName), - [][]any{ - {5, "Late NULL cursor", bigquery.NullInt64{}}, // Will not be captured - {6, "Value with cursor 15", 15}, // Will not be captured (cursor 20 is already seen) - {7, "Value with cursor 25", 25}, - {8, "Another late NULL", bigquery.NullInt64{}}, // Will not be captured - {9, "Final value cursor 30", 30}, - })) - cs.Capture(ctx, t, nil) + // Sorting is required for test stability because null-cursored rows are unordered. + cs.Validator = &st.SortedCaptureValidator{} - cupaloy.SnapshotT(t, cs.Summary()) - }) + // First batch with mix of NULL and non-NULL cursor values + require.NoError(t, parallelSetupQueries(ctx, t, control, + fmt.Sprintf("INSERT INTO %s VALUES (@p0, @p1, @p2)", tableName), + [][]any{ + {0, "Value with NULL cursor", bigquery.NullInt64{}}, + {1, "Value with cursor 10", 10}, + {2, "Another NULL cursor", bigquery.NullInt64{}}, + {3, "Value with cursor 20", 20}, + {4, "Third NULL cursor", bigquery.NullInt64{}}, + })) + cs.Capture(ctx, t, nil) + t.Run("Capture1", func(t *testing.T) { cupaloy.SnapshotT(t, cs.Summary()); cs.Reset() }) + + // Second batch testing NULL handling after initial cursor + require.NoError(t, parallelSetupQueries(ctx, t, control, + fmt.Sprintf("INSERT INTO %s VALUES (@p0, @p1, @p2)", tableName), + [][]any{ + {5, "Late NULL cursor", bigquery.NullInt64{}}, // Will not be captured + {6, "Value with cursor 15", 15}, // Will not be captured (cursor 20 is already seen) + {7, "Value with cursor 25", 25}, + {8, "Another late NULL", bigquery.NullInt64{}}, // Will not be captured + {9, "Final value cursor 30", 30}, + })) + cs.Capture(ctx, t, nil) + t.Run("Capture2", func(t *testing.T) { cupaloy.SnapshotT(t, cs.Summary()); cs.Reset() }) } // TestQueryTemplateOverride exercises a capture configured with an explicit query template