diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml index a56aad2f523..0c558b00684 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml @@ -72,7 +72,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.22.7 + go-version: 1.23.4 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml index 00aab1b78ff..680b0da87e0 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml @@ -76,7 +76,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.22.7 + go-version: 1.23.4 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml index 72426e70a61..dd389663a35 100644 --- a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml +++ b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml @@ -83,7 +83,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.22.7 + go-version: 1.23.4 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml index 5ac1a55334c..e4ccc793933 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml @@ -75,7 +75,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.22.7 + go-version: 1.23.4 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml new file mode 100644 index 00000000000..87a78d0b659 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml @@ -0,0 +1,212 @@ +name: Query Serving (Queries - 2) - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Query Serving (Queries - 2)') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtgate, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Query Serving (Queries - 2) + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_query_serving_queries.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.23.4 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Build current commit's binaries + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + rm -Rf bin/* + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + - name: Convert ErrorContains checks to Error checks + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + find ./go/test/endtoend -name '*.go' -exec sed -i 's/ErrorContains/Error/g' {} + + find ./go/test/endtoend -name '*.go' -exec sed -i 's/EqualError/Error/g' {} + + + # Swap the binaries in the bin. Use vtgate version n-1 and keep vttablet at version n + - name: Use last release's VTGate + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + cp -r /tmp/vitess-build-current/bin/* $PWD/bin/ + rm -f $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vtgate $PWD/bin/vtgate + vtgate --version + + # Running a test with vtgate at version n-1 and vttablet/vtctld at version n + - name: Run query serving tests (vtgate=N-1, vttablet=N, vtctld=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 + + # Swap the binaries again. This time, vtgate will be at version n, and vttablet/vtctld will be at version n-1 + - name: Use current version VTGate, and other version VTTablet/VTctld + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate + + cp /tmp/vitess-build-other/bin/vtctld $PWD/bin + cp /tmp/vitess-build-other/bin/vtctldclient $PWD/bin + cp /tmp/vitess-build-other/bin/vtctl $PWD/bin + cp /tmp/vitess-build-other/bin/vtctlclient $PWD/bin + + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtgate --version + vttablet --version + + # Running a test with vtgate at version n and vttablet/vtctld at version n-1 + - name: Run query serving tests (vtgate=N, vttablet=N-1, vtctld=N-1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml new file mode 100644 index 00000000000..d3cfc662b5b --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml @@ -0,0 +1,208 @@ +name: Query Serving (Queries - 2) Next Release - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Query Serving (Queries - 2) Next Release') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtgate, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Query Serving (Queries - 2) Next Release + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check out commit's code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any nextly installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the next release of Vitess + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + - name: Convert ErrorContains checks to Error checks + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + find ./go/test/endtoend -name '*.go' -exec sed -i 's/ErrorContains/Error/g' {} + + find ./go/test/endtoend -name '*.go' -exec sed -i 's/EqualError/Error/g' {} + + + # Swap the binaries in the bin. Use vtgate version n+1 and keep vttablet at version n + - name: Use next release's VTGate + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + rm -f $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vtgate $PWD/bin/vtgate + vtgate --version + + # Running a test with vtgate at version n+1 and vttablet at version n + - name: Run query serving tests (vtgate=N+1, vttablet=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 + + # Swap the binaries again. This time, vtgate will be at version n, and vttablet will be at version n+1 + - name: Use current version VTGate, and other version VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtgate --version + vttablet --version + + # Running a test with vtgate at version n and vttablet at version n+1 + - name: Run query serving tests (vtgate=N, vttablet=N+1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml index 4b5fad0ab29..5616c247888 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml @@ -75,7 +75,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.22.7 + go-version: 1.23.4 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml index 701025d7ecc..532be0b998e 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml @@ -75,7 +75,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.22.7 + go-version: 1.23.4 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml index 8b121d4af10..2804e757652 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml @@ -75,7 +75,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.22.7 + go-version: 1.23.4 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_semi_sync.yml b/.github/workflows/upgrade_downgrade_test_semi_sync.yml index f12e323654e..76706f77b65 100644 --- a/.github/workflows/upgrade_downgrade_test_semi_sync.yml +++ b/.github/workflows/upgrade_downgrade_test_semi_sync.yml @@ -72,7 +72,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.22.7 + go-version: 1.23.4 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/changelog/22.0/22.0.0/summary.md b/changelog/22.0/22.0.0/summary.md index 3f63b2d868f..e63ffcc3547 100644 --- a/changelog/22.0/22.0.0/summary.md +++ b/changelog/22.0/22.0.0/summary.md @@ -8,7 +8,10 @@ - **[RPC Changes](#rpc-changes)** - **[Prefer not promoting a replica that is currently taking a backup](#reparents-prefer-not-backing-up)** - **[VTOrc Config File Changes](#vtorc-config-file-changes)** + - **[VTGate Config File Changes](#vtgate-config-file-changes)** - **[Support for More Efficient JSON Replication](#efficient-json-replication)** + - **[Support for LAST_INSERT_ID(x)](#last-insert-id)** + - **[Support for Maximum Idle Connections in the Pool](#max-idle-connections)** - **[Minor Changes](#minor-changes)** - **[VTTablet Flags](#flags-vttablet)** - **[Topology read concurrency behaviour changes](#topo-read-concurrency-changes)** @@ -60,12 +63,42 @@ The following fields can be dynamically changed - To upgrade to the newer version of the configuration file, first switch to using the flags in your current deployment before upgrading. Then you can switch to using the configuration file in the newer release. +### VTGate Config File Changes + +The Viper configuration keys for the following flags has been changed to match their flag names. Previously they had a discovery prefix instead of it being part of the name. + +| Flag Name | Old Configuration Key | New Configuration Key | +|--------------------------------------------------|--------------------------------------------------|--------------------------------------------------| +| `discovery_low_replication_lag` | `discovery.low_replication_lag` | `discovery_low_replication_lag` | +| `discovery_high_replication_lag_minimum_serving` | `discovery.high_replication_lag_minimum_serving` | `discovery_high_replication_lag_minimum_serving` | +| `discovery_min_number_serving_vttablets` | `discovery.min_number_serving_vttablets` | `discovery_min_number_serving_vttablets` | +| `discovery_legacy_replication_lag_algorithm` | `discovery.legacy_replication_lag_algorithm` | `discovery_legacy_replication_lag_algorithm` | + +To upgrade to the newer version of the configuration keys, first switch to using the flags in your current deployment before upgrading. Then you can switch to using the new configuration keys in the newer release. + ### Support for More Efficient JSON Replication In [#7345](https://github.com/vitessio/vitess/pull/17345) we added support for [`--binlog-row-value-options=PARTIAL_JSON`](https://dev.mysql.com/doc/refman/en/replication-options-binary-log.html#sysvar_binlog_row_value_options). You can read more about [this feature added to MySQL 8.0 here](https://dev.mysql.com/blog-archive/efficient-json-replication-in-mysql-8-0/). If you are using MySQL 8.0 or later and using JSON columns, you can now enable this MySQL feature across your Vitess cluster(s) to lower the disk space needed for binary logs and improve the CPU and memory usage in both `mysqld` (standard intrashard MySQL replication) and `vttablet` ([VReplication](https://vitess.io/docs/reference/vreplication/vreplication/)) without losing any capabilities or features. +### Support for `LAST_INSERT_ID(x)` + +In [#17408](https://github.com/vitessio/vitess/pull/17408) and [#17409](https://github.com/vitessio/vitess/pull/17409), we added the ability to use `LAST_INSERT_ID(x)` in Vitess directly at vtgate. This improvement allows certain queries—like `SELECT last_insert_id(123);` or `SELECT last_insert_id(count(*)) ...`—to be handled without relying on MySQL for the final value. + +**Limitations**: +- When using `LAST_INSERT_ID(x)` in ordered queries (e.g., `SELECT last_insert_id(col) FROM table ORDER BY foo`), MySQL sets the session’s last-insert-id value according to the *last row returned*. Vitess does not guarantee the same behavior. + +### Support for Maximum Idle Connections in the Pool + +In [#17443](https://github.com/vitessio/vitess/pull/17443) we introduced a new configurable max-idle-count parameter for connection pools. This allows you to specify the maximum number of idle connections retained in each connection pool to optimize performance and resource efficiency. + +You can control idle connection retention for the query server’s query pool, stream pool, and transaction pool with the following flags: +• --queryserver-config-query-pool-max-idle-count: Defines the maximum number of idle connections retained in the query pool. +• --queryserver-config-stream-pool-max-idle-count: Defines the maximum number of idle connections retained in the stream pool. +• --queryserver-config-txpool-max-idle-count: Defines the maximum number of idle connections retained in the transaction pool. + +This feature ensures that, during traffic spikes, idle connections are available for faster responses, while minimizing overhead in low-traffic periods by limiting the number of idle connections retained. It helps strike a balance between performance, efficiency, and cost. ## Minor Changes diff --git a/examples/local/401_backup.sh b/examples/local/401_backup.sh new file mode 100755 index 00000000000..1529440ea26 --- /dev/null +++ b/examples/local/401_backup.sh @@ -0,0 +1,40 @@ +# Copyright 2025 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script takes backups of the 'customer' keyspace and all its shards. + +# Load common environment variables and functions +source ../common/env.sh + +# Set keyspace and shard details for the 'customer' keyspace +KEYSPACE="customer" +SHARDS=("-80" "80-") + +# Ensure the keyspace and shards are healthy +echo "Ensuring keyspace $KEYSPACE exists and shards are healthy..." +for shard in "${SHARDS[@]}"; do + if ! wait_for_healthy_shard "$KEYSPACE" "$shard"; then + echo "Shard $shard is not healthy. Exiting..." + exit 1 + fi +done + +# Backup all shards of the customer keyspace +for shard in "${SHARDS[@]}"; do + echo "Backing up shard $shard in keyspace $KEYSPACE..." + vtctldclient BackupShard "$KEYSPACE/$shard" || fail "Backup failed for shard $shard." + echo "Backup succeeded for shard $shard." +done + +echo "Backup process completed successfully for all shards in $KEYSPACE." diff --git a/examples/local/402_list_backup.sh b/examples/local/402_list_backup.sh new file mode 100755 index 00000000000..c19dc5f6df1 --- /dev/null +++ b/examples/local/402_list_backup.sh @@ -0,0 +1,28 @@ +# Copyright 2025 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Load common environment variables and functions +source ../common/env.sh # Import necessary environment variables and functions from a common script + +# Set keyspace and shard details for the 'customer' keyspace +KEYSPACE="customer" # Define the keyspace to work with +SHARDS=("-80" "80-") # Define the shards within the keyspace to list backups for + +# List backups for each shard +for shard in "${SHARDS[@]}"; do # Loop through each shard defined earlier + echo "Listing available backups for keyspace $KEYSPACE and shard $shard..." # Log the start of the backup listing + vtctldclient GetBackups "$KEYSPACE/$shard" || echo "Failed to list backups for keyspace $KEYSPACE and shard $shard" # Attempt to list backups; log failure if it occurs +done + +echo "Backup listing process completed." # Log completion of the backup listing process diff --git a/examples/local/403_restore_from_backup.sh b/examples/local/403_restore_from_backup.sh new file mode 100755 index 00000000000..73fafd34aa0 --- /dev/null +++ b/examples/local/403_restore_from_backup.sh @@ -0,0 +1,46 @@ +# Copyright 2025 The Vitess Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script restores the first replica tablet from backups for the 'customer' keyspace. + +# Load common environment variables and functions +source ../common/env.sh # Import necessary environment variables and functions from a common script + +# Set keyspace and shard details for the 'customer' keyspace +KEYSPACE="customer" # Define the keyspace to work with +SHARDS=("-80" "80-") # Define the shards within the keyspace to restore + +# Restore all shards of the customer keyspace from backups +for shard in "${SHARDS[@]}"; do # Loop through each shard defined earlier + echo "Finding replica tablets for shard $shard..." # Log the start of the tablet search + + # Fetch the list of replica tablets for the current shard + REPLICA_TABLETS=$(vtctldclient GetTablets --keyspace="$KEYSPACE" --shard="$shard" --tablet-type=replica | awk '{print $1}') # Extract the first column containing tablet names + REPLICA_COUNT=$(echo "$REPLICA_TABLETS" | wc -l) # Count the number of replica tablets found + + # Check if any replica tablets were found + if [ "$REPLICA_COUNT" -lt 1 ]; then # If the count is less than 1, no replicas were found + echo "No replica tablets found for shard $shard. Exiting..." # Log a message and exit if none are found + exit 1 # Exit the script with an error code + fi + + # Choose the first replica for restoration + RESTORE_TABLET=$(echo "$REPLICA_TABLETS" | head -n 1) # Select the first replica tablet from the list + echo "Restoring tablet $RESTORE_TABLET from backup for shard $shard..." # Log the restoration action + + # Restore from backup and handle any failures + vtctldclient RestoreFromBackup "$RESTORE_TABLET" || fail "Restore failed for tablet $RESTORE_TABLET" # Attempt to restore from backup and log an error message if it fails +done + +echo "Restore process completed successfully for $KEYSPACE." # Log completion of the restore process diff --git a/examples/local/401_teardown.sh b/examples/local/501_teardown.sh similarity index 100% rename from examples/local/401_teardown.sh rename to examples/local/501_teardown.sh diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 1c57dd0c08e..052c19ecaae 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -282,11 +282,13 @@ Flags: --queryserver-config-pool-conn-max-lifetime duration query server connection max lifetime, vttablet manages various mysql connection pools. This config means if a connection has lived at least this long, it connection will be removed from pool upon the next time it is returned to the pool. --queryserver-config-pool-size int query server read pool size, connection pool is used by regular queries (non streaming, not in a transaction) (default 16) --queryserver-config-query-cache-memory int query server query cache size in bytes, maximum amount of memory to be used for caching. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache. (default 33554432) + --queryserver-config-query-pool-max-idle-count int query server query pool - maximum number of idle connections to retain in the pool. Use this to balance between faster response times during traffic bursts and resource efficiency during low-traffic periods. --queryserver-config-query-pool-timeout duration query server query pool timeout, it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead. --queryserver-config-query-timeout duration query server query timeout, this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed. (default 30s) --queryserver-config-schema-change-signal query server schema signal, will signal connected vtgates that schema has changed whenever this is detected. VTGates will need to have -schema_change_signal enabled for this to work (default true) --queryserver-config-schema-reload-time duration query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time. (default 30m0s) --queryserver-config-stream-buffer-size int query server stream buffer size, the maximum number of bytes sent from vttablet for each stream call. It's recommended to keep this value in sync with vtgate's stream_buffer_size. (default 32768) + --queryserver-config-stream-pool-max-idle-count int query server stream pool - maximum number of idle connections to retain in the pool. Use this to balance between faster response times during traffic bursts and resource efficiency during low-traffic periods. --queryserver-config-stream-pool-size int query server stream connection pool size, stream pool is used by stream queries: queries that return results to client in a streaming fashion (default 200) --queryserver-config-stream-pool-timeout duration query server stream pool timeout, it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout. --queryserver-config-strict-table-acl only allow queries that pass table acl checks @@ -294,6 +296,7 @@ Flags: --queryserver-config-transaction-cap int query server transaction cap is the maximum number of transactions allowed to happen at any given point of a time for a single vttablet. E.g. by setting transaction cap to 100, there are at most 100 transactions will be processed by a vttablet and the 101th transaction will be blocked (and fail if it cannot get connection within specified timeout) (default 20) --queryserver-config-transaction-timeout duration query server transaction timeout, a transaction will be killed if it takes longer than this value (default 30s) --queryserver-config-truncate-error-len int truncate errors sent to client if they are longer than this value (0 means do not truncate) + --queryserver-config-txpool-max-idle-count int query server transaction pool - maximum number of idle connections to retain in the pool. Use this to balance between faster response times during traffic bursts and resource efficiency during low-traffic periods. --queryserver-config-txpool-timeout duration query server transaction pool timeout, it is how long vttablet waits if tx pool is full (default 1s) --queryserver-config-warn-result-size int query server result size warning threshold, warn if number of rows returned from vttablet for non-streaming queries exceeds this --queryserver-enable-views Enable views support in vttablet. diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index bc647fb5347..e2b0c30db7f 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -274,11 +274,13 @@ Flags: --queryserver-config-pool-conn-max-lifetime duration query server connection max lifetime, vttablet manages various mysql connection pools. This config means if a connection has lived at least this long, it connection will be removed from pool upon the next time it is returned to the pool. --queryserver-config-pool-size int query server read pool size, connection pool is used by regular queries (non streaming, not in a transaction) (default 16) --queryserver-config-query-cache-memory int query server query cache size in bytes, maximum amount of memory to be used for caching. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache. (default 33554432) + --queryserver-config-query-pool-max-idle-count int query server query pool - maximum number of idle connections to retain in the pool. Use this to balance between faster response times during traffic bursts and resource efficiency during low-traffic periods. --queryserver-config-query-pool-timeout duration query server query pool timeout, it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead. --queryserver-config-query-timeout duration query server query timeout, this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed. (default 30s) --queryserver-config-schema-change-signal query server schema signal, will signal connected vtgates that schema has changed whenever this is detected. VTGates will need to have -schema_change_signal enabled for this to work (default true) --queryserver-config-schema-reload-time duration query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time. (default 30m0s) --queryserver-config-stream-buffer-size int query server stream buffer size, the maximum number of bytes sent from vttablet for each stream call. It's recommended to keep this value in sync with vtgate's stream_buffer_size. (default 32768) + --queryserver-config-stream-pool-max-idle-count int query server stream pool - maximum number of idle connections to retain in the pool. Use this to balance between faster response times during traffic bursts and resource efficiency during low-traffic periods. --queryserver-config-stream-pool-size int query server stream connection pool size, stream pool is used by stream queries: queries that return results to client in a streaming fashion (default 200) --queryserver-config-stream-pool-timeout duration query server stream pool timeout, it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout. --queryserver-config-strict-table-acl only allow queries that pass table acl checks @@ -286,6 +288,7 @@ Flags: --queryserver-config-transaction-cap int query server transaction cap is the maximum number of transactions allowed to happen at any given point of a time for a single vttablet. E.g. by setting transaction cap to 100, there are at most 100 transactions will be processed by a vttablet and the 101th transaction will be blocked (and fail if it cannot get connection within specified timeout) (default 20) --queryserver-config-transaction-timeout duration query server transaction timeout, a transaction will be killed if it takes longer than this value (default 30s) --queryserver-config-truncate-error-len int truncate errors sent to client if they are longer than this value (0 means do not truncate) + --queryserver-config-txpool-max-idle-count int query server transaction pool - maximum number of idle connections to retain in the pool. Use this to balance between faster response times during traffic bursts and resource efficiency during low-traffic periods. --queryserver-config-txpool-timeout duration query server transaction pool timeout, it is how long vttablet waits if tx pool is full (default 1s) --queryserver-config-warn-result-size int query server result size warning threshold, warn if number of rows returned from vttablet for non-streaming queries exceeds this --queryserver-enable-views Enable views support in vttablet. diff --git a/go/mysql/binlog_event.go b/go/mysql/binlog_event.go index 72c228b594e..90c2d7b9668 100644 --- a/go/mysql/binlog_event.go +++ b/go/mysql/binlog_event.go @@ -48,7 +48,7 @@ type BinlogEvent interface { IsValid() bool // General protocol events. - NextPosition() uint32 + NextPosition() uint64 // IsFormatDescription returns true if this is a // FORMAT_DESCRIPTION_EVENT. Do not call StripChecksum before diff --git a/go/mysql/binlog_event_common.go b/go/mysql/binlog_event_common.go index 548875c44f7..0bd9d401eaa 100644 --- a/go/mysql/binlog_event_common.go +++ b/go/mysql/binlog_event_common.go @@ -47,7 +47,7 @@ import ( // +----------------------------+ // | extra_headers 19 : x-19 | // +============================+ -// http://dev.mysql.com/doc/internals/en/event-header-fields.html +// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_replication_binlog_event.html#sect_protocol_replication_binlog_event_header type binlogEvent []byte const ( @@ -119,8 +119,9 @@ func (ev binlogEvent) Length() uint32 { } // NextPosition returns the nextPosition field from the header -func (ev binlogEvent) NextPosition() uint32 { - return binary.LittleEndian.Uint32(ev.Bytes()[13 : 13+4]) +func (ev binlogEvent) NextPosition() uint64 { + // Only 4 bytes are used for the next_position field in the header. + return uint64(binary.LittleEndian.Uint32(ev.Bytes()[13:17])) } // IsFormatDescription implements BinlogEvent.IsFormatDescription(). diff --git a/go/mysql/binlog_event_filepos.go b/go/mysql/binlog_event_filepos.go index b7e6ed9e0f2..8c60956faf1 100644 --- a/go/mysql/binlog_event_filepos.go +++ b/go/mysql/binlog_event_filepos.go @@ -75,12 +75,13 @@ func (ev *filePosBinlogEvent) StripChecksum(f BinlogFormat) (BinlogEvent, []byte // nextPosition returns the next file position of the binlog. // If no information is available, it returns 0. -func (ev *filePosBinlogEvent) nextPosition(f BinlogFormat) uint32 { +func (ev *filePosBinlogEvent) nextPosition(f BinlogFormat) uint64 { if f.HeaderLength <= 13 { // Dead code. This is just a failsafe. return 0 } - return binary.LittleEndian.Uint32(ev.Bytes()[13:17]) + // The header only uses 4 bytes for the next_position. + return uint64(binary.LittleEndian.Uint32(ev.Bytes()[13:17])) } // rotate implements BinlogEvent.Rotate(). @@ -139,7 +140,7 @@ type filePosFakeEvent struct { timestamp uint32 } -func (ev filePosFakeEvent) NextPosition() uint32 { +func (ev filePosFakeEvent) NextPosition() uint64 { return 0 } @@ -283,7 +284,7 @@ type filePosGTIDEvent struct { gtid replication.FilePosGTID } -func newFilePosGTIDEvent(file string, pos uint32, timestamp uint32) filePosGTIDEvent { +func newFilePosGTIDEvent(file string, pos uint64, timestamp uint32) filePosGTIDEvent { return filePosGTIDEvent{ filePosFakeEvent: filePosFakeEvent{ timestamp: timestamp, diff --git a/go/mysql/replication.go b/go/mysql/replication.go index 84c65842c7e..4c5a0c9523e 100644 --- a/go/mysql/replication.go +++ b/go/mysql/replication.go @@ -18,6 +18,7 @@ package mysql import ( "fmt" + "math" "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/vt/proto/vtrpc" @@ -32,9 +33,14 @@ const ( // This file contains the methods related to replication. // WriteComBinlogDump writes a ComBinlogDump command. -// See http://dev.mysql.com/doc/internals/en/com-binlog-dump.html for syntax. // Returns a SQLError. -func (c *Conn) WriteComBinlogDump(serverID uint32, binlogFilename string, binlogPos uint32, flags uint16) error { +// See: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_binlog_dump.html +func (c *Conn) WriteComBinlogDump(serverID uint32, binlogFilename string, binlogPos uint64, flags uint16) error { + // The binary log file position is a uint64, but the protocol command + // only uses 4 bytes for the file position. + if binlogPos > math.MaxUint32 { + return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "binlog position %d is too large, it must fit into 32 bits", binlogPos) + } c.sequence = 0 length := 1 + // ComBinlogDump 4 + // binlog-pos @@ -43,7 +49,7 @@ func (c *Conn) WriteComBinlogDump(serverID uint32, binlogFilename string, binlog len(binlogFilename) // binlog-filename data, pos := c.startEphemeralPacketWithHeader(length) pos = writeByte(data, pos, ComBinlogDump) - pos = writeUint32(data, pos, binlogPos) + pos = writeUint32(data, pos, uint32(binlogPos)) pos = writeUint16(data, pos, flags) pos = writeUint32(data, pos, serverID) _ = writeEOFString(data, pos, binlogFilename) diff --git a/go/mysql/replication/filepos_gtid.go b/go/mysql/replication/filepos_gtid.go index 850fb421915..95c7efcd3b1 100644 --- a/go/mysql/replication/filepos_gtid.go +++ b/go/mysql/replication/filepos_gtid.go @@ -33,14 +33,14 @@ func parseFilePosGTID(s string) (GTID, error) { return nil, fmt.Errorf("invalid FilePos GTID (%v): expecting file:pos", s) } - pos, err := strconv.ParseUint(parts[1], 0, 32) + pos, err := strconv.ParseUint(parts[1], 0, 64) if err != nil { return nil, fmt.Errorf("invalid FilePos GTID (%v): expecting pos to be an integer", s) } return FilePosGTID{ File: parts[0], - Pos: uint32(pos), + Pos: pos, }, nil } @@ -56,7 +56,7 @@ func ParseFilePosGTIDSet(s string) (GTIDSet, error) { // FilePosGTID implements GTID. type FilePosGTID struct { File string - Pos uint32 + Pos uint64 } // String implements GTID.String(). diff --git a/go/mysql/replication/filepos_gtid_test.go b/go/mysql/replication/filepos_gtid_test.go index 174aed6ccf9..6cef4756af2 100644 --- a/go/mysql/replication/filepos_gtid_test.go +++ b/go/mysql/replication/filepos_gtid_test.go @@ -23,7 +23,7 @@ import ( func Test_filePosGTID_String(t *testing.T) { type fields struct { file string - pos uint32 + pos uint64 } tests := []struct { name string @@ -35,6 +35,11 @@ func Test_filePosGTID_String(t *testing.T) { fields{file: "mysql-bin.166031", pos: 192394}, "mysql-bin.166031:192394", }, + { + "handles large position correctly", + fields{file: "vt-1448040107-bin.003222", pos: 4663881395}, + "vt-1448040107-bin.003222:4663881395", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -52,7 +57,7 @@ func Test_filePosGTID_String(t *testing.T) { func Test_filePosGTID_ContainsGTID(t *testing.T) { type fields struct { file string - pos uint32 + pos uint64 } type args struct { other GTID diff --git a/go/mysql/replication_test.go b/go/mysql/replication_test.go index c9a54485497..680cb9e68dc 100644 --- a/go/mysql/replication_test.go +++ b/go/mysql/replication_test.go @@ -17,6 +17,7 @@ limitations under the License. package mysql import ( + "math" "reflect" "testing" @@ -37,6 +38,10 @@ func TestComBinlogDump(t *testing.T) { cConn.Close() }() + // Try to write a ComBinlogDump packet with a position greater than 4 bytes. + err := cConn.WriteComBinlogDump(1, "moofarm", math.MaxInt64, 0x0d0e) + require.Error(t, err) + // Write ComBinlogDump packet, read it, compare. if err := cConn.WriteComBinlogDump(0x01020304, "moofarm", 0x05060708, 0x090a); err != nil { t.Fatalf("WriteComBinlogDump failed: %v", err) diff --git a/go/pools/smartconnpool/pool.go b/go/pools/smartconnpool/pool.go index d49032f34a1..b024cc656df 100644 --- a/go/pools/smartconnpool/pool.go +++ b/go/pools/smartconnpool/pool.go @@ -92,6 +92,7 @@ type RefreshCheck func() (bool, error) type Config[C Connection] struct { Capacity int64 + MaxIdleCount int64 IdleTimeout time.Duration MaxLifetime time.Duration RefreshInterval time.Duration @@ -123,6 +124,8 @@ type ConnPool[C Connection] struct { active atomic.Int64 // capacity is the maximum number of connections that this pool can open capacity atomic.Int64 + // maxIdleCount is the maximum idle connections in the pool + idleCount atomic.Int64 // workers is a waitgroup for all the currently running worker goroutines workers sync.WaitGroup @@ -138,6 +141,8 @@ type ConnPool[C Connection] struct { // maxCapacity is the maximum value to which capacity can be set; when the pool // is re-opened, it defaults to this capacity maxCapacity int64 + // maxIdleCount is the maximum idle connections in the pool + maxIdleCount int64 // maxLifetime is the maximum time a connection can be open maxLifetime atomic.Int64 // idleTimeout is the maximum time a connection can remain idle @@ -158,6 +163,7 @@ func NewPool[C Connection](config *Config[C]) *ConnPool[C] { pool := &ConnPool[C]{} pool.freshSettingsStack.Store(-1) pool.config.maxCapacity = config.Capacity + pool.config.maxIdleCount = config.MaxIdleCount pool.config.maxLifetime.Store(config.MaxLifetime.Nanoseconds()) pool.config.idleTimeout.Store(config.IdleTimeout.Nanoseconds()) pool.config.refreshInterval.Store(config.RefreshInterval.Nanoseconds()) @@ -192,6 +198,7 @@ func (pool *ConnPool[C]) runWorker(close <-chan struct{}, interval time.Duration func (pool *ConnPool[C]) open() { pool.close = make(chan struct{}) pool.capacity.Store(pool.config.maxCapacity) + pool.setIdleCount() // The expire worker takes care of removing from the waiter list any clients whose // context has been cancelled. @@ -315,6 +322,16 @@ func (pool *ConnPool[C]) MaxCapacity() int64 { return pool.config.maxCapacity } +func (pool *ConnPool[C]) setIdleCount() { + capacity := pool.Capacity() + maxIdleCount := pool.config.maxIdleCount + if maxIdleCount == 0 || maxIdleCount > capacity { + pool.idleCount.Store(capacity) + } else { + pool.idleCount.Store(maxIdleCount) + } +} + // InUse returns the number of connections that the pool has lent out to clients and that // haven't been returned yet. func (pool *ConnPool[C]) InUse() int64 { @@ -340,6 +357,10 @@ func (pool *ConnPool[C]) SetIdleTimeout(duration time.Duration) { pool.config.idleTimeout.Store(duration.Nanoseconds()) } +func (pool *ConnPool[D]) IdleCount() int64 { + return pool.idleCount.Load() +} + func (pool *ConnPool[D]) RefreshInterval() time.Duration { return time.Duration(pool.config.refreshInterval.Load()) } @@ -396,6 +417,10 @@ func (pool *ConnPool[C]) put(conn *Pooled[C]) { } if !pool.wait.tryReturnConn(conn) { + if pool.closeOnIdleLimitReached(conn) { + return + } + connSetting := conn.Conn.Setting() if connSetting == nil { pool.clean.Push(conn) @@ -407,6 +432,23 @@ func (pool *ConnPool[C]) put(conn *Pooled[C]) { } } +// closeOnIdleLimitReached closes a connection if the number of idle connections (active - inuse) in the pool +// exceeds the idleCount limit. It returns true if the connection is closed, false otherwise. +func (pool *ConnPool[C]) closeOnIdleLimitReached(conn *Pooled[C]) bool { + for { + open := pool.active.Load() + idle := open - pool.borrowed.Load() + if idle <= pool.idleCount.Load() { + return false + } + if pool.active.CompareAndSwap(open, open-1) { + pool.Metrics.idleClosed.Add(1) + conn.Close() + return true + } + } +} + func (pool *ConnPool[D]) extendedMaxLifetime() time.Duration { maxLifetime := pool.config.maxLifetime.Load() if maxLifetime == 0 { @@ -629,6 +671,9 @@ func (pool *ConnPool[C]) setCapacity(ctx context.Context, newcap int64) error { if oldcap == newcap { return nil } + // update the idle count to match the new capacity if necessary + // wait for connections to be returned to the pool if we're reducing the capacity. + defer pool.setIdleCount() const delay = 10 * time.Millisecond @@ -732,6 +777,9 @@ func (pool *ConnPool[C]) RegisterStats(stats *servenv.Exporter, name string) { // the smartconnpool doesn't have a maximum capacity return pool.Capacity() }) + stats.NewGaugeFunc(name+"IdleAllowed", "Tablet server conn pool idle allowed limit", func() int64 { + return pool.IdleCount() + }) stats.NewCounterFunc(name+"WaitCount", "Tablet server conn pool wait count", func() int64 { return pool.Metrics.WaitCount() }) diff --git a/go/pools/smartconnpool/pool_test.go b/go/pools/smartconnpool/pool_test.go index 701327005ad..44bd431d189 100644 --- a/go/pools/smartconnpool/pool_test.go +++ b/go/pools/smartconnpool/pool_test.go @@ -746,6 +746,51 @@ func TestExtendedLifetimeTimeout(t *testing.T) { } } +// TestMaxIdleCount tests the MaxIdleCount setting, to check if the pool closes +// the idle connections when the number of idle connections exceeds the limit. +func TestMaxIdleCount(t *testing.T) { + testMaxIdleCount := func(t *testing.T, setting *Setting, maxIdleCount int64, expClosedConn int) { + var state TestState + + ctx := context.Background() + p := NewPool(&Config[*TestConn]{ + Capacity: 5, + MaxIdleCount: maxIdleCount, + LogWait: state.LogWait, + }).Open(newConnector(&state), nil) + + defer p.Close() + + var conns []*Pooled[*TestConn] + for i := 0; i < 5; i++ { + r, err := p.Get(ctx, setting) + require.NoError(t, err) + assert.EqualValues(t, i+1, state.open.Load()) + assert.EqualValues(t, 0, p.Metrics.IdleClosed()) + + conns = append(conns, r) + } + + for _, conn := range conns { + p.put(conn) + } + + closedConn := 0 + for _, conn := range conns { + if conn.Conn.IsClosed() { + closedConn++ + } + } + assert.EqualValues(t, expClosedConn, closedConn) + assert.EqualValues(t, expClosedConn, p.Metrics.IdleClosed()) + } + + t.Run("WithoutSettings", func(t *testing.T) { testMaxIdleCount(t, nil, 2, 3) }) + t.Run("WithSettings", func(t *testing.T) { testMaxIdleCount(t, sFoo, 2, 3) }) + t.Run("WithoutSettings-MaxIdleCount-Zero", func(t *testing.T) { testMaxIdleCount(t, nil, 0, 0) }) + t.Run("WithSettings-MaxIdleCount-Zero", func(t *testing.T) { testMaxIdleCount(t, sFoo, 0, 0) }) +} + func TestCreateFail(t *testing.T) { var state TestState state.chaos.failConnect = true diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index 4253fbb5860..2adbdf13250 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -70,7 +70,11 @@ type VtgateProcess struct { } type VTGateConfiguration struct { - TransactionMode string `json:"transaction_mode,omitempty"` + TransactionMode string `json:"transaction_mode,omitempty"` + DiscoveryLowReplicationLag string `json:"discovery_low_replication_lag,omitempty"` + DiscoveryHighReplicationLag string `json:"discovery_high_replication_lag,omitempty"` + DiscoveryMinServingVttablets string `json:"discovery_min_number_serving_vttablets,omitempty"` + DiscoveryLegacyReplicationLagAlgo string `json:"discovery_legacy_replication_lag_algorithm"` } // ToJSONString will marshal this configuration as JSON diff --git a/go/test/endtoend/migration/migration_test.go b/go/test/endtoend/migration/migration_test.go index eca112e388d..54afd2cb3ee 100644 --- a/go/test/endtoend/migration/migration_test.go +++ b/go/test/endtoend/migration/migration_test.go @@ -132,10 +132,6 @@ three streams although only two are required. This is to show that there can exi streams from the same source. The main difference between an external source vs a vitess source is that the source proto contains an "external_mysql" field instead of keyspace and shard. That field is the key into the externalConnections section of the input yaml. - -VReplicationExec: insert into _vt.vreplication (workflow, db_name, source, pos, max_tps, max_replication_lag, tablet_types, time_updated, transaction_timestamp, state) values('product', 'vt_commerce', 'filter: > external_mysql:\"product\" ', ”, 9999, 9999, 'primary', 0, 0, 'Running') -VReplicationExec: insert into _vt.vreplication (workflow, db_name, source, pos, max_tps, max_replication_lag, tablet_types, time_updated, transaction_timestamp, state) values('customer', 'vt_commerce', 'filter: > external_mysql:\"customer\" ', ”, 9999, 9999, 'primary', 0, 0, 'Running') -VReplicationExec: insert into _vt.vreplication (workflow, db_name, source, pos, max_tps, max_replication_lag, tablet_types, time_updated, transaction_timestamp, state) values('orders', 'vt_commerce', 'filter: > external_mysql:\"customer\" ', ”, 9999, 9999, 'primary', 0, 0, 'Running') */ func TestMigration(t *testing.T) { yamlFile := startCluster(t) @@ -155,7 +151,7 @@ func TestMigration(t *testing.T) { migrate(t, "customer", "commerce", []string{"customer"}) migrate(t, "customer", "commerce", []string{"orders"}) vttablet := keyspaces["commerce"].Shards[0].Vttablets[0].VttabletProcess - waitForVReplicationToCatchup(t, vttablet, 1*time.Second) + waitForVReplicationToCatchup(t, vttablet, 30*time.Second) testcases := []struct { query string @@ -217,11 +213,11 @@ func migrate(t *testing.T, fromdb, toks string, tables []string) { var sqlEscaped bytes.Buffer val.EncodeSQL(&sqlEscaped) query := fmt.Sprintf("insert into _vt.vreplication "+ - "(workflow, db_name, source, pos, max_tps, max_replication_lag, tablet_types, time_updated, transaction_timestamp, state) values"+ - "('%s', '%s', %s, '', 9999, 9999, 'primary', 0, 0, 'Running')", tables[0], "vt_"+toks, sqlEscaped.String()) - fmt.Printf("VReplicationExec: %s\n", query) - vttablet := keyspaces[toks].Shards[0].Vttablets[0].VttabletProcess - err := clusterInstance.VtctldClientProcess.ExecuteCommand("VReplicationExec", vttablet.TabletPath, query) + "(workflow, db_name, source, pos, max_tps, max_replication_lag, tablet_types, time_updated, transaction_timestamp, state, options) values"+ + "('%s', '%s', %s, '', 9999, 9999, 'primary', 0, 0, 'Running', '{}')", tables[0], "vt_"+toks, sqlEscaped.String()) + fmt.Printf("VReplication insert: %s\n", query) + vttablet := keyspaces[toks].Shards[0].Vttablets[0].Alias + err := clusterInstance.VtctldClientProcess.ExecuteCommand("ExecuteFetchAsDBA", vttablet, query) require.NoError(t, err) } diff --git a/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go b/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go index ee8141860f4..d34b63b833b 100644 --- a/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go +++ b/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go @@ -244,7 +244,7 @@ func TestOnlineDDLFlow(t *testing.T) { select { case <-ticker.C: case <-workloadCtx.Done(): - t.Logf("Terminating routine throttler check") + fmt.Println("Terminating routine throttler check") return } } @@ -258,8 +258,8 @@ func TestOnlineDDLFlow(t *testing.T) { wg.Add(1) go func() { defer cancel() - defer t.Logf("Terminating workload") defer wg.Done() + defer fmt.Println("Terminating workload") runMultipleConnections(workloadCtx, t) }() }) diff --git a/go/test/endtoend/preparestmt/benchmark_test.go b/go/test/endtoend/preparestmt/benchmark_test.go new file mode 100644 index 00000000000..fcf10a6948a --- /dev/null +++ b/go/test/endtoend/preparestmt/benchmark_test.go @@ -0,0 +1,144 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preparestmt + +import ( + "math/rand/v2" + "testing" + + "github.com/icrowley/fake" +) + +/* +export ver=v1 p=~/benchmark && go test \ +-run '^$' -bench '^BenchmarkPreparedStmt' \ +-benchtime 2s -count 6 -cpu 4 \ +| tee $p/${ver}.txt +*/ +func BenchmarkPreparedStmt(b *testing.B) { + dbo := Connect(b) + defer dbo.Close() + + // prepare statement + insertStmt := `insert into sks.t1 (name, age, email, created_at, is_active) values(?, ?, ?, current_timestamp, ?)` + selectStmt := `select id, name, age, email from sks.t1 where age between ? and ? and is_active = ? limit ?` + updateStmt := `update sks.t1 set is_active = ? where id = ?` + deleteStmt := `delete from sks.t1 where is_active = ? and age = ?` + + joinStmt := `SELECT + user.id AS user_id +FROM + sks.t1 AS user +LEFT JOIN + sks.t1 AS parent ON user.id = parent.id AND parent.age = ? +LEFT JOIN + sks.t1 AS manager ON user.id = manager.id AND manager.is_active = ? +LEFT JOIN + sks.t1 AS child ON user.id = child.id +WHERE + user.is_active = ? + AND user.id = ? + AND parent.id = ? + AND manager.id = ?` + + iStmt, err := dbo.Prepare(insertStmt) + if err != nil { + b.Fatal(err) + } + defer iStmt.Close() + + b.Run("Insert", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := iStmt.Exec(fake.FirstName(), rand.IntN(100), fake.EmailAddress(), rand.IntN(2)) + if err != nil { + b.Fatal(err) + } + } + }) + + sStmt, err := dbo.Prepare(selectStmt) + if err != nil { + b.Fatal(err) + } + defer sStmt.Close() + + b.Run("Select", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + age := rand.IntN(80) + r, err := sStmt.Query(age, age+20, rand.IntN(2), rand.IntN(10)) + if err != nil { + b.Fatal(err) + } + r.Close() + } + }) + + jStmt, err := dbo.Prepare(joinStmt) + if err != nil { + b.Fatal(err) + } + defer jStmt.Close() + + b.Run("Join Select:Simple Route", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + age := rand.IntN(80) + active := rand.IntN(2) + id := rand.IntN(2000) + r, err := jStmt.Query(age, active, active, id, id, id) + if err != nil { + b.Fatal(err) + } + r.Close() + } + }) + + uStmt, err := dbo.Prepare(updateStmt) + if err != nil { + b.Fatal(err) + } + defer sStmt.Close() + + b.Run("Update", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err = uStmt.Exec(rand.IntN(2), rand.IntN(2000)) + if err != nil { + b.Fatal(err) + } + } + }) + + dStmt, err := dbo.Prepare(deleteStmt) + if err != nil { + b.Fatal(err) + } + defer sStmt.Close() + + b.Run("Delete", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err = dStmt.Exec(rand.IntN(2), rand.IntN(100)) + if err != nil { + b.Fatal(err) + } + } + }) + +} diff --git a/go/test/endtoend/preparestmt/main_test.go b/go/test/endtoend/preparestmt/main_test.go index 0e067062c94..dd0094fe825 100644 --- a/go/test/endtoend/preparestmt/main_test.go +++ b/go/test/endtoend/preparestmt/main_test.go @@ -18,6 +18,7 @@ package preparestmt import ( "database/sql" + _ "embed" "flag" "fmt" "os" @@ -51,7 +52,7 @@ type DBInfo struct { } func init() { - dbInfo.KeyspaceName = keyspaceName + dbInfo.KeyspaceName = uks dbInfo.Username = "testuser1" dbInfo.Password = "testpassword1" dbInfo.Params = []string{ @@ -65,9 +66,9 @@ var ( clusterInstance *cluster.LocalProcessCluster dbInfo DBInfo hostname = "localhost" - keyspaceName = "test_keyspace" + uks = "uks" + sks = "sks" testingID = 1 - tableName = "vt_prepare_stmt_test" cell = "zone1" mysqlAuthServerStatic = "mysql_auth_server_static.json" jsonExample = `{ @@ -108,57 +109,18 @@ var ( } } }` - sqlSchema = `create table ` + tableName + ` ( - id bigint auto_increment, - msg varchar(64), - keyspace_id bigint(20) unsigned NOT NULL, - tinyint_unsigned TINYINT, - bool_signed BOOL, - smallint_unsigned SMALLINT, - mediumint_unsigned MEDIUMINT, - int_unsigned INT, - float_unsigned FLOAT(10,2), - double_unsigned DOUBLE(16,2), - decimal_unsigned DECIMAL, - t_date DATE, - t_datetime DATETIME, - t_datetime_micros DATETIME(6), - t_time TIME, - t_timestamp TIMESTAMP, - c8 bit(8) DEFAULT NULL, - c16 bit(16) DEFAULT NULL, - c24 bit(24) DEFAULT NULL, - c32 bit(32) DEFAULT NULL, - c40 bit(40) DEFAULT NULL, - c48 bit(48) DEFAULT NULL, - c56 bit(56) DEFAULT NULL, - c63 bit(63) DEFAULT NULL, - c64 bit(64) DEFAULT NULL, - json_col JSON, - text_col TEXT, - data longblob, - tinyint_min TINYINT, - tinyint_max TINYINT, - tinyint_pos TINYINT, - tinyint_neg TINYINT, - smallint_min SMALLINT, - smallint_max SMALLINT, - smallint_pos SMALLINT, - smallint_neg SMALLINT, - medint_min MEDIUMINT, - medint_max MEDIUMINT, - medint_pos MEDIUMINT, - medint_neg MEDIUMINT, - int_min INT, - int_max INT, - int_pos INT, - int_neg INT, - bigint_min BIGINT, - bigint_max BIGINT, - bigint_pos BIGINT, - bigint_neg BIGINT, - primary key (id) - ) Engine=InnoDB` + + //go:embed uSchema.sql + uSQLSchema string + + //go:embed uVschema.json + uVschema string + + //go:embed sSchema.sql + sSQLSchema string + + //go:embed sVschema.json + sVschema string ) func TestMain(m *testing.M) { @@ -185,15 +147,13 @@ func TestMain(m *testing.M) { } // Start keyspace - keyspace := &cluster.Keyspace{ - Name: keyspaceName, - SchemaSQL: sqlSchema, - } - uks := &cluster.Keyspace{Name: "uks"} - if err := clusterInstance.StartUnshardedKeyspace(*keyspace, 1, false); err != nil { + ks := cluster.Keyspace{Name: uks, SchemaSQL: uSQLSchema, VSchema: uVschema} + if err := clusterInstance.StartUnshardedKeyspace(ks, 1, false); err != nil { return 1, err } - if err := clusterInstance.StartUnshardedKeyspace(*uks, 0, false); err != nil { + + ks = cluster.Keyspace{Name: sks, SchemaSQL: sSQLSchema, VSchema: sVschema} + if err := clusterInstance.StartKeyspace(ks, []string{"-"}, 0, false); err != nil { return 1, err } @@ -203,7 +163,8 @@ func TestMain(m *testing.M) { vtgateInstance.ExtraArgs = []string{ "--mysql_server_query_timeout", "1s", "--mysql_auth_server_static_file", clusterInstance.TmpDirectory + "/" + mysqlAuthServerStatic, - "--mysql_server_version", "8.0.16-7", + "--pprof-http", + "--schema_change_signal=false", } // Start vtgate @@ -251,7 +212,7 @@ func createConfig(name, data string) error { } // Connect will connect the vtgate through mysql protocol. -func Connect(t *testing.T, params ...string) *sql.DB { +func Connect(t testing.TB, params ...string) *sql.DB { dbo, err := sql.Open("mysql", dbInfo.ConnectionString(params...)) require.Nil(t, err) return dbo @@ -285,7 +246,7 @@ func execErr(dbo *sql.DB, stmt string, params ...any) *mysql.MySQLError { func selectWhere(t *testing.T, dbo *sql.DB, where string, params ...any) []tableData { var out []tableData // prepare query - qry := "SELECT msg, data, text_col, t_datetime, t_datetime_micros FROM " + tableName + qry := "SELECT msg, data, text_col, t_datetime, t_datetime_micros FROM vt_prepare_stmt_test" if where != "" { qry += " WHERE (" + where + ")" } @@ -307,7 +268,7 @@ func selectWhere(t *testing.T, dbo *sql.DB, where string, params ...any) []table func selectWhereWithTx(t *testing.T, tx *sql.Tx, where string, params ...any) []tableData { var out []tableData // prepare query - qry := "SELECT msg, data, text_col, t_datetime, t_datetime_micros FROM " + tableName + qry := "SELECT msg, data, text_col, t_datetime, t_datetime_micros FROM vt_prepare_stmt_test" if where != "" { qry += " WHERE (" + where + ")" } diff --git a/go/test/endtoend/preparestmt/sSchema.sql b/go/test/endtoend/preparestmt/sSchema.sql new file mode 100644 index 00000000000..7da6b759a9a --- /dev/null +++ b/go/test/endtoend/preparestmt/sSchema.sql @@ -0,0 +1,10 @@ +CREATE TABLE t1 +( + id BIGINT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + age INT, + email VARCHAR(100), + created_at DATETIME, + is_active BOOLEAN, + INDEX age_idx (age) +); \ No newline at end of file diff --git a/go/test/endtoend/preparestmt/sVschema.json b/go/test/endtoend/preparestmt/sVschema.json new file mode 100644 index 00000000000..77801827d42 --- /dev/null +++ b/go/test/endtoend/preparestmt/sVschema.json @@ -0,0 +1,22 @@ +{ + "sharded": true, + "vindexes": { + "xxhash": { + "type": "xxhash" + } + }, + "tables": { + "t1": { + "column_vindexes": [ + { + "column": "id", + "name": "xxhash" + } + ], + "auto_increment":{ + "column" : "id", + "sequence" : "uks.t1_seq" + } + } + } +} diff --git a/go/test/endtoend/preparestmt/stmt_methods_test.go b/go/test/endtoend/preparestmt/stmt_methods_test.go index 5768c6eec7a..e8e001fdca7 100644 --- a/go/test/endtoend/preparestmt/stmt_methods_test.go +++ b/go/test/endtoend/preparestmt/stmt_methods_test.go @@ -48,7 +48,7 @@ func TestSelectDatabase(t *testing.T) { require.True(t, rows.Next(), "no rows found") err = rows.Scan(&resultBytes) require.NoError(t, err) - assert.Equal(t, string(resultBytes), "test_keyspace") + assert.Equal(t, string(resultBytes), "uks") } // TestInsertUpdateDelete validates all insert, update and @@ -57,7 +57,7 @@ func TestInsertUpdateDelete(t *testing.T) { dbo := Connect(t) defer dbo.Close() // prepare insert statement - insertStmt := `insert into ` + tableName + ` values( ?, ?, ?, ?, ?, ?, ?, ?, + insertStmt := `insert into vt_prepare_stmt_test values( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);` @@ -129,7 +129,7 @@ func testReplica(t *testing.T) { // testcount validates inserted rows count with expected count. func testcount(t *testing.T, dbo *sql.DB, except int) { - r, err := dbo.Query("SELECT count(1) FROM " + tableName) + r, err := dbo.Query("SELECT count(1) FROM vt_prepare_stmt_test") require.Nil(t, err) r.Next() @@ -145,7 +145,7 @@ func TestAutoIncColumns(t *testing.T) { dbo := Connect(t) defer dbo.Close() // insert a row without id - insertStmt := "INSERT INTO " + tableName + ` ( + insertStmt := "INSERT INTO vt_prepare_stmt_test" + ` ( msg,keyspace_id,tinyint_unsigned,bool_signed,smallint_unsigned, mediumint_unsigned,int_unsigned,float_unsigned,double_unsigned, decimal_unsigned,t_date,t_datetime,t_datetime_micros,t_time,t_timestamp,c8,c16,c24, @@ -180,7 +180,7 @@ func TestAutoIncColumns(t *testing.T) { // deleteRecord test deletion operation corresponds to the testingID. func deleteRecord(t *testing.T, dbo *sql.DB) { // delete the record with id 1 - exec(t, dbo, "DELETE FROM "+tableName+" WHERE id = ?;", testingID) + exec(t, dbo, "DELETE FROM vt_prepare_stmt_test WHERE id = ?;", testingID) data := selectWhere(t, dbo, "id = ?", testingID) assert.Equal(t, 0, len(data)) @@ -192,7 +192,7 @@ func updateRecord(t *testing.T, dbo *sql.DB) { // update the record with id 1 updateData := "new data value" updateTextCol := "new text col value" - updateQuery := "update " + tableName + " set data = ? , text_col = ? where id = ?;" + updateQuery := "update vt_prepare_stmt_test set data = ? , text_col = ? where id = ?;" exec(t, dbo, updateQuery, updateData, updateTextCol, testingID) @@ -226,7 +226,7 @@ func TestColumnParameter(t *testing.T) { id := 1000 parameter1 := "param1" message := "TestColumnParameter" - insertStmt := "INSERT INTO " + tableName + " (id, msg, keyspace_id) VALUES (?, ?, ?);" + insertStmt := "INSERT INTO vt_prepare_stmt_test (id, msg, keyspace_id) VALUES (?, ?, ?);" values := []any{ id, message, @@ -237,7 +237,7 @@ func TestColumnParameter(t *testing.T) { var param, msg string var recID int - selectStmt := "SELECT COALESCE(?, id), msg FROM " + tableName + " WHERE msg = ? LIMIT ?" + selectStmt := "SELECT COALESCE(?, id), msg FROM vt_prepare_stmt_test WHERE msg = ? LIMIT ?" results1, err := dbo.Query(selectStmt, parameter1, message, 1) require.Nil(t, err) @@ -313,10 +313,7 @@ func TestSelectDBA(t *testing.T) { dbo := Connect(t) defer dbo.Close() - _, err := dbo.Exec("use uks") - require.NoError(t, err) - - _, err = dbo.Exec("CREATE TABLE `a` (`one` int NOT NULL,`two` int NOT NULL,PRIMARY KEY(`one`, `two`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4") + _, err := dbo.Exec("CREATE TABLE `a` (`one` int NOT NULL,`two` int NOT NULL,PRIMARY KEY(`one`, `two`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4") require.NoError(t, err) prepare, err := dbo.Prepare(`SELECT @@ -332,10 +329,10 @@ func TestSelectDBA(t *testing.T) { extra extra, table_name table_name FROM information_schema.columns - WHERE table_schema = ? + WHERE table_schema = ? and table_name = ? ORDER BY ordinal_position`) require.NoError(t, err) - rows, err := prepare.Query("uks") + rows, err := prepare.Query("uks", "a") require.NoError(t, err) defer rows.Close() var rec columns @@ -441,6 +438,6 @@ func TestBinaryColumn(t *testing.T) { AND column_info.table_schema = ? -- Exclude views. AND table_info.table_type = 'BASE TABLE' - ORDER BY BINARY table_info.table_name`, keyspaceName, keyspaceName) + ORDER BY BINARY table_info.table_name`, uks, uks) require.NoError(t, err) } diff --git a/go/test/endtoend/preparestmt/uSchema.sql b/go/test/endtoend/preparestmt/uSchema.sql new file mode 100644 index 00000000000..5284cacded6 --- /dev/null +++ b/go/test/endtoend/preparestmt/uSchema.sql @@ -0,0 +1,56 @@ +create table vt_prepare_stmt_test +( + id bigint auto_increment, + msg varchar(64), + keyspace_id bigint(20) unsigned NOT NULL, + tinyint_unsigned TINYINT, + bool_signed BOOL, + smallint_unsigned SMALLINT, + mediumint_unsigned MEDIUMINT, + int_unsigned INT, + float_unsigned FLOAT(10, 2), + double_unsigned DOUBLE(16, 2), + decimal_unsigned DECIMAL, + t_date DATE, + t_datetime DATETIME, + t_datetime_micros DATETIME(6), + t_time TIME, + t_timestamp TIMESTAMP, + c8 bit(8) DEFAULT NULL, + c16 bit(16) DEFAULT NULL, + c24 bit(24) DEFAULT NULL, + c32 bit(32) DEFAULT NULL, + c40 bit(40) DEFAULT NULL, + c48 bit(48) DEFAULT NULL, + c56 bit(56) DEFAULT NULL, + c63 bit(63) DEFAULT NULL, + c64 bit(64) DEFAULT NULL, + json_col JSON, + text_col TEXT, + data longblob, + tinyint_min TINYINT, + tinyint_max TINYINT, + tinyint_pos TINYINT, + tinyint_neg TINYINT, + smallint_min SMALLINT, + smallint_max SMALLINT, + smallint_pos SMALLINT, + smallint_neg SMALLINT, + medint_min MEDIUMINT, + medint_max MEDIUMINT, + medint_pos MEDIUMINT, + medint_neg MEDIUMINT, + int_min INT, + int_max INT, + int_pos INT, + int_neg INT, + bigint_min BIGINT, + bigint_max BIGINT, + bigint_pos BIGINT, + bigint_neg BIGINT, + primary key (id) +) Engine = InnoDB; + +CREATE TABLE t1_seq ( id INT, next_id BIGINT, cache BIGINT, PRIMARY KEY(id)) comment 'vitess_sequence'; + +INSERT INTO t1_seq (id, next_id, cache) values (0, 1, 500000); \ No newline at end of file diff --git a/go/test/endtoend/preparestmt/uVschema.json b/go/test/endtoend/preparestmt/uVschema.json new file mode 100644 index 00000000000..1f4de22e018 --- /dev/null +++ b/go/test/endtoend/preparestmt/uVschema.json @@ -0,0 +1,8 @@ +{ + "sharded": false, + "tables": { + "sequence_test_seq": { + "type": "sequence" + } + } +} diff --git a/go/test/endtoend/utils/cmp.go b/go/test/endtoend/utils/cmp.go index dd9614e79fa..b2e1eca03e9 100644 --- a/go/test/endtoend/utils/cmp.go +++ b/go/test/endtoend/utils/cmp.go @@ -215,6 +215,18 @@ func (mcmp *MySQLCompare) Exec(query string) *sqltypes.Result { return vtQr } +// ExecVitessAndMySQLDifferentQueries executes Vitess and MySQL with the queries provided. +func (mcmp *MySQLCompare) ExecVitessAndMySQLDifferentQueries(vtQ, mQ string) *sqltypes.Result { + mcmp.t.Helper() + vtQr, err := mcmp.VtConn.ExecuteFetch(vtQ, 1000, true) + require.NoError(mcmp.t, err, "[Vitess Error] for query: "+vtQ) + + mysqlQr, err := mcmp.MySQLConn.ExecuteFetch(mQ, 1000, true) + require.NoError(mcmp.t, err, "[MySQL Error] for query: "+mQ) + compareVitessAndMySQLResults(mcmp.t, vtQ, mcmp.VtConn, vtQr, mysqlQr, CompareOptions{}) + return vtQr +} + // ExecAssert is the same as Exec, but it only does assertions, it won't FailNow func (mcmp *MySQLCompare) ExecAssert(query string) *sqltypes.Result { mcmp.t.Helper() diff --git a/go/test/endtoend/vreplication/global_routing_test.go b/go/test/endtoend/vreplication/global_routing_test.go new file mode 100644 index 00000000000..667c6352e2e --- /dev/null +++ b/go/test/endtoend/vreplication/global_routing_test.go @@ -0,0 +1,296 @@ +/* +Copyright 2025 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "bytes" + "fmt" + "strings" + "testing" + "text/template" + "time" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + vttablet "vitess.io/vitess/go/vt/vttablet/common" +) + +type tgrTestConfig struct { + ksU1, ksU2, ksS1 string + ksU1Tables, ksU2Tables, ksS1Tables []string +} + +var grTestConfig = tgrTestConfig{ + ksU1: "unsharded1", + ksU2: "unsharded2", + ksS1: "sharded1", + ksU1Tables: []string{"t1", "t2", "t3"}, + ksU2Tables: []string{"t2", "t4", "t5"}, + ksS1Tables: []string{"t2", "t4", "t6"}, +} + +type grTestExpectations struct { + postKsU1, postKsU2, postKsS1 func(t *testing.T) +} + +// Scope helpers to this test file so we don't pollute the global namespace. +type grHelpers struct { + t *testing.T +} + +func (h *grHelpers) getSchema(tables []string) string { + var createSQL string + for _, table := range tables { + createSQL += fmt.Sprintf("CREATE TABLE %s (id int primary key, val varchar(32)) ENGINE=InnoDB;\n", table) + } + return createSQL +} + +func (h *grHelpers) getShardedVSchema(tables []string) string { + const vSchemaTmpl = `{ + "sharded": true, + "vindexes": { + "reverse_bits": { + "type": "reverse_bits" + } + }, + "tables": { + {{- range $i, $table := .Tables}} + {{- if gt $i 0}},{{end}} + "{{ $table }}": { + "column_vindexes": [ + { + "column": "id", + "name": "reverse_bits" + } + ] + } + {{- end}} + } +} +` + type VSchemaData struct { + Tables []string + } + tmpl, err := template.New("vschema").Parse(vSchemaTmpl) + require.NoError(h.t, err) + var buf bytes.Buffer + err = tmpl.Execute(&buf, VSchemaData{tables}) + require.NoError(h.t, err) + return buf.String() +} + +func (h *grHelpers) insertData(t *testing.T, keyspace string, table string, id int, val string) { + vtgateConn, cancel := getVTGateConn() + defer cancel() + _, err := vtgateConn.ExecuteFetch(fmt.Sprintf("insert into %s.%s(id, val) values(%d, '%s')", + keyspace, table, id, val), 1, false) + require.NoError(t, err) +} + +// There is a race between when a table is created and it is updated in the global table cache in vtgate. +// This function waits for the table to be available in vtgate before proceeding. +func (h *grHelpers) waitForTableAvailability(t *testing.T, vtgateConn *mysql.Conn, table string) { + timer := time.NewTimer(defaultTimeout) + defer timer.Stop() + for { + _, err := vtgateConn.ExecuteFetch(fmt.Sprintf("select * from %s", table), 1, false) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("table %s not found", table)) { + return + } + select { + case <-timer.C: + require.FailNow(t, "timed out waiting for table availability for %s", table) + default: + time.Sleep(defaultTick) + } + } +} + +// Check for the expected global routing behavior for the given tables. Expected logic is implemented in the callback. +func (h *grHelpers) checkForTable( + t *testing.T, + tables []string, + queryCallback func(rs *sqltypes.Result, err error), +) { + vtgateConn, cancel := getVTGateConn() + defer cancel() + + for _, table := range tables { + for _, target := range []string{"", "@primary"} { + _, err := vtgateConn.ExecuteFetch(fmt.Sprintf("use %s", target), 1, false) + require.NoError(t, err) + h.waitForTableAvailability(t, vtgateConn, table) + rs, err := vtgateConn.ExecuteFetch(fmt.Sprintf("select * from %s", table), 1, false) + queryCallback(rs, err) + } + } +} + +func (h *grHelpers) isGlobal(t *testing.T, tables []string, expectedVal string) bool { + asExpected := true + + h.checkForTable(t, tables, func(rs *sqltypes.Result, err error) { + require.NoError(t, err) + gotVal := rs.Rows[0][1].ToString() + if gotVal != expectedVal { + asExpected = false + } + }) + + return asExpected +} + +func (h *grHelpers) isAmbiguous(t *testing.T, tables []string) bool { + asExpected := true + + h.checkForTable(t, tables, func(rs *sqltypes.Result, err error) { + if err == nil || !strings.Contains(err.Error(), "ambiguous") { + asExpected = false + } + }) + + return asExpected +} + +// getExpectations returns a map of expectations for global routing tests. The key is a boolean indicating whether +// the unsharded keyspace has a vschema. The value is a struct containing callbacks for verifying the global routing +// behavior after each keyspace is added. +func (h *grHelpers) getExpectations() *map[bool]*grTestExpectations { + var exp = make(map[bool]*grTestExpectations) + exp[false] = &grTestExpectations{ + postKsU1: func(t *testing.T) { + require.True(t, h.isGlobal(t, []string{"t1", "t2", "t3"}, grTestConfig.ksU1)) + }, + postKsU2: func(t *testing.T) { + require.True(t, h.isGlobal(t, []string{"t1", "t3"}, grTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t4", "t5"}, grTestConfig.ksU2)) + require.True(t, h.isAmbiguous(t, []string{"t2"})) + }, + postKsS1: func(t *testing.T) { + require.True(t, h.isGlobal(t, []string{"t2", "t4"}, grTestConfig.ksS1)) + require.True(t, h.isGlobal(t, []string{"t1", "t3"}, grTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t5"}, grTestConfig.ksU2)) + require.True(t, h.isGlobal(t, []string{"t6"}, grTestConfig.ksS1)) + }, + } + exp[true] = &grTestExpectations{ + postKsU1: func(t *testing.T) { + require.True(t, h.isGlobal(t, []string{"t1", "t2", "t3"}, grTestConfig.ksU1)) + }, + postKsU2: func(t *testing.T) { + require.True(t, h.isGlobal(t, []string{"t1", "t3"}, grTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t4", "t5"}, grTestConfig.ksU2)) + require.True(t, h.isAmbiguous(t, []string{"t2"})) + }, + postKsS1: func(t *testing.T) { + require.True(t, h.isAmbiguous(t, []string{"t2", "t4"})) + require.True(t, h.isGlobal(t, []string{"t1", "t3"}, grTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t5"}, grTestConfig.ksU2)) + }, + } + return &exp +} + +func (h *grHelpers) getUnshardedVschema(unshardedHasVSchema bool, tables []string) string { + if !unshardedHasVSchema { + return "" + } + vschema := `{"tables": {` + for i, table := range tables { + if i != 0 { + vschema += `,` + } + vschema += fmt.Sprintf(`"%s": {}`, table) + } + vschema += `}}` + return vschema +} + +func (h *grHelpers) rebuildGraphs(t *testing.T, keyspaces []string) { + var err error + for _, ks := range keyspaces { + err = vc.VtctldClient.ExecuteCommand("RebuildKeyspaceGraph", ks) + require.NoError(t, err) + } + require.NoError(t, err) + err = vc.VtctldClient.ExecuteCommand("RebuildVSchemaGraph") + require.NoError(t, err) +} + +// TestGlobalRouting tests global routing for unsharded and sharded keyspaces by setting up keyspaces +// with different table configurations and verifying that the tables are globally routed +// by querying via vtgate. +func TestGlobalRouting(t *testing.T) { + h := grHelpers{t} + exp := *h.getExpectations() + for unshardedHasVSchema, funcs := range exp { + require.NotNil(t, funcs) + testGlobalRouting(t, unshardedHasVSchema, funcs) + } +} + +func testGlobalRouting(t *testing.T, unshardedHasVSchema bool, funcs *grTestExpectations) { + h := grHelpers{t: t} + setSidecarDBName("_vt") + vttablet.InitVReplicationConfigDefaults() + + vc = NewVitessCluster(t, nil) + defer vc.TearDown() + zone1 := vc.Cells["zone1"] + config := grTestConfig + vc.AddKeyspace(t, []*Cell{zone1}, config.ksU1, "0", h.getUnshardedVschema(unshardedHasVSchema, config.ksU1Tables), + h.getSchema(config.ksU1Tables), 1, 0, 100, nil) + verifyClusterHealth(t, vc) + for _, table := range config.ksU1Tables { + h.insertData(t, config.ksU1, table, 1, config.ksU1) + vtgateConn, cancel := getVTGateConn() + waitForRowCount(t, vtgateConn, config.ksU1+"@replica", table, 1) + cancel() + } + keyspaces := []string{config.ksU1} + h.rebuildGraphs(t, keyspaces) + funcs.postKsU1(t) + + vc.AddKeyspace(t, []*Cell{zone1}, config.ksU2, "0", h.getUnshardedVschema(unshardedHasVSchema, config.ksU2Tables), + h.getSchema(config.ksU2Tables), 1, 0, 200, nil) + verifyClusterHealth(t, vc) + for _, table := range config.ksU2Tables { + h.insertData(t, config.ksU2, table, 1, config.ksU2) + vtgateConn, cancel := getVTGateConn() + waitForRowCount(t, vtgateConn, config.ksU2+"@replica", table, 1) + cancel() + } + keyspaces = append(keyspaces, config.ksU2) + h.rebuildGraphs(t, keyspaces) + funcs.postKsU2(t) + + vc.AddKeyspace(t, []*Cell{zone1}, config.ksS1, "-80,80-", h.getShardedVSchema(config.ksS1Tables), h.getSchema(config.ksS1Tables), + 1, 0, 300, nil) + verifyClusterHealth(t, vc) + for _, table := range config.ksS1Tables { + h.insertData(t, config.ksS1, table, 1, config.ksS1) + vtgateConn, cancel := getVTGateConn() + waitForRowCount(t, vtgateConn, config.ksS1+"@replica", table, 1) + cancel() + } + keyspaces = append(keyspaces, config.ksS1) + h.rebuildGraphs(t, keyspaces) + funcs.postKsS1(t) +} diff --git a/go/test/endtoend/vreplication/helper_test.go b/go/test/endtoend/vreplication/helper_test.go index 0102b9b5e2d..6bfce05119a 100644 --- a/go/test/endtoend/vreplication/helper_test.go +++ b/go/test/endtoend/vreplication/helper_test.go @@ -368,29 +368,33 @@ func waitForWorkflowState(t *testing.T, vc *VitessCluster, ksWorkflow string, wa require.NoError(t, err, output) done = true state := "" - streams := gjson.Get(output, "workflows.0.shard_streams.*.streams") - streams.ForEach(func(streamId, stream gjson.Result) bool { // For each stream - info := stream.Map() - // We need to wait for all streams to have the desired state. - state = info["state"].String() - if state == wantState { - for i := 0; i < len(fieldEqualityChecks); i++ { - if kvparts := strings.Split(fieldEqualityChecks[i], "=="); len(kvparts) == 2 { - key := kvparts[0] - val := kvparts[1] - res := info[key].String() - if !strings.EqualFold(res, val) { - done = false + shardStreams := gjson.Get(output, "workflows.0.shard_streams") + // We need to wait for all streams in all shard streams to have the desired state. + shardStreams.ForEach(func(shardStreamId, shardStream gjson.Result) bool { + streams := shardStream.Get("*") + streams.ForEach(func(streamId, stream gjson.Result) bool { + info := stream.Map() + state = info["state"].String() + if state == wantState { + for i := 0; i < len(fieldEqualityChecks); i++ { + if kvparts := strings.Split(fieldEqualityChecks[i], "=="); len(kvparts) == 2 { + key := kvparts[0] + val := kvparts[1] + res := info[key].String() + if !strings.EqualFold(res, val) { + done = false + } } } - } - if wantState == binlogdatapb.VReplicationWorkflowState_Running.String() && - (info["position"].Exists() && info["position"].String() == "") { + if wantState == binlogdatapb.VReplicationWorkflowState_Running.String() && + (info["position"].Exists() && info["position"].String() == "") { + done = false + } + } else { done = false } - } else { - done = false - } + return true + }) return true }) if done { diff --git a/go/test/endtoend/vtgate/gen4/main_test.go b/go/test/endtoend/vtgate/gen4/main_test.go index e8280b3aa06..4012017b0c3 100644 --- a/go/test/endtoend/vtgate/gen4/main_test.go +++ b/go/test/endtoend/vtgate/gen4/main_test.go @@ -61,6 +61,10 @@ var ( } ]} ` + unsharded2Ks = "uks2" + + //go:embed unsharded2_schema.sql + unsharded2SchemaSQL string ) func TestMain(m *testing.M) { @@ -100,6 +104,17 @@ func TestMain(m *testing.M) { return 1 } + // This keyspace is used to test automatic addition of tables to global routing rules when + // there are multiple unsharded keyspaces. + uKs2 := &cluster.Keyspace{ + Name: unsharded2Ks, + SchemaSQL: unsharded2SchemaSQL, + } + err = clusterInstance.StartUnshardedKeyspace(*uKs2, 0, false) + if err != nil { + return 1 + } + // apply routing rules err = clusterInstance.VtctldClientProcess.ApplyRoutingRules(routingRules) if err != nil { diff --git a/go/test/endtoend/vtgate/gen4/unsharded2_schema.sql b/go/test/endtoend/vtgate/gen4/unsharded2_schema.sql new file mode 100644 index 00000000000..c3df5bd0a56 --- /dev/null +++ b/go/test/endtoend/vtgate/gen4/unsharded2_schema.sql @@ -0,0 +1,13 @@ +create table u2_a +( + id bigint, + a bigint, + primary key (id) +) Engine = InnoDB; + +create table u2_b +( + id bigint, + b varchar(50), + primary key (id) +) Engine = InnoDB; diff --git a/go/test/endtoend/vtgate/misc_test.go b/go/test/endtoend/vtgate/misc_test.go index bbcb338fa50..55fa139b290 100644 --- a/go/test/endtoend/vtgate/misc_test.go +++ b/go/test/endtoend/vtgate/misc_test.go @@ -814,6 +814,81 @@ func TestDDLTargeted(t *testing.T) { utils.AssertMatches(t, conn, `select id from ddl_targeted`, `[[INT64(1)]]`) } +// TestDynamicConfig tests the dynamic configurations. +func TestDynamicConfig(t *testing.T) { + t.Run("DiscoveryLowReplicationLag", func(t *testing.T) { + // Test initial config value + err := clusterInstance.VtgateProcess.WaitForConfig(`"discovery_low_replication_lag":30000000000`) + require.NoError(t, err) + defer func() { + // Restore default back. + clusterInstance.VtgateProcess.Config.DiscoveryLowReplicationLag = "30s" + err = clusterInstance.VtgateProcess.RewriteConfiguration() + require.NoError(t, err) + }() + clusterInstance.VtgateProcess.Config.DiscoveryLowReplicationLag = "15s" + err = clusterInstance.VtgateProcess.RewriteConfiguration() + require.NoError(t, err) + // Test final config value. + err = clusterInstance.VtgateProcess.WaitForConfig(`"discovery_low_replication_lag":"15s"`) + require.NoError(t, err) + }) + + t.Run("DiscoveryHighReplicationLag", func(t *testing.T) { + // Test initial config value + err := clusterInstance.VtgateProcess.WaitForConfig(`"discovery_high_replication_lag":7200000000000`) + require.NoError(t, err) + defer func() { + // Restore default back. + clusterInstance.VtgateProcess.Config.DiscoveryHighReplicationLag = "2h" + err = clusterInstance.VtgateProcess.RewriteConfiguration() + require.NoError(t, err) + }() + clusterInstance.VtgateProcess.Config.DiscoveryHighReplicationLag = "1h" + err = clusterInstance.VtgateProcess.RewriteConfiguration() + require.NoError(t, err) + // Test final config value. + err = clusterInstance.VtgateProcess.WaitForConfig(`"discovery_high_replication_lag":"1h"`) + require.NoError(t, err) + }) + + t.Run("DiscoveryMinServingVttablets", func(t *testing.T) { + // Test initial config value + err := clusterInstance.VtgateProcess.WaitForConfig(`"discovery_min_number_serving_vttablets":2`) + require.NoError(t, err) + defer func() { + // Restore default back. + clusterInstance.VtgateProcess.Config.DiscoveryMinServingVttablets = "2" + err = clusterInstance.VtgateProcess.RewriteConfiguration() + require.NoError(t, err) + }() + clusterInstance.VtgateProcess.Config.DiscoveryMinServingVttablets = "1" + err = clusterInstance.VtgateProcess.RewriteConfiguration() + require.NoError(t, err) + // Test final config value. + err = clusterInstance.VtgateProcess.WaitForConfig(`"discovery_min_number_serving_vttablets":"1"`) + require.NoError(t, err) + }) + + t.Run("DiscoveryLegacyReplicationLagAlgo", func(t *testing.T) { + // Test initial config value + err := clusterInstance.VtgateProcess.WaitForConfig(`"discovery_legacy_replication_lag_algorithm":""`) + require.NoError(t, err) + defer func() { + // Restore default back. + clusterInstance.VtgateProcess.Config.DiscoveryLegacyReplicationLagAlgo = "true" + err = clusterInstance.VtgateProcess.RewriteConfiguration() + require.NoError(t, err) + }() + clusterInstance.VtgateProcess.Config.DiscoveryLegacyReplicationLagAlgo = "false" + err = clusterInstance.VtgateProcess.RewriteConfiguration() + require.NoError(t, err) + // Test final config value. + err = clusterInstance.VtgateProcess.WaitForConfig(`"discovery_legacy_replication_lag_algorithm":"false"`) + require.NoError(t, err) + }) +} + func TestLookupErrorMetric(t *testing.T) { conn, closer := start(t) defer closer() diff --git a/go/test/endtoend/vtgate/plan_tests/main_test.go b/go/test/endtoend/vtgate/plan_tests/main_test.go index 504ec3ffb26..2dc2e70120b 100644 --- a/go/test/endtoend/vtgate/plan_tests/main_test.go +++ b/go/test/endtoend/vtgate/plan_tests/main_test.go @@ -22,6 +22,7 @@ import ( "os" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" @@ -86,7 +87,7 @@ func TestMain(m *testing.M) { // TODO: (@GuptaManan100/@systay): Also run the tests with normalizer on. clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, "--normalize_queries=false", - "--schema_change_signal=false", + "--schema_change_signal=true", ) // Start vtgate @@ -178,7 +179,7 @@ func verifyTestExpectations(t *testing.T, pd engine.PrimitiveDescription, test p // 1. Verify that the Join primitive sees atleast 1 row on the left side. engine.WalkPrimitiveDescription(pd, func(description engine.PrimitiveDescription) { if description.OperatorType == "Join" { - require.NotZero(t, description.Inputs[0].RowsReceived[0]) + assert.NotZero(t, description.Inputs[0].RowsReceived[0]) } }) diff --git a/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go b/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go index 5c5447fe6b6..0068616c3b8 100644 --- a/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go +++ b/go/test/endtoend/vtgate/plan_tests/plan_e2e_test.go @@ -19,11 +19,22 @@ package plan_tests import ( "testing" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/vt/sqlparser" ) func TestE2ECases(t *testing.T) { - e2eTestCaseFiles := []string{"select_cases.json", "filter_cases.json", "dml_cases.json"} + err := utils.WaitForAuthoritative(t, "main", "source_of_ref", clusterInstance.VtgateProcess.ReadVSchema) + require.NoError(t, err) + + e2eTestCaseFiles := []string{ + "select_cases.json", + "filter_cases.json", + "dml_cases.json", + "reference_cases.json", + } mcmp, closer := start(t) defer closer() loadSampleData(t, mcmp) @@ -34,7 +45,11 @@ func TestE2ECases(t *testing.T) { if test.SkipE2E { mcmp.AsT().Skip(test.Query) } - mcmp.Exec(test.Query) + stmt, err := sqlparser.NewTestParser().Parse(test.Query) + require.NoError(mcmp.AsT(), err) + sqlparser.RemoveKeyspaceIgnoreSysSchema(stmt) + + mcmp.ExecVitessAndMySQLDifferentQueries(test.Query, sqlparser.String(stmt)) pd := utils.ExecTrace(mcmp.AsT(), mcmp.VtConn, test.Query) verifyTestExpectations(mcmp.AsT(), pd, test) if mcmp.VtConn.IsClosed() { diff --git a/go/test/endtoend/vtgate/queries/misc/main_test.go b/go/test/endtoend/vtgate/queries/misc/main_test.go index ee9be542634..536dfa7500a 100644 --- a/go/test/endtoend/vtgate/queries/misc/main_test.go +++ b/go/test/endtoend/vtgate/queries/misc/main_test.go @@ -95,7 +95,7 @@ func TestMain(m *testing.M) { vtParams = clusterInstance.GetVTParams(keyspaceName) // create mysql instance and connection parameters - conn, closer, err := utils.NewMySQL(clusterInstance, keyspaceName, schemaSQL) + conn, closer, err := utils.NewMySQL(clusterInstance, keyspaceName, schemaSQL, uschemaSQL) if err != nil { fmt.Println(err) return 1 diff --git a/go/test/endtoend/vtgate/queries/misc/misc_test.go b/go/test/endtoend/vtgate/queries/misc/misc_test.go index 9f9860bd0e0..7ab0fe7ef54 100644 --- a/go/test/endtoend/vtgate/queries/misc/misc_test.go +++ b/go/test/endtoend/vtgate/queries/misc/misc_test.go @@ -25,6 +25,7 @@ import ( "time" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/sqlparser" _ "github.com/go-sql-driver/mysql" "github.com/stretchr/testify/assert" @@ -163,6 +164,8 @@ func TestSetAndGetLastInsertID(t *testing.T) { "update t1 set id2 = last_insert_id(%d) where id1 = 2", "update t1 set id2 = 88 where id1 = last_insert_id(%d)", "delete from t1 where id1 = last_insert_id(%d)", + "select id2, last_insert_id(count(*)) from t1 where %d group by id2", + "set @x = last_insert_id(%d)", } for _, workload := range []string{"olap", "oltp"} { @@ -175,7 +178,7 @@ func TestSetAndGetLastInsertID(t *testing.T) { require.NoError(t, err) } - // Insert a row for UPDATE tests + // Insert a few rows for UPDATE tests mcmp.Exec("insert into t1 (id1, id2) values (1, 10)") for _, query := range queries { @@ -186,6 +189,96 @@ func TestSetAndGetLastInsertID(t *testing.T) { } } +func TestSetAndGetLastInsertIDWithInsertUnsharded(t *testing.T) { + mcmp, closer := start(t) + defer closer() + + tests := []string{ + "insert into uks.unsharded(id1, id2) values (last_insert_id(%d),12)", + "insert into uks.unsharded(id1, id2) select last_insert_id(%d), 453", + } + + i := 0 + getVal := func() int { + defer func() { i++ }() + return i + } + + runTests := func(mcmp *utils.MySQLCompare) { + for _, test := range tests { + + lastInsertID := getVal() + query := fmt.Sprintf(test, lastInsertID) + + stmt, err := sqlparser.NewTestParser().Parse(query) + require.NoError(mcmp.AsT(), err) + sqlparser.RemoveKeyspaceIgnoreSysSchema(stmt) + + mcmp.ExecVitessAndMySQLDifferentQueries(query, sqlparser.String(stmt)) + mcmp.Exec("select last_insert_id()") + } + } + + for _, workload := range []string{"olap", "oltp"} { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { + _, err := mcmp.VtConn.ExecuteFetch("set workload = "+workload, 1, false) + require.NoError(t, err) + runTests(mcmp) + + // run the queries again, but inside a transaction this time + mcmp.Exec("begin") + runTests(mcmp) + mcmp.Exec("commit") + }) + } + + // Now test to set the last insert id to 0, see that it has changed correctly even if the value is 0 + mcmp.ExecVitessAndMySQLDifferentQueries( + "insert into uks.unsharded(id1, id2) values (last_insert_id(0),12)", + "insert into unsharded(id1, id2) values (last_insert_id(0),12)", + ) + mcmp.Exec("select last_insert_id()") +} + +func TestSetAndGetLastInsertIDWithInsert(t *testing.T) { + mcmp, closer := start(t) + defer closer() + + tests := []string{ + "insert into t1(id1, id2) values (last_insert_id(%d) ,%d)", + "insert into t1(id1, id2) values (%d, last_insert_id(%d))", + "insert into t1(id1, id2) select last_insert_id(%d), %d", + "insert into t1(id1, id2) select last_insert_id(id1+%d), 12 from t1 where 1 > %d", + } + + i := 0 + getVal := func() int { + defer func() { i++ }() + return i + } + + runTests := func(mcmp *utils.MySQLCompare) { + for _, test := range tests { + query := fmt.Sprintf(test, getVal(), getVal()) + mcmp.Exec(query) + mcmp.Exec("select last_insert_id()") + } + } + + for _, workload := range []string{"olap", "oltp"} { + mcmp.Run(workload, func(mcmp *utils.MySQLCompare) { + _, err := mcmp.VtConn.ExecuteFetch("set workload = "+workload, 1, false) + require.NoError(t, err) + runTests(mcmp) + + // run the queries again, but inside a transaction this time + mcmp.Exec("begin") + runTests(mcmp) + mcmp.Exec("commit") + }) + } +} + // TestVindexHints tests that vindex hints work as intended. func TestVindexHints(t *testing.T) { mcmp, closer := start(t) diff --git a/go/test/endtoend/vtorc/readtopologyinstance/main_test.go b/go/test/endtoend/vtorc/readtopologyinstance/main_test.go index 419a2e843c3..c58e8e9bb45 100644 --- a/go/test/endtoend/vtorc/readtopologyinstance/main_test.go +++ b/go/test/endtoend/vtorc/readtopologyinstance/main_test.go @@ -86,7 +86,7 @@ func TestReadTopologyInstanceBufferable(t *testing.T) { assert.Equal(t, "ON", primaryInstance.GTIDMode) assert.Equal(t, "FULL", primaryInstance.BinlogRowImage) assert.Contains(t, primaryInstance.SelfBinlogCoordinates.LogFile, fmt.Sprintf("vt-0000000%d-bin", primary.TabletUID)) - assert.Greater(t, primaryInstance.SelfBinlogCoordinates.LogPos, uint32(0)) + assert.Greater(t, primaryInstance.SelfBinlogCoordinates.LogPos, uint64(0)) assert.True(t, primaryInstance.SemiSyncPrimaryEnabled) assert.True(t, primaryInstance.SemiSyncReplicaEnabled) assert.True(t, primaryInstance.SemiSyncPrimaryStatus) @@ -138,7 +138,7 @@ func TestReadTopologyInstanceBufferable(t *testing.T) { assert.Equal(t, utils.Hostname, replicaInstance.SourceHost) assert.Equal(t, primary.MySQLPort, replicaInstance.SourcePort) assert.Contains(t, replicaInstance.SelfBinlogCoordinates.LogFile, fmt.Sprintf("vt-0000000%d-bin", replica.TabletUID)) - assert.Greater(t, replicaInstance.SelfBinlogCoordinates.LogPos, uint32(0)) + assert.Greater(t, replicaInstance.SelfBinlogCoordinates.LogPos, uint64(0)) assert.False(t, replicaInstance.SemiSyncPrimaryEnabled) assert.True(t, replicaInstance.SemiSyncReplicaEnabled) assert.False(t, replicaInstance.SemiSyncPrimaryStatus) @@ -156,11 +156,11 @@ func TestReadTopologyInstanceBufferable(t *testing.T) { assert.True(t, replicaInstance.ReplicationIOThreadRuning) assert.True(t, replicaInstance.ReplicationSQLThreadRuning) assert.Equal(t, replicaInstance.ReadBinlogCoordinates.LogFile, primaryInstance.SelfBinlogCoordinates.LogFile) - assert.Greater(t, replicaInstance.ReadBinlogCoordinates.LogPos, uint32(0)) + assert.Greater(t, replicaInstance.ReadBinlogCoordinates.LogPos, uint64(0)) assert.Equal(t, replicaInstance.ExecBinlogCoordinates.LogFile, primaryInstance.SelfBinlogCoordinates.LogFile) - assert.Greater(t, replicaInstance.ExecBinlogCoordinates.LogPos, uint32(0)) + assert.Greater(t, replicaInstance.ExecBinlogCoordinates.LogPos, uint64(0)) assert.Contains(t, replicaInstance.RelaylogCoordinates.LogFile, fmt.Sprintf("vt-0000000%d-relay", replica.TabletUID)) - assert.Greater(t, replicaInstance.RelaylogCoordinates.LogPos, uint32(0)) + assert.Greater(t, replicaInstance.RelaylogCoordinates.LogPos, uint64(0)) assert.Empty(t, replicaInstance.LastIOError) assert.Empty(t, replicaInstance.LastSQLError) assert.EqualValues(t, 0, replicaInstance.SQLDelay) diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index 9592440196a..7814be8ca83 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -28,10 +28,9 @@ import ( ) var ( - configKey = viperutil.KeyPrefixFunc("discovery") // lowReplicationLag defines the duration that replication lag is low enough that the VTTablet is considered healthy. lowReplicationLag = viperutil.Configure( - configKey("low_replication_lag"), + "discovery_low_replication_lag", viperutil.Options[time.Duration]{ FlagName: "discovery_low_replication_lag", Default: 30 * time.Second, @@ -39,7 +38,7 @@ var ( }, ) highReplicationLagMinServing = viperutil.Configure( - configKey("high_replication_lag"), + "discovery_high_replication_lag", viperutil.Options[time.Duration]{ FlagName: "discovery_high_replication_lag_minimum_serving", Default: 2 * time.Hour, @@ -47,7 +46,7 @@ var ( }, ) minNumTablets = viperutil.Configure( - configKey("min_number_serving_vttablets"), + "discovery_min_number_serving_vttablets", viperutil.Options[int]{ FlagName: "min_number_serving_vttablets", Default: 2, @@ -55,7 +54,7 @@ var ( }, ) legacyReplicationLagAlgorithm = viperutil.Configure( - configKey("legacy_replication_lag_algorithm"), + "discovery_legacy_replication_lag_algorithm", viperutil.Options[bool]{ FlagName: "legacy_replication_lag_algorithm", Default: true, diff --git a/go/vt/schemadiff/key.go b/go/vt/schemadiff/key.go index 865073a5a98..97f3af1630c 100644 --- a/go/vt/schemadiff/key.go +++ b/go/vt/schemadiff/key.go @@ -68,6 +68,16 @@ func (i *IndexDefinitionEntity) IsUnique() bool { return i.IndexDefinition.Info.IsUnique() } +// HasExpression returns true if the index uses an expression, e.g. `KEY idx1 ((id + 1))`. +func (i *IndexDefinitionEntity) HasExpression() bool { + for _, col := range i.IndexDefinition.Columns { + if col.Expression != nil { + return true + } + } + return false +} + // HasNullable returns true if any of the columns in the index are nullable. func (i *IndexDefinitionEntity) HasNullable() bool { for _, col := range i.ColumnList.Entities { diff --git a/go/vt/schemadiff/onlineddl.go b/go/vt/schemadiff/onlineddl.go index f02ccb1224d..06f3384c8fd 100644 --- a/go/vt/schemadiff/onlineddl.go +++ b/go/vt/schemadiff/onlineddl.go @@ -162,6 +162,11 @@ func PrioritizedUniqueKeys(createTableEntity *CreateTableEntity) *IndexDefinitio if !key.IsUnique() { continue } + if key.HasExpression() { + // If the key has an expression this means it unreliably covers the columns, + // we cannot trust it. + continue + } uniqueKeys = append(uniqueKeys, key) } sort.SliceStable(uniqueKeys, func(i, j int) bool { diff --git a/go/vt/schemadiff/onlineddl_test.go b/go/vt/schemadiff/onlineddl_test.go index 834490dca1b..f5309b4f943 100644 --- a/go/vt/schemadiff/onlineddl_test.go +++ b/go/vt/schemadiff/onlineddl_test.go @@ -932,6 +932,11 @@ func TestRevertible(t *testing.T) { toSchema: `id int primary key, e1 set('a', 'b'), e2 set('a'), e3 set('a', 'b', 'c'), e4 set('a', 'x'), e5 set('a', 'x', 'b'), e6 set('b'), e7 varchar(1), e8 tinyint`, expandedColumnNames: `e3,e4,e5,e6,e7,e8`, }, + { + name: "index with expression", + fromSchema: "id int, primary key (id), key idx1 ((id + 1))", + toSchema: "id int, primary key (id), key idx2 ((id + 2))", + }, } var ( diff --git a/go/vt/schemadiff/table.go b/go/vt/schemadiff/table.go index e002ef18e15..e9bb35cb3bd 100644 --- a/go/vt/schemadiff/table.go +++ b/go/vt/schemadiff/table.go @@ -500,9 +500,14 @@ func (c *CreateTableEntity) IndexDefinitionEntities() []*IndexDefinitionEntity { keys := c.CreateTable.TableSpec.Indexes entities := make([]*IndexDefinitionEntity, len(keys)) for i, key := range keys { - colEntities := make([]*ColumnDefinitionEntity, len(key.Columns)) - for i, keyCol := range key.Columns { - colEntities[i] = colMap[keyCol.Column.Lowered()] + colEntities := []*ColumnDefinitionEntity{} + for _, keyCol := range key.Columns { + colEntity, ok := colMap[keyCol.Column.Lowered()] + if !ok { + // This can happen if the index is on an expression, e.g. `KEY idx1 ((id + 1))`. + continue + } + colEntities = append(colEntities, colEntity) } entities[i] = NewIndexDefinitionEntity(c.Env, key, NewColumnDefinitionEntityList(colEntities)) } diff --git a/go/vt/schemadiff/table_test.go b/go/vt/schemadiff/table_test.go index 84c40d769c2..ac871dbd4af 100644 --- a/go/vt/schemadiff/table_test.go +++ b/go/vt/schemadiff/table_test.go @@ -891,6 +891,18 @@ func TestCreateTableDiff(t *testing.T) { "+ KEY `i_idx` (`i`) INVISIBLE", }, }, + { + name: "keys with expression", + from: "create table t1 (id int, primary key (id), key idx1 ((id + 1)))", + to: "create table t1 (id int, primary key (id), key idx2 ((id + 2)))", + diff: "alter table t1 drop key idx1, add key idx2 ((id + 2))", + cdiff: "ALTER TABLE `t1` DROP KEY `idx1`, ADD KEY `idx2` ((`id` + 2))", + textdiffs: []string{ + "- KEY `idx1` ((`id` + 1))", + "+ KEY `idx2` ((`id` + 2))", + }, + }, + // FULLTEXT keys { name: "add one fulltext key", @@ -2564,6 +2576,12 @@ func TestValidate(t *testing.T) { alter: "alter table t engine=innodb", expectErr: &DuplicateKeyNameError{Table: "t", Key: "PRIMARY"}, }, + { + name: "key with expression", + from: "create table t (id int, primary key (id), key idx1 ((id + 1)))", + alter: "alter table t add key idx2 ((id + 2))", + to: "create table t (id int, primary key (id), key idx1 ((id + 1)), key idx2 ((id + 2)))", + }, // partitions { name: "drop column used by partitions", diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index 836c824010d..2891d532d16 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -2440,6 +2440,25 @@ func RemoveKeyspace(in SQLNode) { }) } +// RemoveKeyspaceIgnoreSysSchema removes the Qualifier.Qualifier on all ColNames and Qualifier on all TableNames in the AST +// except for the system schema. +func RemoveKeyspaceIgnoreSysSchema(in SQLNode) { + Rewrite(in, nil, func(cursor *Cursor) bool { + switch expr := cursor.Node().(type) { + case *ColName: + if expr.Qualifier.Qualifier.NotEmpty() && !SystemSchema(expr.Qualifier.Qualifier.String()) { + expr.Qualifier.Qualifier = NewIdentifierCS("") + } + case TableName: + if expr.Qualifier.NotEmpty() && !SystemSchema(expr.Qualifier.String()) { + expr.Qualifier = NewIdentifierCS("") + cursor.Replace(expr) + } + } + return true + }) +} + func convertStringToInt(integer string) int { val, _ := strconv.Atoi(integer) return val diff --git a/go/vt/sqlparser/ast_test.go b/go/vt/sqlparser/ast_test.go index f01b47cbd7b..c1484df7cc4 100644 --- a/go/vt/sqlparser/ast_test.go +++ b/go/vt/sqlparser/ast_test.go @@ -917,3 +917,11 @@ func TestCloneComments(t *testing.T) { assert.Equal(t, "b", val) } } + +func TestRemoveKeyspace(t *testing.T) { + stmt, err := NewTestParser().Parse("select 1 from uks.unsharded") + require.NoError(t, err) + RemoveKeyspaceIgnoreSysSchema(stmt) + + require.Equal(t, "select 1 from unsharded", String(stmt)) +} diff --git a/go/vt/topo/stats_conn_test.go b/go/vt/topo/stats_conn_test.go index 9bc1d51d9ed..9e48ec58407 100644 --- a/go/vt/topo/stats_conn_test.go +++ b/go/vt/topo/stats_conn_test.go @@ -29,8 +29,18 @@ import ( "vitess.io/vitess/go/vt/vterrors" ) +// testStatsConnReadSem is a semaphore for unit tests. +// It intentionally has a concurrency limit of '1' to +// allow semaphore contention in tests. var testStatsConnReadSem = semaphore.NewWeighted(1) +// testStatsConnStatsReset resets StatsConn-based stats. +func testStatsConnStatsReset() { + topoStatsConnErrors.ResetAll() + topoStatsConnReadWaitTimings.Reset() + topoStatsConnTimings.Reset() +} + // The fakeConn is a wrapper for a Conn that emits stats for every operation type fakeConn struct { v Version @@ -185,238 +195,216 @@ func (st *fakeConn) IsReadOnly() bool { } // createTestReadSemaphoreContention simulates semaphore contention on the test read semaphore. -func createTestReadSemaphoreContention(ctx context.Context, duration time.Duration) { +func createTestReadSemaphoreContention(ctx context.Context, duration time.Duration, semAcquiredChan chan struct{}) { if err := testStatsConnReadSem.Acquire(ctx, 1); err != nil { panic(err) } defer testStatsConnReadSem.Release(1) + semAcquiredChan <- struct{}{} time.Sleep(duration) } // TestStatsConnTopoListDir emits stats on ListDir func TestStatsConnTopoListDir(t *testing.T) { + testStatsConnStatsReset() + defer testStatsConnStatsReset() + conn := &fakeConn{} statsConn := NewStatsConn("global", conn, testStatsConnReadSem) ctx := context.Background() - go createTestReadSemaphoreContention(ctx, 100*time.Millisecond) + semAcquiredChan := make(chan struct{}) + go createTestReadSemaphoreContention(ctx, 100*time.Millisecond, semAcquiredChan) + <-semAcquiredChan statsConn.ListDir(ctx, "", true) - timingCounts := topoStatsConnTimings.Counts()["ListDir.global"] - if got, want := timingCounts, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["ListDir.global"]) + require.NotZero(t, topoStatsConnTimings.Time()) - waitTimingsCounts := topoStatsConnReadWaitTimings.Counts()["ListDir.global"] - if got := waitTimingsCounts; got != 1 { - t.Errorf("stats were not properly recorded: got = %d, want = 1", got) - } + require.Equal(t, int64(1), topoStatsConnReadWaitTimings.Counts()["ListDir.global"]) + require.NotZero(t, topoStatsConnReadWaitTimings.Time()) // error is zero before getting an error - errorCount := topoStatsConnErrors.Counts()["ListDir.global"] - if got, want := errorCount, int64(0); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Zero(t, topoStatsConnErrors.Counts()["ListDir.global"]) statsConn.ListDir(ctx, "error", true) // error stats gets emitted - errorCount = topoStatsConnErrors.Counts()["ListDir.global"] - if got, want := errorCount, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnErrors.Counts()["ListDir.global"]) } // TestStatsConnTopoCreate emits stats on Create func TestStatsConnTopoCreate(t *testing.T) { + testStatsConnStatsReset() + defer testStatsConnStatsReset() + conn := &fakeConn{} statsConn := NewStatsConn("global", conn, testStatsConnReadSem) ctx := context.Background() statsConn.Create(ctx, "", []byte{}) - timingCounts := topoStatsConnTimings.Counts()["Create.global"] - if got, want := timingCounts, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["Create.global"]) + require.NotZero(t, topoStatsConnTimings.Time()) + require.Zero(t, topoStatsConnReadWaitTimings.Time()) // error is zero before getting an error - errorCount := topoStatsConnErrors.Counts()["Create.global"] - if got, want := errorCount, int64(0); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Zero(t, topoStatsConnErrors.Counts()["Create.global"]) statsConn.Create(ctx, "error", []byte{}) // error stats gets emitted - errorCount = topoStatsConnErrors.Counts()["Create.global"] - if got, want := errorCount, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnErrors.Counts()["Create.global"]) } // TestStatsConnTopoUpdate emits stats on Update func TestStatsConnTopoUpdate(t *testing.T) { + testStatsConnStatsReset() + defer testStatsConnStatsReset() + conn := &fakeConn{} statsConn := NewStatsConn("global", conn, testStatsConnReadSem) ctx := context.Background() statsConn.Update(ctx, "", []byte{}, conn.v) - timingCounts := topoStatsConnTimings.Counts()["Update.global"] - if got, want := timingCounts, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["Update.global"]) + require.NotZero(t, topoStatsConnTimings.Time()) + require.Zero(t, topoStatsConnReadWaitTimings.Time()) // error is zero before getting an error - errorCount := topoStatsConnErrors.Counts()["Update.global"] - if got, want := errorCount, int64(0); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Zero(t, topoStatsConnErrors.Counts()["Update.global"]) statsConn.Update(ctx, "error", []byte{}, conn.v) // error stats gets emitted - errorCount = topoStatsConnErrors.Counts()["Update.global"] - if got, want := errorCount, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnErrors.Counts()["Update.global"]) } // TestStatsConnTopoGet emits stats on Get func TestStatsConnTopoGet(t *testing.T) { + testStatsConnStatsReset() + defer testStatsConnStatsReset() + conn := &fakeConn{} statsConn := NewStatsConn("global", conn, testStatsConnReadSem) ctx := context.Background() - go createTestReadSemaphoreContention(ctx, time.Millisecond*100) + semAcquiredChan := make(chan struct{}) + go createTestReadSemaphoreContention(ctx, time.Millisecond*100, semAcquiredChan) + <-semAcquiredChan statsConn.Get(ctx, "") - timingCounts := topoStatsConnTimings.Counts()["Get.global"] - if got, want := timingCounts, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["Get.global"]) + require.NotZero(t, topoStatsConnTimings.Time()) - waitTimingsCounts := topoStatsConnReadWaitTimings.Counts()["Get.global"] - if got := waitTimingsCounts; got != 1 { - t.Errorf("stats were not properly recorded: got = %d, want = 1", got) - } + require.Equal(t, int64(1), topoStatsConnReadWaitTimings.Counts()["Get.global"]) + require.NotZero(t, topoStatsConnReadWaitTimings.Time()) // error is zero before getting an error - errorCount := topoStatsConnErrors.Counts()["Get.global"] - if got, want := errorCount, int64(0); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Zero(t, topoStatsConnErrors.Counts()["Get.global"]) statsConn.Get(ctx, "error") // error stats gets emitted - errorCount = topoStatsConnErrors.Counts()["Get.global"] - if got, want := errorCount, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnErrors.Counts()["Get.global"]) } // TestStatsConnTopoDelete emits stats on Delete func TestStatsConnTopoDelete(t *testing.T) { + testStatsConnStatsReset() + defer testStatsConnStatsReset() + conn := &fakeConn{} statsConn := NewStatsConn("global", conn, testStatsConnReadSem) ctx := context.Background() statsConn.Delete(ctx, "", conn.v) - timingCounts := topoStatsConnTimings.Counts()["Delete.global"] - if got, want := timingCounts, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["Delete.global"]) + require.NotZero(t, topoStatsConnTimings.Time()) + require.Zero(t, topoStatsConnReadWaitTimings.Time()) // error is zero before getting an error - errorCount := topoStatsConnErrors.Counts()["Delete.global"] - if got, want := errorCount, int64(0); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Zero(t, topoStatsConnErrors.Counts()["Delete.global"]) statsConn.Delete(ctx, "error", conn.v) // error stats gets emitted - errorCount = topoStatsConnErrors.Counts()["Delete.global"] - if got, want := errorCount, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnErrors.Counts()["Delete.global"]) } // TestStatsConnTopoLock emits stats on Lock func TestStatsConnTopoLock(t *testing.T) { + testStatsConnStatsReset() + defer testStatsConnStatsReset() + conn := &fakeConn{} statsConn := NewStatsConn("global", conn, testStatsConnReadSem) ctx := context.Background() statsConn.Lock(ctx, "", "") - timingCounts := topoStatsConnTimings.Counts()["Lock.global"] - require.Equal(t, timingCounts, int64(1)) + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["Lock.global"]) + require.NotZero(t, topoStatsConnTimings.Time()) + require.Zero(t, topoStatsConnReadWaitTimings.Time()) statsConn.LockWithTTL(ctx, "", "", time.Second) - timingCounts = topoStatsConnTimings.Counts()["LockWithTTL.global"] - require.Equal(t, timingCounts, int64(1)) + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["LockWithTTL.global"]) statsConn.LockName(ctx, "", "") - timingCounts = topoStatsConnTimings.Counts()["LockName.global"] - require.Equal(t, timingCounts, int64(1)) + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["LockName.global"]) // Error is zero before getting an error. - errorCount := topoStatsConnErrors.Counts()["Lock.global"] - require.Equal(t, errorCount, int64(0)) + require.Zero(t, topoStatsConnErrors.Counts()["Lock.global"]) statsConn.Lock(ctx, "error", "") // Error stats gets emitted. - errorCount = topoStatsConnErrors.Counts()["Lock.global"] - require.Equal(t, errorCount, int64(1)) + require.Equal(t, int64(1), topoStatsConnErrors.Counts()["Lock.global"]) } // TestStatsConnTopoWatch emits stats on Watch func TestStatsConnTopoWatch(t *testing.T) { + testStatsConnStatsReset() + defer testStatsConnStatsReset() + conn := &fakeConn{} statsConn := NewStatsConn("global", conn, testStatsConnReadSem) ctx := context.Background() statsConn.Watch(ctx, "") - timingCounts := topoStatsConnTimings.Counts()["Watch.global"] - if got, want := timingCounts, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } - + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["Watch.global"]) + require.NotZero(t, topoStatsConnTimings.Time()) + require.Zero(t, topoStatsConnReadWaitTimings.Time()) } // TestStatsConnTopoNewLeaderParticipation emits stats on NewLeaderParticipation func TestStatsConnTopoNewLeaderParticipation(t *testing.T) { + testStatsConnStatsReset() + defer testStatsConnStatsReset() + conn := &fakeConn{} statsConn := NewStatsConn("global", conn, testStatsConnReadSem) _, _ = statsConn.NewLeaderParticipation("", "") - timingCounts := topoStatsConnTimings.Counts()["NewLeaderParticipation.global"] - if got, want := timingCounts, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["NewLeaderParticipation.global"]) + require.NotZero(t, topoStatsConnTimings.Time()) + require.Zero(t, topoStatsConnReadWaitTimings.Time()) // error is zero before getting an error - errorCount := topoStatsConnErrors.Counts()["NewLeaderParticipation.global"] - if got, want := errorCount, int64(0); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Zero(t, topoStatsConnErrors.Counts()["NewLeaderParticipation.global"]) _, _ = statsConn.NewLeaderParticipation("error", "") // error stats gets emitted - errorCount = topoStatsConnErrors.Counts()["NewLeaderParticipation.global"] - if got, want := errorCount, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnErrors.Counts()["NewLeaderParticipation.global"]) } // TestStatsConnTopoClose emits stats on Close func TestStatsConnTopoClose(t *testing.T) { + testStatsConnStatsReset() + defer testStatsConnStatsReset() + conn := &fakeConn{} statsConn := NewStatsConn("global", conn, testStatsConnReadSem) statsConn.Close() - timingCounts := topoStatsConnTimings.Counts()["Close.global"] - if got, want := timingCounts, int64(1); got != want { - t.Errorf("stats were not properly recorded: got = %d, want = %d", got, want) - } + require.Equal(t, int64(1), topoStatsConnTimings.Counts()["Close.global"]) + require.NotZero(t, topoStatsConnTimings.Time()) + require.Zero(t, topoStatsConnReadWaitTimings.Time()) } diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index 8123416eb41..f27851275b6 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "math" "slices" "sort" "strings" @@ -28,7 +27,6 @@ import ( "text/template" "time" - "github.com/google/uuid" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" "google.golang.org/grpc/codes" @@ -41,7 +39,6 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/concurrency" - "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" @@ -1249,287 +1246,6 @@ func (s *Server) ReshardCreate(ctx context.Context, req *vtctldatapb.ReshardCrea }) } -// VDiffCreate is part of the vtctlservicepb.VtctldServer interface. -// It passes on the request to the target primary tablets that are -// participating in the given workflow and VDiff. -func (s *Server) VDiffCreate(ctx context.Context, req *vtctldatapb.VDiffCreateRequest) (*vtctldatapb.VDiffCreateResponse, error) { - span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffCreate") - defer span.Finish() - - span.Annotate("keyspace", req.TargetKeyspace) - span.Annotate("workflow", req.Workflow) - span.Annotate("uuid", req.Uuid) - span.Annotate("source_cells", req.SourceCells) - span.Annotate("target_cells", req.TargetCells) - span.Annotate("tablet_types", req.TabletTypes) - span.Annotate("tables", req.Tables) - span.Annotate("auto_retry", req.AutoRetry) - span.Annotate("max_diff_duration", req.MaxDiffDuration) - if req.AutoStart != nil { - span.Annotate("auto_start", req.GetAutoStart()) - } - - var err error - req.Uuid = strings.TrimSpace(req.Uuid) - if req.Uuid == "" { // Generate a UUID - req.Uuid = uuid.New().String() - } else { // Validate UUID if provided - if err = uuid.Validate(req.Uuid); err != nil { - return nil, vterrors.Wrapf(err, "invalid UUID provided: %s", req.Uuid) - } - } - - tabletTypesStr := discovery.BuildTabletTypesString(req.TabletTypes, req.TabletSelectionPreference) - - if req.Limit == 0 { // This would produce no useful results - req.Limit = math.MaxInt64 - } - // This is a pointer so there's no ZeroValue in the message - // and an older v18 client will not provide it. - if req.MaxDiffDuration == nil { - req.MaxDiffDuration = &vttimepb.Duration{} - } - // The other vttime.Duration vars should not be nil as the - // client should always provide them, but we check anyway to - // be safe. - if req.FilteredReplicationWaitTime == nil { - // A value of 0 is not valid as the vdiff will never succeed. - req.FilteredReplicationWaitTime = &vttimepb.Duration{ - Seconds: int64(DefaultTimeout.Seconds()), - } - } - if req.WaitUpdateInterval == nil { - req.WaitUpdateInterval = &vttimepb.Duration{} - } - - autoStart := true - if req.AutoStart != nil { - autoStart = req.GetAutoStart() - } - - options := &tabletmanagerdatapb.VDiffOptions{ - PickerOptions: &tabletmanagerdatapb.VDiffPickerOptions{ - TabletTypes: tabletTypesStr, - SourceCell: strings.Join(req.SourceCells, ","), - TargetCell: strings.Join(req.TargetCells, ","), - }, - CoreOptions: &tabletmanagerdatapb.VDiffCoreOptions{ - Tables: strings.Join(req.Tables, ","), - AutoRetry: req.AutoRetry, - MaxRows: req.Limit, - TimeoutSeconds: req.FilteredReplicationWaitTime.Seconds, - MaxExtraRowsToCompare: req.MaxExtraRowsToCompare, - UpdateTableStats: req.UpdateTableStats, - MaxDiffSeconds: req.MaxDiffDuration.Seconds, - AutoStart: &autoStart, - }, - ReportOptions: &tabletmanagerdatapb.VDiffReportOptions{ - OnlyPks: req.OnlyPKs, - DebugQuery: req.DebugQuery, - MaxSampleRows: req.MaxReportSampleRows, - RowDiffColumnTruncateAt: req.RowDiffColumnTruncateAt, - }, - } - - tabletreq := &tabletmanagerdatapb.VDiffRequest{ - Keyspace: req.TargetKeyspace, - Workflow: req.Workflow, - Action: string(vdiff.CreateAction), - Options: options, - VdiffUuid: req.Uuid, - } - - ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) - if err != nil { - return nil, err - } - if ts.frozen { - return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "invalid VDiff run: writes have been already been switched for workflow %s.%s", - req.TargetKeyspace, req.Workflow) - } - - workflowStatus, err := s.getWorkflowStatus(ctx, req.TargetKeyspace, req.Workflow) - if err != nil { - return nil, err - } - if workflowStatus != binlogdatapb.VReplicationWorkflowState_Running { - s.Logger().Infof("Workflow %s.%s is not running, cannot start VDiff in state %s", req.TargetKeyspace, req.Workflow, workflowStatus) - return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, - "not all streams are running in workflow %s.%s", req.TargetKeyspace, req.Workflow) - } - - err = ts.ForAllTargets(func(target *MigrationTarget) error { - _, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) - return err - }) - if err != nil { - s.Logger().Errorf("Error executing vdiff create action: %v", err) - return nil, err - } - - return &vtctldatapb.VDiffCreateResponse{ - UUID: req.Uuid, - }, nil -} - -// VDiffDelete is part of the vtctlservicepb.VtctldServer interface. -func (s *Server) VDiffDelete(ctx context.Context, req *vtctldatapb.VDiffDeleteRequest) (*vtctldatapb.VDiffDeleteResponse, error) { - span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffDelete") - defer span.Finish() - - span.Annotate("keyspace", req.TargetKeyspace) - span.Annotate("workflow", req.Workflow) - span.Annotate("argument", req.Arg) - - tabletreq := &tabletmanagerdatapb.VDiffRequest{ - Keyspace: req.TargetKeyspace, - Workflow: req.Workflow, - Action: string(vdiff.DeleteAction), - ActionArg: req.Arg, - } - - ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) - if err != nil { - return nil, err - } - - err = ts.ForAllTargets(func(target *MigrationTarget) error { - _, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) - return err - }) - if err != nil { - s.Logger().Errorf("Error executing vdiff delete action: %v", err) - return nil, err - } - - return &vtctldatapb.VDiffDeleteResponse{}, nil -} - -// VDiffResume is part of the vtctlservicepb.VtctldServer interface. -func (s *Server) VDiffResume(ctx context.Context, req *vtctldatapb.VDiffResumeRequest) (*vtctldatapb.VDiffResumeResponse, error) { - span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffResume") - defer span.Finish() - - targetShards := req.GetTargetShards() - - span.Annotate("keyspace", req.TargetKeyspace) - span.Annotate("workflow", req.Workflow) - span.Annotate("uuid", req.Uuid) - span.Annotate("target_shards", targetShards) - - tabletreq := &tabletmanagerdatapb.VDiffRequest{ - Keyspace: req.TargetKeyspace, - Workflow: req.Workflow, - Action: string(vdiff.ResumeAction), - VdiffUuid: req.Uuid, - } - - ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) - if err != nil { - return nil, err - } - - if len(targetShards) > 0 { - if err := applyTargetShards(ts, targetShards); err != nil { - return nil, err - } - } - - err = ts.ForAllTargets(func(target *MigrationTarget) error { - _, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) - return err - }) - if err != nil { - s.Logger().Errorf("Error executing vdiff resume action: %v", err) - return nil, err - } - - return &vtctldatapb.VDiffResumeResponse{}, nil -} - -// VDiffShow is part of the vtctlservicepb.VtctldServer interface. -func (s *Server) VDiffShow(ctx context.Context, req *vtctldatapb.VDiffShowRequest) (*vtctldatapb.VDiffShowResponse, error) { - span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffShow") - defer span.Finish() - - span.Annotate("keyspace", req.TargetKeyspace) - span.Annotate("workflow", req.Workflow) - span.Annotate("argument", req.Arg) - - tabletreq := &tabletmanagerdatapb.VDiffRequest{ - Keyspace: req.TargetKeyspace, - Workflow: req.Workflow, - Action: string(vdiff.ShowAction), - ActionArg: req.Arg, - } - - ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) - if err != nil { - return nil, err - } - - output := &vdiffOutput{ - responses: make(map[string]*tabletmanagerdatapb.VDiffResponse, len(ts.targets)), - err: nil, - } - output.err = ts.ForAllTargets(func(target *MigrationTarget) error { - resp, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) - output.mu.Lock() - defer output.mu.Unlock() - output.responses[target.GetShard().ShardName()] = resp - return err - }) - if output.err != nil { - s.Logger().Errorf("Error executing vdiff show action: %v", output.err) - return nil, output.err - } - return &vtctldatapb.VDiffShowResponse{ - TabletResponses: output.responses, - }, nil -} - -// VDiffStop is part of the vtctlservicepb.VtctldServer interface. -func (s *Server) VDiffStop(ctx context.Context, req *vtctldatapb.VDiffStopRequest) (*vtctldatapb.VDiffStopResponse, error) { - span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffStop") - defer span.Finish() - - targetShards := req.GetTargetShards() - - span.Annotate("keyspace", req.TargetKeyspace) - span.Annotate("workflow", req.Workflow) - span.Annotate("uuid", req.Uuid) - span.Annotate("target_shards", targetShards) - - tabletreq := &tabletmanagerdatapb.VDiffRequest{ - Keyspace: req.TargetKeyspace, - Workflow: req.Workflow, - Action: string(vdiff.StopAction), - VdiffUuid: req.Uuid, - } - - ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) - if err != nil { - return nil, err - } - - if len(targetShards) > 0 { - if err := applyTargetShards(ts, targetShards); err != nil { - return nil, err - } - } - - err = ts.ForAllTargets(func(target *MigrationTarget) error { - _, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) - return err - }) - if err != nil { - s.Logger().Errorf("Error executing vdiff stop action: %v", err) - return nil, err - } - - return &vtctldatapb.VDiffStopResponse{}, nil -} - // WorkflowDelete is part of the vtctlservicepb.VtctldServer interface. // It passes on the request to the target primary tablets that are // participating in the given workflow. diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index 26d722f1de0..8bb4a06f23a 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -27,7 +27,6 @@ import ( "testing" "time" - "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/prototext" @@ -39,7 +38,6 @@ import ( "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vtenv" - "vitess.io/vitess/go/vt/vttablet/tabletmanager/vdiff" "vitess.io/vitess/go/vt/vttablet/tmclient" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" @@ -191,257 +189,6 @@ func TestCheckReshardingJournalExistsOnTablet(t *testing.T) { } } -// TestVDiffCreate performs some basic tests of the VDiffCreate function -// to ensure that it behaves as expected given a specific request. -func TestVDiffCreate(t *testing.T) { - ctx := context.Background() - workflowName := "wf1" - sourceKeyspace := &testKeyspace{ - KeyspaceName: "source", - ShardNames: []string{"0"}, - } - targetKeyspace := &testKeyspace{ - KeyspaceName: "target", - ShardNames: []string{"-80", "80-"}, - } - env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) - defer env.close() - - tests := []struct { - name string - req *vtctldatapb.VDiffCreateRequest - wantErr string - }{ - { - name: "no values", - req: &vtctldatapb.VDiffCreateRequest{}, - // We did not provide any keyspace or shard. - wantErr: "FindAllShardsInKeyspace() invalid keyspace name: UnescapeID err: invalid input identifier ''", - }, - { - name: "generated UUID", - req: &vtctldatapb.VDiffCreateRequest{ - TargetKeyspace: targetKeyspace.KeyspaceName, - Workflow: workflowName, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.wantErr == "" { - env.tmc.expectVRQueryResultOnKeyspaceTablets(targetKeyspace.KeyspaceName, &queryResult{ - query: "select vrepl_id, table_name, lastpk from _vt.copy_state where vrepl_id in (1) and id in (select max(id) from _vt.copy_state where vrepl_id in (1) group by vrepl_id, table_name)", - result: &querypb.QueryResult{}, - }) - } - got, err := env.ws.VDiffCreate(ctx, tt.req) - if tt.wantErr != "" { - require.EqualError(t, err, tt.wantErr) - return - } - require.NoError(t, err) - require.NotNil(t, got) - // Ensure that we always use a valid UUID. - err = uuid.Validate(got.UUID) - require.NoError(t, err) - }) - } -} - -func TestVDiffResume(t *testing.T) { - ctx := context.Background() - sourceKeyspace := &testKeyspace{ - KeyspaceName: "sourceks", - ShardNames: []string{"0"}, - } - targetKeyspace := &testKeyspace{ - KeyspaceName: "targetks", - ShardNames: []string{"-80", "80-"}, - } - workflow := "testwf" - uuid := uuid.New().String() - env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) - defer env.close() - - env.tmc.strict = true - action := string(vdiff.ResumeAction) - - tests := []struct { - name string - req *vtctldatapb.VDiffResumeRequest // vtctld requests - expectedVDiffRequests map[*topodatapb.Tablet]*vdiffRequestResponse // tablet requests - wantErr string - }{ - { - name: "basic resume", // Both target shards - req: &vtctldatapb.VDiffResumeRequest{ - TargetKeyspace: targetKeyspace.KeyspaceName, - Workflow: workflow, - Uuid: uuid, - }, - expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ - env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { - req: &tabletmanagerdatapb.VDiffRequest{ - Keyspace: targetKeyspace.KeyspaceName, - Workflow: workflow, - Action: action, - VdiffUuid: uuid, - }, - }, - env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID+tabletUIDStep]: { - req: &tabletmanagerdatapb.VDiffRequest{ - Keyspace: targetKeyspace.KeyspaceName, - Workflow: workflow, - Action: action, - VdiffUuid: uuid, - }, - }, - }, - }, - { - name: "resume on first shard", - req: &vtctldatapb.VDiffResumeRequest{ - TargetKeyspace: targetKeyspace.KeyspaceName, - TargetShards: targetKeyspace.ShardNames[:1], - Workflow: workflow, - Uuid: uuid, - }, - expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ - env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { - req: &tabletmanagerdatapb.VDiffRequest{ - Keyspace: targetKeyspace.KeyspaceName, - Workflow: workflow, - Action: action, - VdiffUuid: uuid, - }, - }, - }, - }, - { - name: "resume on invalid shard", - req: &vtctldatapb.VDiffResumeRequest{ - TargetKeyspace: targetKeyspace.KeyspaceName, - TargetShards: []string{"0"}, - Workflow: workflow, - Uuid: uuid, - }, - wantErr: fmt.Sprintf("specified target shard 0 not a valid target for workflow %s", workflow), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - for tab, vdr := range tt.expectedVDiffRequests { - env.tmc.expectVDiffRequest(tab, vdr) - } - got, err := env.ws.VDiffResume(ctx, tt.req) - if tt.wantErr != "" { - require.EqualError(t, err, tt.wantErr) - } else { - require.NoError(t, err) - require.NotNil(t, got) - } - env.tmc.confirmVDiffRequests(t) - }) - } -} - -func TestVDiffStop(t *testing.T) { - ctx := context.Background() - sourceKeyspace := &testKeyspace{ - KeyspaceName: "sourceks", - ShardNames: []string{"0"}, - } - targetKeyspace := &testKeyspace{ - KeyspaceName: "targetks", - ShardNames: []string{"-80", "80-"}, - } - workflow := "testwf" - uuid := uuid.New().String() - env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) - defer env.close() - - env.tmc.strict = true - action := string(vdiff.StopAction) - - tests := []struct { - name string - req *vtctldatapb.VDiffStopRequest // vtctld requests - expectedVDiffRequests map[*topodatapb.Tablet]*vdiffRequestResponse // tablet requests - wantErr string - }{ - { - name: "basic stop", // Both target shards - req: &vtctldatapb.VDiffStopRequest{ - TargetKeyspace: targetKeyspace.KeyspaceName, - Workflow: workflow, - Uuid: uuid, - }, - expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ - env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { - req: &tabletmanagerdatapb.VDiffRequest{ - Keyspace: targetKeyspace.KeyspaceName, - Workflow: workflow, - Action: action, - VdiffUuid: uuid, - }, - }, - env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID+tabletUIDStep]: { - req: &tabletmanagerdatapb.VDiffRequest{ - Keyspace: targetKeyspace.KeyspaceName, - Workflow: workflow, - Action: action, - VdiffUuid: uuid, - }, - }, - }, - }, - { - name: "stop on first shard", - req: &vtctldatapb.VDiffStopRequest{ - TargetKeyspace: targetKeyspace.KeyspaceName, - TargetShards: targetKeyspace.ShardNames[:1], - Workflow: workflow, - Uuid: uuid, - }, - expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ - env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { - req: &tabletmanagerdatapb.VDiffRequest{ - Keyspace: targetKeyspace.KeyspaceName, - Workflow: workflow, - Action: action, - VdiffUuid: uuid, - }, - }, - }, - }, - { - name: "stop on invalid shard", - req: &vtctldatapb.VDiffStopRequest{ - TargetKeyspace: targetKeyspace.KeyspaceName, - TargetShards: []string{"0"}, - Workflow: workflow, - Uuid: uuid, - }, - wantErr: fmt.Sprintf("specified target shard 0 not a valid target for workflow %s", workflow), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - for tab, vdr := range tt.expectedVDiffRequests { - env.tmc.expectVDiffRequest(tab, vdr) - } - got, err := env.ws.VDiffStop(ctx, tt.req) - if tt.wantErr != "" { - require.EqualError(t, err, tt.wantErr) - } else { - require.NoError(t, err) - require.NotNil(t, got) - } - env.tmc.confirmVDiffRequests(t) - }) - } -} - func TestMoveTablesComplete(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() diff --git a/go/vt/vtctl/workflow/vdiff.go b/go/vt/vtctl/workflow/vdiff.go index 6be5fe3c3b5..30953868b0b 100644 --- a/go/vt/vtctl/workflow/vdiff.go +++ b/go/vt/vtctl/workflow/vdiff.go @@ -17,16 +17,26 @@ limitations under the License. package workflow import ( + "context" "encoding/json" "math" "sort" "strings" "time" + "github.com/google/uuid" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/trace" + "vitess.io/vitess/go/vt/discovery" + "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/tabletmanager/vdiff" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + vttimepb "vitess.io/vitess/go/vt/proto/vttime" ) // TableSummary aggregates the current state of the table diff from all shards. @@ -278,3 +288,284 @@ func BuildProgressReport(rowsCompared int64, rowsToCompare int64, startedAt stri } return report } + +// VDiffCreate is part of the vtctlservicepb.VtctldServer interface. +// It passes on the request to the target primary tablets that are +// participating in the given workflow and VDiff. +func (s *Server) VDiffCreate(ctx context.Context, req *vtctldatapb.VDiffCreateRequest) (*vtctldatapb.VDiffCreateResponse, error) { + span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffCreate") + defer span.Finish() + + span.Annotate("keyspace", req.TargetKeyspace) + span.Annotate("workflow", req.Workflow) + span.Annotate("uuid", req.Uuid) + span.Annotate("source_cells", req.SourceCells) + span.Annotate("target_cells", req.TargetCells) + span.Annotate("tablet_types", req.TabletTypes) + span.Annotate("tables", req.Tables) + span.Annotate("auto_retry", req.AutoRetry) + span.Annotate("max_diff_duration", req.MaxDiffDuration) + if req.AutoStart != nil { + span.Annotate("auto_start", req.GetAutoStart()) + } + + var err error + req.Uuid = strings.TrimSpace(req.Uuid) + if req.Uuid == "" { // Generate a UUID + req.Uuid = uuid.New().String() + } else { // Validate UUID if provided + if err = uuid.Validate(req.Uuid); err != nil { + return nil, vterrors.Wrapf(err, "invalid UUID provided: %s", req.Uuid) + } + } + + tabletTypesStr := discovery.BuildTabletTypesString(req.TabletTypes, req.TabletSelectionPreference) + + if req.Limit == 0 { // This would produce no useful results + req.Limit = math.MaxInt64 + } + // This is a pointer so there's no ZeroValue in the message + // and an older v18 client will not provide it. + if req.MaxDiffDuration == nil { + req.MaxDiffDuration = &vttimepb.Duration{} + } + // The other vttime.Duration vars should not be nil as the + // client should always provide them, but we check anyway to + // be safe. + if req.FilteredReplicationWaitTime == nil { + // A value of 0 is not valid as the vdiff will never succeed. + req.FilteredReplicationWaitTime = &vttimepb.Duration{ + Seconds: int64(DefaultTimeout.Seconds()), + } + } + if req.WaitUpdateInterval == nil { + req.WaitUpdateInterval = &vttimepb.Duration{} + } + + autoStart := true + if req.AutoStart != nil { + autoStart = req.GetAutoStart() + } + + options := &tabletmanagerdatapb.VDiffOptions{ + PickerOptions: &tabletmanagerdatapb.VDiffPickerOptions{ + TabletTypes: tabletTypesStr, + SourceCell: strings.Join(req.SourceCells, ","), + TargetCell: strings.Join(req.TargetCells, ","), + }, + CoreOptions: &tabletmanagerdatapb.VDiffCoreOptions{ + Tables: strings.Join(req.Tables, ","), + AutoRetry: req.AutoRetry, + MaxRows: req.Limit, + TimeoutSeconds: req.FilteredReplicationWaitTime.Seconds, + MaxExtraRowsToCompare: req.MaxExtraRowsToCompare, + UpdateTableStats: req.UpdateTableStats, + MaxDiffSeconds: req.MaxDiffDuration.Seconds, + AutoStart: &autoStart, + }, + ReportOptions: &tabletmanagerdatapb.VDiffReportOptions{ + OnlyPks: req.OnlyPKs, + DebugQuery: req.DebugQuery, + MaxSampleRows: req.MaxReportSampleRows, + RowDiffColumnTruncateAt: req.RowDiffColumnTruncateAt, + }, + } + + tabletreq := &tabletmanagerdatapb.VDiffRequest{ + Keyspace: req.TargetKeyspace, + Workflow: req.Workflow, + Action: string(vdiff.CreateAction), + Options: options, + VdiffUuid: req.Uuid, + } + + ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) + if err != nil { + return nil, err + } + if ts.frozen { + return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "invalid VDiff run: writes have been already been switched for workflow %s.%s", + req.TargetKeyspace, req.Workflow) + } + + workflowStatus, err := s.getWorkflowStatus(ctx, req.TargetKeyspace, req.Workflow) + if err != nil { + return nil, err + } + if workflowStatus != binlogdatapb.VReplicationWorkflowState_Running { + s.Logger().Infof("Workflow %s.%s is not running, cannot start VDiff in state %s", req.TargetKeyspace, req.Workflow, workflowStatus) + return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, + "not all streams are running in workflow %s.%s", req.TargetKeyspace, req.Workflow) + } + + err = ts.ForAllTargets(func(target *MigrationTarget) error { + _, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) + return err + }) + if err != nil { + s.Logger().Errorf("Error executing vdiff create action: %v", err) + return nil, err + } + + return &vtctldatapb.VDiffCreateResponse{ + UUID: req.Uuid, + }, nil +} + +// VDiffDelete is part of the vtctlservicepb.VtctldServer interface. +func (s *Server) VDiffDelete(ctx context.Context, req *vtctldatapb.VDiffDeleteRequest) (*vtctldatapb.VDiffDeleteResponse, error) { + span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffDelete") + defer span.Finish() + + span.Annotate("keyspace", req.TargetKeyspace) + span.Annotate("workflow", req.Workflow) + span.Annotate("argument", req.Arg) + + tabletreq := &tabletmanagerdatapb.VDiffRequest{ + Keyspace: req.TargetKeyspace, + Workflow: req.Workflow, + Action: string(vdiff.DeleteAction), + ActionArg: req.Arg, + } + + ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) + if err != nil { + return nil, err + } + + err = ts.ForAllTargets(func(target *MigrationTarget) error { + _, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) + return err + }) + if err != nil { + s.Logger().Errorf("Error executing vdiff delete action: %v", err) + return nil, err + } + + return &vtctldatapb.VDiffDeleteResponse{}, nil +} + +// VDiffResume is part of the vtctlservicepb.VtctldServer interface. +func (s *Server) VDiffResume(ctx context.Context, req *vtctldatapb.VDiffResumeRequest) (*vtctldatapb.VDiffResumeResponse, error) { + span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffResume") + defer span.Finish() + + targetShards := req.GetTargetShards() + + span.Annotate("keyspace", req.TargetKeyspace) + span.Annotate("workflow", req.Workflow) + span.Annotate("uuid", req.Uuid) + span.Annotate("target_shards", targetShards) + + tabletreq := &tabletmanagerdatapb.VDiffRequest{ + Keyspace: req.TargetKeyspace, + Workflow: req.Workflow, + Action: string(vdiff.ResumeAction), + VdiffUuid: req.Uuid, + } + + ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) + if err != nil { + return nil, err + } + + if len(targetShards) > 0 { + if err := applyTargetShards(ts, targetShards); err != nil { + return nil, err + } + } + + err = ts.ForAllTargets(func(target *MigrationTarget) error { + _, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) + return err + }) + if err != nil { + s.Logger().Errorf("Error executing vdiff resume action: %v", err) + return nil, err + } + + return &vtctldatapb.VDiffResumeResponse{}, nil +} + +// VDiffShow is part of the vtctlservicepb.VtctldServer interface. +func (s *Server) VDiffShow(ctx context.Context, req *vtctldatapb.VDiffShowRequest) (*vtctldatapb.VDiffShowResponse, error) { + span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffShow") + defer span.Finish() + + span.Annotate("keyspace", req.TargetKeyspace) + span.Annotate("workflow", req.Workflow) + span.Annotate("argument", req.Arg) + + tabletreq := &tabletmanagerdatapb.VDiffRequest{ + Keyspace: req.TargetKeyspace, + Workflow: req.Workflow, + Action: string(vdiff.ShowAction), + ActionArg: req.Arg, + } + + ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) + if err != nil { + return nil, err + } + + output := &vdiffOutput{ + responses: make(map[string]*tabletmanagerdatapb.VDiffResponse, len(ts.targets)), + err: nil, + } + output.err = ts.ForAllTargets(func(target *MigrationTarget) error { + resp, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) + output.mu.Lock() + defer output.mu.Unlock() + output.responses[target.GetShard().ShardName()] = resp + return err + }) + if output.err != nil { + s.Logger().Errorf("Error executing vdiff show action: %v", output.err) + return nil, output.err + } + return &vtctldatapb.VDiffShowResponse{ + TabletResponses: output.responses, + }, nil +} + +// VDiffStop is part of the vtctlservicepb.VtctldServer interface. +func (s *Server) VDiffStop(ctx context.Context, req *vtctldatapb.VDiffStopRequest) (*vtctldatapb.VDiffStopResponse, error) { + span, ctx := trace.NewSpan(ctx, "workflow.Server.VDiffStop") + defer span.Finish() + + targetShards := req.GetTargetShards() + + span.Annotate("keyspace", req.TargetKeyspace) + span.Annotate("workflow", req.Workflow) + span.Annotate("uuid", req.Uuid) + span.Annotate("target_shards", targetShards) + + tabletreq := &tabletmanagerdatapb.VDiffRequest{ + Keyspace: req.TargetKeyspace, + Workflow: req.Workflow, + Action: string(vdiff.StopAction), + VdiffUuid: req.Uuid, + } + + ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow) + if err != nil { + return nil, err + } + + if len(targetShards) > 0 { + if err := applyTargetShards(ts, targetShards); err != nil { + return nil, err + } + } + + err = ts.ForAllTargets(func(target *MigrationTarget) error { + _, err := s.tmc.VDiff(ctx, target.GetPrimary().Tablet, tabletreq) + return err + }) + if err != nil { + s.Logger().Errorf("Error executing vdiff stop action: %v", err) + return nil, err + } + + return &vtctldatapb.VDiffStopResponse{}, nil +} diff --git a/go/vt/vtctl/workflow/vdiff_test.go b/go/vt/vtctl/workflow/vdiff_test.go index e5578afc170..0da4a3ef480 100644 --- a/go/vt/vtctl/workflow/vdiff_test.go +++ b/go/vt/vtctl/workflow/vdiff_test.go @@ -17,13 +17,21 @@ limitations under the License. package workflow import ( + "context" + "fmt" "math" "testing" "time" + "github.com/google/uuid" "github.com/stretchr/testify/require" "vitess.io/vitess/go/vt/vttablet/tabletmanager/vdiff" + + querypb "vitess.io/vitess/go/vt/proto/query" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) func TestBuildProgressReport(t *testing.T) { @@ -134,3 +142,350 @@ func TestBuildProgressReport(t *testing.T) { }) } } + +// TestVDiffCreate performs some basic tests of the VDiffCreate function +// to ensure that it behaves as expected given a specific request. +func TestVDiffCreate(t *testing.T) { + ctx := context.Background() + workflowName := "wf1" + sourceKeyspace := &testKeyspace{ + KeyspaceName: "source", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "target", + ShardNames: []string{"-80", "80-"}, + } + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + + tests := []struct { + name string + req *vtctldatapb.VDiffCreateRequest + wantErr string + }{ + { + name: "no values", + req: &vtctldatapb.VDiffCreateRequest{}, + // We did not provide any keyspace or shard. + wantErr: "FindAllShardsInKeyspace() invalid keyspace name: UnescapeID err: invalid input identifier ''", + }, + { + name: "generated UUID", + req: &vtctldatapb.VDiffCreateRequest{ + TargetKeyspace: targetKeyspace.KeyspaceName, + Workflow: workflowName, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == "" { + env.tmc.expectVRQueryResultOnKeyspaceTablets(targetKeyspace.KeyspaceName, &queryResult{ + query: "select vrepl_id, table_name, lastpk from _vt.copy_state where vrepl_id in (1) and id in (select max(id) from _vt.copy_state where vrepl_id in (1) group by vrepl_id, table_name)", + result: &querypb.QueryResult{}, + }) + } + got, err := env.ws.VDiffCreate(ctx, tt.req) + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.NotNil(t, got) + // Ensure that we always use a valid UUID. + err = uuid.Validate(got.UUID) + require.NoError(t, err) + }) + } +} + +func TestVDiffResume(t *testing.T) { + ctx := context.Background() + sourceKeyspace := &testKeyspace{ + KeyspaceName: "sourceks", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "targetks", + ShardNames: []string{"-80", "80-"}, + } + workflow := "testwf" + uuid := uuid.New().String() + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + + env.tmc.strict = true + action := string(vdiff.ResumeAction) + + tests := []struct { + name string + req *vtctldatapb.VDiffResumeRequest // vtctld requests + expectedVDiffRequests map[*topodatapb.Tablet]*vdiffRequestResponse // tablet requests + wantErr string + }{ + { + name: "basic resume", // Both target shards + req: &vtctldatapb.VDiffResumeRequest{ + TargetKeyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Uuid: uuid, + }, + expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + VdiffUuid: uuid, + }, + }, + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID+tabletUIDStep]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + VdiffUuid: uuid, + }, + }, + }, + }, + { + name: "resume on first shard", + req: &vtctldatapb.VDiffResumeRequest{ + TargetKeyspace: targetKeyspace.KeyspaceName, + TargetShards: targetKeyspace.ShardNames[:1], + Workflow: workflow, + Uuid: uuid, + }, + expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + VdiffUuid: uuid, + }, + }, + }, + }, + { + name: "resume on invalid shard", + req: &vtctldatapb.VDiffResumeRequest{ + TargetKeyspace: targetKeyspace.KeyspaceName, + TargetShards: []string{"0"}, + Workflow: workflow, + Uuid: uuid, + }, + wantErr: fmt.Sprintf("specified target shard 0 not a valid target for workflow %s", workflow), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for tab, vdr := range tt.expectedVDiffRequests { + env.tmc.expectVDiffRequest(tab, vdr) + } + got, err := env.ws.VDiffResume(ctx, tt.req) + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + } else { + require.NoError(t, err) + require.NotNil(t, got) + } + env.tmc.confirmVDiffRequests(t) + }) + } +} + +func TestVDiffStop(t *testing.T) { + ctx := context.Background() + sourceKeyspace := &testKeyspace{ + KeyspaceName: "sourceks", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "targetks", + ShardNames: []string{"-80", "80-"}, + } + workflow := "testwf" + uuid := uuid.New().String() + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + + env.tmc.strict = true + action := string(vdiff.StopAction) + + tests := []struct { + name string + req *vtctldatapb.VDiffStopRequest // vtctld requests + expectedVDiffRequests map[*topodatapb.Tablet]*vdiffRequestResponse // tablet requests + wantErr string + }{ + { + name: "basic stop", // Both target shards + req: &vtctldatapb.VDiffStopRequest{ + TargetKeyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Uuid: uuid, + }, + expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + VdiffUuid: uuid, + }, + }, + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID+tabletUIDStep]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + VdiffUuid: uuid, + }, + }, + }, + }, + { + name: "stop on first shard", + req: &vtctldatapb.VDiffStopRequest{ + TargetKeyspace: targetKeyspace.KeyspaceName, + TargetShards: targetKeyspace.ShardNames[:1], + Workflow: workflow, + Uuid: uuid, + }, + expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + VdiffUuid: uuid, + }, + }, + }, + }, + { + name: "stop on invalid shard", + req: &vtctldatapb.VDiffStopRequest{ + TargetKeyspace: targetKeyspace.KeyspaceName, + TargetShards: []string{"0"}, + Workflow: workflow, + Uuid: uuid, + }, + wantErr: fmt.Sprintf("specified target shard 0 not a valid target for workflow %s", workflow), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for tab, vdr := range tt.expectedVDiffRequests { + env.tmc.expectVDiffRequest(tab, vdr) + } + got, err := env.ws.VDiffStop(ctx, tt.req) + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + } else { + require.NoError(t, err) + require.NotNil(t, got) + } + env.tmc.confirmVDiffRequests(t) + }) + } +} + +func TestVDiffDelete(t *testing.T) { + ctx := context.Background() + sourceKeyspace := &testKeyspace{ + KeyspaceName: "sourceks", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "targetks", + ShardNames: []string{"-80", "80-"}, + } + workflow := "testwf" + uuid := uuid.New().String() + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + + env.tmc.strict = true + action := string(vdiff.DeleteAction) + + tests := []struct { + name string + req *vtctldatapb.VDiffDeleteRequest + expectedVDiffRequests map[*topodatapb.Tablet]*vdiffRequestResponse + wantErr string + }{ + { + name: "basic delete", + req: &vtctldatapb.VDiffDeleteRequest{ + TargetKeyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Arg: uuid, + }, + expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + ActionArg: uuid, + }, + }, + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID+tabletUIDStep]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + ActionArg: uuid, + }, + }, + }, + }, + { + name: "invalid delete", + req: &vtctldatapb.VDiffDeleteRequest{ + TargetKeyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Arg: uuid, + }, + expectedVDiffRequests: map[*topodatapb.Tablet]*vdiffRequestResponse{ + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + ActionArg: uuid, + }, + err: fmt.Errorf("error on invalid delete"), + }, + env.tablets[targetKeyspace.KeyspaceName][startingTargetTabletUID+tabletUIDStep]: { + req: &tabletmanagerdatapb.VDiffRequest{ + Keyspace: targetKeyspace.KeyspaceName, + Workflow: workflow, + Action: action, + ActionArg: uuid, + }, + }, + }, + wantErr: "error on invalid delete", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for tab, vdr := range tt.expectedVDiffRequests { + env.tmc.expectVDiffRequest(tab, vdr) + } + got, err := env.ws.VDiffDelete(ctx, tt.req) + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + } else { + require.NoError(t, err) + require.NotNil(t, got) + } + env.tmc.confirmVDiffRequests(t) + }) + } +} diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index e59832cdab5..50d3a4b6bbf 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -465,7 +465,7 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(240) + size += int64(224) } // field InsertCommon vitess.io/vitess/go/vt/vtgate/engine.InsertCommon size += cached.InsertCommon.CachedSize(false) diff --git a/go/vt/vtgate/engine/fake_vcursor_test.go b/go/vt/vtgate/engine/fake_vcursor_test.go index f27ca380876..aac3e9b584c 100644 --- a/go/vt/vtgate/engine/fake_vcursor_test.go +++ b/go/vt/vtgate/engine/fake_vcursor_test.go @@ -400,6 +400,20 @@ func (t *noopVCursor) GetDBDDLPluginName() string { panic("unimplemented") } +func (t *noopVCursor) SetLastInsertID(uint64) {} +func (t *noopVCursor) VExplainLogging() {} +func (t *noopVCursor) DisableLogging() {} +func (t *noopVCursor) GetVExplainLogs() []ExecuteEntry { + return nil +} +func (t *noopVCursor) GetLogs() ([]ExecuteEntry, error) { + return nil, nil +} + +// RecordMirrorStats implements VCursor. +func (t *noopVCursor) RecordMirrorStats(sourceExecTime, targetExecTime time.Duration, targetErr error) { +} + var ( _ VCursor = (*loggingVCursor)(nil) _ SessionActions = (*loggingVCursor)(nil) @@ -893,20 +907,6 @@ func (t *loggingVCursor) RecordMirrorStats(sourceExecTime, targetExecTime time.D } } -func (t *noopVCursor) VExplainLogging() {} -func (t *noopVCursor) DisableLogging() {} -func (t *noopVCursor) GetVExplainLogs() []ExecuteEntry { - return nil -} - -func (t *noopVCursor) GetLogs() ([]ExecuteEntry, error) { - return nil, nil -} - -// RecordMirrorStats implements VCursor. -func (t *noopVCursor) RecordMirrorStats(sourceExecTime, targetExecTime time.Duration, targetErr error) { -} - func expectResult(t *testing.T, result, want *sqltypes.Result) { t.Helper() fieldsResult := fmt.Sprintf("%v", result.Fields) diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 10a4048572f..5bc206f7465 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -58,11 +58,9 @@ type Insert struct { // Alias represents the row alias with columns if specified in the query. Alias string - - FetchLastInsertID bool } -// newQueryInsert creates an Insert with a query string. +// newQueryInsert creates an Insert with a query string. Used in testing. func newQueryInsert(opcode InsertOpcode, keyspace *vindexes.Keyspace, query string) *Insert { return &Insert{ InsertCommon: InsertCommon{ @@ -73,7 +71,7 @@ func newQueryInsert(opcode InsertOpcode, keyspace *vindexes.Keyspace, query stri } } -// newInsert creates a new Insert. +// newInsert creates a new Insert. Used in testing. func newInsert( opcode InsertOpcode, ignore bool, diff --git a/go/vt/vtgate/engine/insert_common.go b/go/vt/vtgate/engine/insert_common.go index 629d848d978..d4cae045e86 100644 --- a/go/vt/vtgate/engine/insert_common.go +++ b/go/vt/vtgate/engine/insert_common.go @@ -161,7 +161,7 @@ func (ins *InsertCommon) executeUnshardedTableQuery(ctx context.Context, vcursor if err != nil { return nil, err } - qr, err := execShard(ctx, loggingPrimitive, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */, false) + qr, err := execShard(ctx, loggingPrimitive, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */, ins.FetchLastInsertID) if err != nil { return nil, err } diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go index bccee5f2cf9..af834858175 100644 --- a/go/vt/vtgate/engine/insert_select.go +++ b/go/vt/vtgate/engine/insert_select.go @@ -51,7 +51,7 @@ type ( } ) -// newInsertSelect creates a new InsertSelect. +// newInsertSelect creates a new InsertSelect. Used in testing. func newInsertSelect( ignore bool, keyspace *vindexes.Keyspace, diff --git a/go/vt/vtgate/engine/primitive.go b/go/vt/vtgate/engine/primitive.go index e6fa102581e..7734dd81a6b 100644 --- a/go/vt/vtgate/engine/primitive.go +++ b/go/vt/vtgate/engine/primitive.go @@ -147,6 +147,8 @@ type ( // RecordMirrorStats is used to record stats about a mirror query. RecordMirrorStats(time.Duration, time.Duration, error) + + SetLastInsertID(uint64) } // SessionActions gives primitives ability to interact with the session state diff --git a/go/vt/vtgate/engine/set.go b/go/vt/vtgate/engine/set.go index 95fb5c87a32..f0de330cbed 100644 --- a/go/vt/vtgate/engine/set.go +++ b/go/vt/vtgate/engine/set.go @@ -22,23 +22,18 @@ import ( "fmt" "strings" - "vitess.io/vitess/go/vt/sqlparser" - "vitess.io/vitess/go/vt/sysvars" - - vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" - - "vitess.io/vitess/go/vt/log" - - "vitess.io/vitess/go/vt/srvtopo" - - "vitess.io/vitess/go/vt/vtgate/evalengine" - "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" + "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" + vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/schema" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/sysvars" "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/vt/vtgate/vindexes" ) diff --git a/go/vt/vtgate/evalengine/arena.go b/go/vt/vtgate/evalengine/arena.go index ccfe63f514f..c5457f076e8 100644 --- a/go/vt/vtgate/evalengine/arena.go +++ b/go/vt/vtgate/evalengine/arena.go @@ -71,7 +71,7 @@ func (a *Arena) newEvalEnum(raw []byte, values *EnumSetValues) *evalEnum { } else { a.aEnum = append(a.aEnum, evalEnum{}) } - val := &a.aEnum[len(a.aInt64)-1] + val := &a.aEnum[len(a.aEnum)-1] s := string(raw) val.string = s val.value = valueIdx(values, s) @@ -84,7 +84,7 @@ func (a *Arena) newEvalSet(raw []byte, values *EnumSetValues) *evalSet { } else { a.aSet = append(a.aSet, evalSet{}) } - val := &a.aSet[len(a.aInt64)-1] + val := &a.aSet[len(a.aSet)-1] s := string(raw) val.string = s val.set = evalSetBits(values, s) diff --git a/go/vt/vtgate/evalengine/cached_size.go b/go/vt/vtgate/evalengine/cached_size.go index c1ed1f9475c..d51c65c75b4 100644 --- a/go/vt/vtgate/evalengine/cached_size.go +++ b/go/vt/vtgate/evalengine/cached_size.go @@ -1181,6 +1181,18 @@ func (cached *builtinLastDay) CachedSize(alloc bool) int64 { size += cached.CallExpr.CachedSize(false) return size } +func (cached *builtinLastInsertID) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(48) + } + // field CallExpr vitess.io/vitess/go/vt/vtgate/evalengine.CallExpr + size += cached.CallExpr.CachedSize(false) + return size +} func (cached *builtinLeftRight) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) diff --git a/go/vt/vtgate/evalengine/compiler_asm.go b/go/vt/vtgate/evalengine/compiler_asm.go index dfb1a30bffc..7dda215353f 100644 --- a/go/vt/vtgate/evalengine/compiler_asm.go +++ b/go/vt/vtgate/evalengine/compiler_asm.go @@ -5138,3 +5138,18 @@ func (asm *assembler) Introduce(offset int, t sqltypes.Type, col collations.Type return 1 }, "INTRODUCE (SP-1)") } + +func (asm *assembler) Fn_LAST_INSERT_ID() { + asm.emit(func(env *ExpressionEnv) int { + arg := env.vm.stack[env.vm.sp-1] + if arg == nil { + env.VCursor().SetLastInsertID(0) + } else { + iarg := evalToInt64(arg) + uarg := env.vm.arena.newEvalUint64(uint64(iarg.i)) + env.vm.stack[env.vm.sp-1] = uarg + env.VCursor().SetLastInsertID(uarg.u) + } + return 1 + }, "FN LAST_INSERT_ID UINT64(SP-1)") +} diff --git a/go/vt/vtgate/evalengine/compiler_test.go b/go/vt/vtgate/evalengine/compiler_test.go index 88c13a479ed..343bb0cd043 100644 --- a/go/vt/vtgate/evalengine/compiler_test.go +++ b/go/vt/vtgate/evalengine/compiler_test.go @@ -760,6 +760,20 @@ func TestCompilerSingle(t *testing.T) { expression: `WEEK(timestamp '2024-01-01 10:34:58', 1)`, result: `INT64(1)`, }, + { + expression: `column0 + 1`, + values: []sqltypes.Value{sqltypes.MakeTrusted(sqltypes.Enum, []byte("foo"))}, + // Returns 0, as unknown enums evaluate here to -1. We have this test to + // exercise the path to push enums onto the stack. + result: `FLOAT64(0)`, + }, + { + expression: `column0 + 1`, + values: []sqltypes.Value{sqltypes.MakeTrusted(sqltypes.Set, []byte("foo"))}, + // Returns 1, as unknown sets evaluate here to 0. We have this test to + // exercise the path to push sets onto the stack. + result: `FLOAT64(1)`, + }, } tz, _ := time.LoadLocation("Europe/Madrid") @@ -903,6 +917,99 @@ func TestBindVarLiteral(t *testing.T) { } } +type testVcursor struct { + lastInsertID *uint64 + env *vtenv.Environment +} + +func (t *testVcursor) TimeZone() *time.Location { + return time.UTC +} + +func (t *testVcursor) GetKeyspace() string { + return "apa" +} + +func (t *testVcursor) SQLMode() string { + return "oltp" +} + +func (t *testVcursor) Environment() *vtenv.Environment { + return t.env +} + +func (t *testVcursor) SetLastInsertID(id uint64) { + t.lastInsertID = &id +} + +var _ evalengine.VCursor = (*testVcursor)(nil) + +func TestLastInsertID(t *testing.T) { + var testCases = []struct { + expression string + result uint64 + missing bool + }{ + { + expression: `last_insert_id(1)`, + result: 1, + }, { + expression: `12`, + missing: true, + }, { + expression: `last_insert_id(666)`, + result: 666, + }, { + expression: `last_insert_id(null)`, + result: 0, + }, + } + + venv := vtenv.NewTestEnv() + for _, tc := range testCases { + t.Run(tc.expression, func(t *testing.T) { + expr, err := venv.Parser().ParseExpr(tc.expression) + require.NoError(t, err) + + cfg := &evalengine.Config{ + Collation: collations.CollationUtf8mb4ID, + NoConstantFolding: true, + NoCompilation: false, + Environment: venv, + } + t.Run("eval", func(t *testing.T) { + cfg.NoCompilation = true + runTest(t, expr, cfg, tc) + }) + t.Run("compiled", func(t *testing.T) { + cfg.NoCompilation = false + runTest(t, expr, cfg, tc) + }) + }) + } +} + +func runTest(t *testing.T, expr sqlparser.Expr, cfg *evalengine.Config, tc struct { + expression string + result uint64 + missing bool +}) { + converted, err := evalengine.Translate(expr, cfg) + require.NoError(t, err) + + vc := &testVcursor{env: vtenv.NewTestEnv()} + env := evalengine.NewExpressionEnv(context.Background(), nil, vc) + + _, err = env.Evaluate(converted) + require.NoError(t, err) + if tc.missing { + require.Nil(t, vc.lastInsertID) + } else { + require.NotNil(t, vc.lastInsertID) + require.Equal(t, tc.result, *vc.lastInsertID) + } +} + func TestCompilerNonConstant(t *testing.T) { var testCases = []struct { expression string diff --git a/go/vt/vtgate/evalengine/expr_env.go b/go/vt/vtgate/evalengine/expr_env.go index 38a65f9b4e0..4a7f9849ab0 100644 --- a/go/vt/vtgate/evalengine/expr_env.go +++ b/go/vt/vtgate/evalengine/expr_env.go @@ -35,6 +35,7 @@ type VCursor interface { GetKeyspace() string SQLMode() string Environment() *vtenv.Environment + SetLastInsertID(id uint64) } type ( @@ -140,6 +141,7 @@ func (e *emptyVCursor) GetKeyspace() string { func (e *emptyVCursor) SQLMode() string { return config.DefaultSQLMode } +func (e *emptyVCursor) SetLastInsertID(_ uint64) {} func NewEmptyVCursor(env *vtenv.Environment, tz *time.Location) VCursor { return &emptyVCursor{env: env, tz: tz} diff --git a/go/vt/vtgate/evalengine/fn_misc.go b/go/vt/vtgate/evalengine/fn_misc.go index 8813b62f823..2a2119ee6f4 100644 --- a/go/vt/vtgate/evalengine/fn_misc.go +++ b/go/vt/vtgate/evalengine/fn_misc.go @@ -81,6 +81,10 @@ type ( builtinUUIDToBin struct { CallExpr } + + builtinLastInsertID struct { + CallExpr + } ) var _ IR = (*builtinInetAton)(nil) @@ -95,6 +99,7 @@ var _ IR = (*builtinBinToUUID)(nil) var _ IR = (*builtinIsUUID)(nil) var _ IR = (*builtinUUID)(nil) var _ IR = (*builtinUUIDToBin)(nil) +var _ IR = (*builtinLastInsertID)(nil) func (call *builtinInetAton) eval(env *ExpressionEnv) (eval, error) { arg, err := call.arg1(env) @@ -194,6 +199,33 @@ func (call *builtinInet6Aton) compile(c *compiler) (ctype, error) { return ctype{Type: sqltypes.VarBinary, Flag: flagNullable, Col: collationBinary}, nil } +func (call *builtinLastInsertID) eval(env *ExpressionEnv) (eval, error) { + arg, err := call.arg1(env) + if err != nil { + return nil, err + } + if arg == nil { + env.VCursor().SetLastInsertID(0) + return nil, err + } + insertID := uint64(evalToInt64(arg).i) + env.VCursor().SetLastInsertID(insertID) + return newEvalUint64(insertID), nil +} + +func (call *builtinLastInsertID) compile(c *compiler) (ctype, error) { + arg, err := call.Arguments[0].compile(c) + if err != nil { + return ctype{}, err + } + c.asm.Fn_LAST_INSERT_ID() + return ctype{Type: sqltypes.Uint64, Flag: arg.Flag & flagNullable, Col: collationNumeric}, nil +} + +func (call *builtinLastInsertID) constant() bool { + return false // we don't want this function to be simplified away +} + func printIPv6AsIPv4(addr netip.Addr) (netip.Addr, bool) { b := addr.AsSlice() if len(b) != 16 { diff --git a/go/vt/vtgate/evalengine/integration/comparison_test.go b/go/vt/vtgate/evalengine/integration/comparison_test.go index ea327601975..d559cb8ab1d 100644 --- a/go/vt/vtgate/evalengine/integration/comparison_test.go +++ b/go/vt/vtgate/evalengine/integration/comparison_test.go @@ -209,6 +209,10 @@ type vcursor struct { env *vtenv.Environment } +func (vc *vcursor) SetLastInsertID(id uint64) {} + +var _ evalengine.VCursor = (*vcursor)(nil) + func (vc *vcursor) GetKeyspace() string { return "vttest" } diff --git a/go/vt/vtgate/evalengine/translate_builtin.go b/go/vt/vtgate/evalengine/translate_builtin.go index 476ee32483b..1f8bd7798aa 100644 --- a/go/vt/vtgate/evalengine/translate_builtin.go +++ b/go/vt/vtgate/evalengine/translate_builtin.go @@ -662,6 +662,11 @@ func (ast *astCompiler) translateFuncExpr(fn *sqlparser.FuncExpr) (IR, error) { return nil, argError(method) } return &builtinReplace{CallExpr: call, collate: ast.cfg.Collation}, nil + case "last_insert_id": + if len(args) != 1 { + return nil, argError(method) + } + return &builtinLastInsertID{CallExpr: call}, nil default: return nil, translateExprNotSupported(fn) } diff --git a/go/vt/vtgate/executorcontext/vcursor_impl.go b/go/vt/vtgate/executorcontext/vcursor_impl.go index 40317f5103a..3f8d7def797 100644 --- a/go/vt/vtgate/executorcontext/vcursor_impl.go +++ b/go/vt/vtgate/executorcontext/vcursor_impl.go @@ -22,6 +22,7 @@ import ( "io" "sort" "strings" + "sync" "sync/atomic" "time" @@ -154,6 +155,8 @@ type ( observer ResultsObserver + // this protects the interOpStats and shardsStats fields from concurrent writes + mu sync.Mutex // this is a map of the number of rows that every primitive has returned // if this field is nil, it means that we are not logging operator traffic interOpStats map[engine.Primitive]engine.RowsReceived @@ -642,21 +645,29 @@ func (vc *VCursorImpl) ExecutePrimitive(ctx context.Context, primitive engine.Pr } func (vc *VCursorImpl) logOpTraffic(primitive engine.Primitive, res *sqltypes.Result) { - if vc.interOpStats != nil { - rows := vc.interOpStats[primitive] - if res == nil { - rows = append(rows, 0) - } else { - rows = append(rows, len(res.Rows)) - } - vc.interOpStats[primitive] = rows + if vc.interOpStats == nil { + return + } + + vc.mu.Lock() + defer vc.mu.Unlock() + + rows := vc.interOpStats[primitive] + if res == nil { + rows = append(rows, 0) + } else { + rows = append(rows, len(res.Rows)) } + vc.interOpStats[primitive] = rows } func (vc *VCursorImpl) logShardsQueried(primitive engine.Primitive, shardsNb int) { - if vc.shardsStats != nil { - vc.shardsStats[primitive] += engine.ShardsQueried(shardsNb) + if vc.shardsStats == nil { + return } + vc.mu.Lock() + defer vc.mu.Unlock() + vc.shardsStats[primitive] += engine.ShardsQueried(shardsNb) } func (vc *VCursorImpl) ExecutePrimitiveStandalone(ctx context.Context, primitive engine.Primitive, bindVars map[string]*querypb.BindVariable, wantfields bool) (*sqltypes.Result, error) { @@ -1583,3 +1594,9 @@ func (vc *VCursorImpl) GetContextWithTimeOut(ctx context.Context) (context.Conte func (vc *VCursorImpl) IgnoreMaxMemoryRows() bool { return vc.ignoreMaxMemoryRows } + +func (vc *VCursorImpl) SetLastInsertID(id uint64) { + vc.SafeSession.mu.Lock() + defer vc.SafeSession.mu.Unlock() + vc.SafeSession.LastInsertId = id +} diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index 80516871623..d3ad5afac72 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -51,7 +51,7 @@ func gen4InsertStmtPlanner(version querypb.ExecuteOptions_PlannerVersion, insStm } if ks != nil { if tables[0].AutoIncrement == nil && !ctx.SemTable.ForeignKeysPresent() { - plan := insertUnshardedShortcut(insStmt, ks, tables) + plan := insertUnshardedShortcut(ctx, insStmt, ks, tables) setCommentDirectivesOnPlan(plan, insStmt) return newPlanResult(plan, operators.QualifiedTables(ks, tables)...), nil } @@ -90,12 +90,13 @@ func errOutIfPlanCannotBeConstructed(ctx *plancontext.PlanningContext, vTbl *vin return ctx.SemTable.NotUnshardedErr } -func insertUnshardedShortcut(stmt *sqlparser.Insert, ks *vindexes.Keyspace, tables []*vindexes.Table) engine.Primitive { +func insertUnshardedShortcut(ctx *plancontext.PlanningContext, stmt *sqlparser.Insert, ks *vindexes.Keyspace, tables []*vindexes.Table) engine.Primitive { eIns := &engine.Insert{ InsertCommon: engine.InsertCommon{ - Opcode: engine.InsertUnsharded, - Keyspace: ks, - TableName: tables[0].Name.String(), + Opcode: engine.InsertUnsharded, + Keyspace: ks, + TableName: tables[0].Name.String(), + FetchLastInsertID: ctx.SemTable.ShouldFetchLastInsertID(), }, } eIns.Query = generateQuery(stmt) diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index bc71c7195b4..b51eac449fc 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -190,6 +190,7 @@ func transformInsertionSelection(ctx *plancontext.PlanningContext, op *operators ForceNonStreaming: op.ForceNonStreaming, Generate: autoIncGenerate(ins.AutoIncrement), ColVindexes: ins.ColVindexes, + FetchLastInsertID: ctx.SemTable.ShouldFetchLastInsertID(), }, VindexValueOffset: ins.VindexValueOffset, } @@ -659,9 +660,8 @@ func buildInsertPrimitive( } eins := &engine.Insert{ - InsertCommon: ic, - VindexValues: ins.VindexValues, - FetchLastInsertID: ctx.SemTable.ShouldFetchLastInsertID(), + InsertCommon: ic, + VindexValues: ins.VindexValues, } // we would need to generate the query on the fly. The only exception here is diff --git a/go/vt/vtgate/planbuilder/operators/cte_merging.go b/go/vt/vtgate/planbuilder/operators/cte_merging.go index cb19e06b2a7..0c1556c81e4 100644 --- a/go/vt/vtgate/planbuilder/operators/cte_merging.go +++ b/go/vt/vtgate/planbuilder/operators/cte_merging.go @@ -31,7 +31,7 @@ func tryMergeRecurse(ctx *plancontext.PlanningContext, in *RecurseCTE) (Operator } func tryMergeCTE(ctx *plancontext.PlanningContext, seed, term Operator, in *RecurseCTE) *Route { - seedRoute, termRoute, routingA, routingB, a, b, sameKeyspace := prepareInputRoutes(seed, term) + seedRoute, termRoute, routingA, routingB, a, b, sameKeyspace := prepareInputRoutes(ctx, seed, term) if seedRoute == nil { return nil } diff --git a/go/vt/vtgate/planbuilder/operators/delete.go b/go/vt/vtgate/planbuilder/operators/delete.go index 81e36d54315..015220470e0 100644 --- a/go/vt/vtgate/planbuilder/operators/delete.go +++ b/go/vt/vtgate/planbuilder/operators/delete.go @@ -124,7 +124,7 @@ func createDeleteWithInputOp(ctx *plancontext.PlanningContext, del *sqlparser.De } var delOps []dmlOp - for _, target := range ctx.SemTable.Targets.Constituents() { + for _, target := range ctx.SemTable.DMLTargets.Constituents() { op := createDeleteOpWithTarget(ctx, target, del.Ignore) delOps = append(delOps, op) } @@ -322,7 +322,7 @@ func updateQueryGraphWithSource(ctx *plancontext.PlanningContext, input Operator return op, NoRewrite } if len(qg.Tables) > 1 { - panic(vterrors.VT12001("DELETE on reference table with join")) + panic(vterrors.VT12001("DML on reference table with join")) } for _, tbl := range qg.Tables { if tbl.ID != tblID { diff --git a/go/vt/vtgate/planbuilder/operators/join_merging.go b/go/vt/vtgate/planbuilder/operators/join_merging.go index c035b7d11ed..cb3569cf79e 100644 --- a/go/vt/vtgate/planbuilder/operators/join_merging.go +++ b/go/vt/vtgate/planbuilder/operators/join_merging.go @@ -28,7 +28,7 @@ import ( // If they can be merged, a new operator with the merged routing is returned // If they cannot be merged, nil is returned. func (jm *joinMerger) mergeJoinInputs(ctx *plancontext.PlanningContext, lhs, rhs Operator, joinPredicates []sqlparser.Expr) *Route { - lhsRoute, rhsRoute, routingA, routingB, a, b, sameKeyspace := prepareInputRoutes(lhs, rhs) + lhsRoute, rhsRoute, routingA, routingB, a, b, sameKeyspace := prepareInputRoutes(ctx, lhs, rhs) if lhsRoute == nil { return nil } @@ -102,13 +102,13 @@ func mergeAnyShardRoutings(ctx *plancontext.PlanningContext, a, b *AnyShardRouti } } -func prepareInputRoutes(lhs Operator, rhs Operator) (*Route, *Route, Routing, Routing, routingType, routingType, bool) { +func prepareInputRoutes(ctx *plancontext.PlanningContext, lhs Operator, rhs Operator) (*Route, *Route, Routing, Routing, routingType, routingType, bool) { lhsRoute, rhsRoute := operatorsToRoutes(lhs, rhs) if lhsRoute == nil || rhsRoute == nil { return nil, nil, nil, nil, 0, 0, false } - lhsRoute, rhsRoute, routingA, routingB, sameKeyspace := getRoutesOrAlternates(lhsRoute, rhsRoute) + lhsRoute, rhsRoute, routingA, routingB, sameKeyspace := getRoutesOrAlternates(ctx, lhsRoute, rhsRoute) a, b := getRoutingType(routingA), getRoutingType(routingB) return lhsRoute, rhsRoute, routingA, routingB, a, b, sameKeyspace @@ -159,7 +159,7 @@ func (rt routingType) String() string { // getRoutesOrAlternates gets the Routings from each Route. If they are from different keyspaces, // we check if this is a table with alternates in other keyspaces that we can use -func getRoutesOrAlternates(lhsRoute, rhsRoute *Route) (*Route, *Route, Routing, Routing, bool) { +func getRoutesOrAlternates(ctx *plancontext.PlanningContext, lhsRoute, rhsRoute *Route) (*Route, *Route, Routing, Routing, bool) { routingA := lhsRoute.Routing routingB := rhsRoute.Routing sameKeyspace := routingA.Keyspace() == routingB.Keyspace() @@ -171,13 +171,17 @@ func getRoutesOrAlternates(lhsRoute, rhsRoute *Route) (*Route, *Route, Routing, return lhsRoute, rhsRoute, routingA, routingB, sameKeyspace } - if refA, ok := routingA.(*AnyShardRouting); ok { + // If we have a reference route, we will try to find an alternate route in same keyspace as other routing keyspace. + // If the reference route is part of DML table update target, alternate keyspace route cannot be considered. + if refA, ok := routingA.(*AnyShardRouting); ok && + !TableID(lhsRoute).IsOverlapping(ctx.SemTable.DMLTargets) { if altARoute := refA.AlternateInKeyspace(routingB.Keyspace()); altARoute != nil { return altARoute, rhsRoute, altARoute.Routing, routingB, true } } - if refB, ok := routingB.(*AnyShardRouting); ok { + if refB, ok := routingB.(*AnyShardRouting); ok && + !TableID(rhsRoute).IsOverlapping(ctx.SemTable.DMLTargets) { if altBRoute := refB.AlternateInKeyspace(routingA.Keyspace()); altBRoute != nil { return lhsRoute, altBRoute, routingA, altBRoute.Routing, true } diff --git a/go/vt/vtgate/planbuilder/operators/subquery_planning.go b/go/vt/vtgate/planbuilder/operators/subquery_planning.go index a2aca74fb6e..e222ae0f343 100644 --- a/go/vt/vtgate/planbuilder/operators/subquery_planning.go +++ b/go/vt/vtgate/planbuilder/operators/subquery_planning.go @@ -730,7 +730,7 @@ func mergeSubqueryInputs(ctx *plancontext.PlanningContext, in, out Operator, joi return nil } - inRoute, outRoute, inRouting, outRouting, sameKeyspace := getRoutesOrAlternates(inRoute, outRoute) + inRoute, outRoute, inRouting, outRouting, sameKeyspace := getRoutesOrAlternates(ctx, inRoute, outRoute) inner, outer := getRoutingType(inRouting), getRoutingType(outRouting) switch { diff --git a/go/vt/vtgate/planbuilder/operators/union_merging.go b/go/vt/vtgate/planbuilder/operators/union_merging.go index 000d176b61a..6173b59e0dc 100644 --- a/go/vt/vtgate/planbuilder/operators/union_merging.go +++ b/go/vt/vtgate/planbuilder/operators/union_merging.go @@ -108,7 +108,7 @@ func mergeUnionInputs( lhsExprs, rhsExprs sqlparser.SelectExprs, distinct bool, ) (Operator, sqlparser.SelectExprs) { - lhsRoute, rhsRoute, routingA, routingB, a, b, sameKeyspace := prepareInputRoutes(lhs, rhs) + lhsRoute, rhsRoute, routingA, routingB, a, b, sameKeyspace := prepareInputRoutes(ctx, lhs, rhs) if lhsRoute == nil { return nil, nil } diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index dd0a86c2de2..18a81175f7b 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -164,7 +164,7 @@ func createUpdateWithInputOp(ctx *plancontext.PlanningContext, upd *sqlparser.Up ueMap := prepareUpdateExpressionList(ctx, upd) var updOps []dmlOp - for _, target := range ctx.SemTable.Targets.Constituents() { + for _, target := range ctx.SemTable.DMLTargets.Constituents() { op := createUpdateOpWithTarget(ctx, upd, target, ueMap[target]) updOps = append(updOps, op) } @@ -308,7 +308,7 @@ func errIfUpdateNotSupported(ctx *plancontext.PlanningContext, stmt *sqlparser.U } } - // Now we check if any of the foreign key columns that are being udpated have dependencies on other updated columns. + // Now we check if any of the foreign key columns that are being updated have dependencies on other updated columns. // This is unsafe, and we currently don't support this in Vitess. if err := ctx.SemTable.ErrIfFkDependentColumnUpdated(stmt.Exprs); err != nil { panic(err) diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index df813b04dea..f3bed93e3c8 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -84,6 +84,7 @@ func (s *planTestSuite) TestPlan() { s.addPKsProvided(vschema, "user", []string{"user_extra"}, []string{"id", "user_id"}) s.addPKsProvided(vschema, "ordering", []string{"order"}, []string{"oid", "region_id"}) s.addPKsProvided(vschema, "ordering", []string{"order_event"}, []string{"oid", "ename"}) + s.addPKsProvided(vschema, "main", []string{"source_of_ref"}, []string{"id"}) // You will notice that some tests expect user.Id instead of user.id. // This is because we now pre-create vindex columns in the symbol @@ -305,6 +306,7 @@ func (s *planTestSuite) TestOne() { s.addPKsProvided(vschema, "user", []string{"user_extra"}, []string{"id", "user_id"}) s.addPKsProvided(vschema, "ordering", []string{"order"}, []string{"oid", "region_id"}) s.addPKsProvided(vschema, "ordering", []string{"order_event"}, []string{"oid", "ename"}) + s.addPKsProvided(vschema, "main", []string{"source_of_ref"}, []string{"id"}) s.testFile("onecase.json", vw, false) } @@ -666,7 +668,7 @@ func (s *planTestSuite) testFile(filename string, vschema *vschemawrapper.VSchem current := PlanTest{ Comment: tcase.Comment, Query: tcase.Query, - SkipE2E: true, + SkipE2E: tcase.SkipE2E, } vschema.Version = Gen4 out := getPlanOutput(tcase, vschema, render) diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index 49a03a8f05a..1ecbf3d4ff9 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -6160,6 +6160,44 @@ ] } }, + { + "comment": "last_insert_id on aggregation calculated at the vtgate level", + "query": "select last_insert_id(count(*)) from user", + "plan": { + "QueryType": "SELECT", + "Original": "select last_insert_id(count(*)) from user", + "Instructions": { + "OperatorType": "Projection", + "Expressions": [ + "last_insert_id(count(*)) as last_insert_id(count(*))" + ], + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Scalar", + "Aggregates": "sum_count_star(0) AS count(*)", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FetchLastInsertID": true, + "FieldQuery": "select count(*) from `user` where 1 != 1", + "Query": "select count(*) from `user`", + "Table": "`user`" + } + ] + } + ] + }, + "TablesUsed": [ + "user.user" + ] + } + }, { "comment": "aggregation on top of aggregation works fine", "query": "select distinct count(*) from user, (select distinct count(*) from user) X", diff --git a/go/vt/vtgate/planbuilder/testdata/dml_cases.json b/go/vt/vtgate/planbuilder/testdata/dml_cases.json index 8893b4df0c0..95cb14e38f5 100644 --- a/go/vt/vtgate/planbuilder/testdata/dml_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/dml_cases.json @@ -2648,6 +2648,30 @@ }, "skip_e2e": true }, + { + "comment": "insert using last_insert_id with argument (already an e2e test for this plan)", + "query": "insert into unsharded values(last_insert_id(789), 2)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into unsharded values(last_insert_id(789), 2)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "FetchLastInsertID": true, + "Query": "insert into unsharded values (last_insert_id(789), 2)", + "TableName": "unsharded" + }, + "TablesUsed": [ + "main.unsharded" + ] + }, + "skip_e2e": true + }, { "comment": "update vindex value to null with multiple primary keyspace id", "query": "update user set name = null where id in (1, 2, 3)", diff --git a/go/vt/vtgate/planbuilder/testdata/reference_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_cases.json index 6aa01355934..1bf893beeef 100644 --- a/go/vt/vtgate/planbuilder/testdata/reference_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/reference_cases.json @@ -2,6 +2,7 @@ { "comment": "select from unqualified ambiguous reference routes to reference source", "query": "select * from ambiguous_ref_with_source", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from ambiguous_ref_with_source", @@ -24,6 +25,7 @@ { "comment": "join with unqualified ambiguous reference table routes to optimal keyspace", "query": "select user.col from user join ambiguous_ref_with_source", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col from user join ambiguous_ref_with_source", @@ -47,6 +49,7 @@ { "comment": "ambiguous unqualified reference table self-join routes to reference source", "query": "select r1.col from ambiguous_ref_with_source r1 join ambiguous_ref_with_source", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select r1.col from ambiguous_ref_with_source r1 join ambiguous_ref_with_source", @@ -69,6 +72,7 @@ { "comment": "ambiguous unqualified reference table can merge with other opcodes left to right.", "query": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join user", @@ -92,6 +96,7 @@ { "comment": "ambiguous unqualified reference table can merge with other opcodes left to right and vindex value is in the plan", "query": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join (select aa from user where user.id=1) user", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join (select aa from user where user.id=1) user", @@ -119,6 +124,7 @@ { "comment": "qualified join to reference table routes to optimal keyspace", "query": "select user.col from user join main.ambiguous_ref_with_source", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col from user join main.ambiguous_ref_with_source", @@ -142,6 +148,7 @@ { "comment": "insert into ambiguous unqualified reference table routes to source", "query": "insert into ambiguous_ref_with_source(col) values(1)", + "skip_e2e": true, "plan": { "QueryType": "INSERT", "Original": "insert into ambiguous_ref_with_source(col) values(1)", @@ -164,6 +171,7 @@ { "comment": "Reference tables using left join with a derived table having a limit clause", "query": "SELECT u.id FROM ( SELECT a.id, a.u_id FROM user.ref_with_source AS a WHERE a.id IN (3) ORDER BY a.d_at LIMIT 1) as u LEFT JOIN user.ref_with_source AS u0 ON u.u_id = u0.u_uid ORDER BY u.id", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "SELECT u.id FROM ( SELECT a.id, a.u_id FROM user.ref_with_source AS a WHERE a.id IN (3) ORDER BY a.d_at LIMIT 1) as u LEFT JOIN user.ref_with_source AS u0 ON u.u_id = u0.u_uid ORDER BY u.id", @@ -208,6 +216,7 @@ { "comment": "insert into qualified ambiguous reference table routes to source", "query": "insert into user.ambiguous_ref_with_source(col) values(1)", + "skip_e2e": true, "plan": { "QueryType": "INSERT", "Original": "insert into user.ambiguous_ref_with_source(col) values(1)", @@ -230,6 +239,7 @@ { "comment": "update unqualified ambiguous reference table routes to source", "query": "update ambiguous_ref_with_source set col = 1", + "skip_e2e": true, "plan": { "QueryType": "UPDATE", "Original": "update ambiguous_ref_with_source set col = 1", @@ -252,6 +262,7 @@ { "comment": "update qualified ambiguous reference table route to source", "query": "update user.ambiguous_ref_with_source set col = 1", + "skip_e2e": true, "plan": { "QueryType": "UPDATE", "Original": "update user.ambiguous_ref_with_source set col = 1", @@ -274,6 +285,7 @@ { "comment": "delete from unqualified ambiguous reference table routes to source", "query": "delete from ambiguous_ref_with_source where col = 1", + "skip_e2e": true, "plan": { "QueryType": "DELETE", "Original": "delete from ambiguous_ref_with_source where col = 1", @@ -296,6 +308,7 @@ { "comment": "delete from qualified ambiguous reference table route to source", "query": "delete from user.ambiguous_ref_with_source where col = 1", + "skip_e2e": true, "plan": { "QueryType": "DELETE", "Original": "delete from user.ambiguous_ref_with_source where col = 1", @@ -318,6 +331,7 @@ { "comment": "join with unqualified unambiguous ref with source routes to requested table", "query": "select user.col from user join ref_with_source", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col from user join ref_with_source", @@ -341,6 +355,7 @@ { "comment": "join with unqualified reference optimize routes when source & reference have different names", "query": "select user.col from user join source_of_ref", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col from user join source_of_ref", @@ -364,6 +379,7 @@ { "comment": "join with unqualified reference respects routing rules", "query": "select user.col from user join rerouted_ref", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col from user join rerouted_ref", @@ -387,6 +403,7 @@ { "comment": "join with reference to unqualified source routes to optimal keyspace", "query": "select user.col from user join global_ref", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select user.col from user join global_ref", @@ -410,6 +427,7 @@ { "comment": "insert into qualified reference with unqualified source routes to source", "query": "insert into user.global_ref(col) values(1)", + "skip_e2e": true, "plan": { "QueryType": "INSERT", "Original": "insert into user.global_ref(col) values(1)", @@ -432,6 +450,7 @@ { "comment": "delete from reference table with another name - query send to source table", "query": "delete from user.ref_with_source where col = 1", + "skip_e2e": true, "plan": { "QueryType": "DELETE", "Original": "delete from user.ref_with_source where col = 1", @@ -454,6 +473,7 @@ { "comment": "update from reference table with another name - query send to source table", "query": "update user.ref_with_source set x = 4 where col = 1", + "skip_e2e": true, "plan": { "QueryType": "UPDATE", "Original": "update user.ref_with_source set x = 4 where col = 1", @@ -476,6 +496,7 @@ { "comment": "insert from reference table with another name - query send to source table", "query": "insert into user.ref_with_source(x) values(4)", + "skip_e2e": true, "plan": { "QueryType": "INSERT", "Original": "insert into user.ref_with_source(x) values(4)", @@ -498,6 +519,7 @@ { "comment": "delete from reference table - query send to source table", "query": "delete from source_of_ref where col = 1", + "skip_e2e": true, "plan": { "QueryType": "DELETE", "Original": "delete from source_of_ref where col = 1", @@ -520,6 +542,7 @@ { "comment": "update from reference table - query send to source table", "query": "update source_of_ref set x = 4 where col = 1", + "skip_e2e": true, "plan": { "QueryType": "UPDATE", "Original": "update source_of_ref set x = 4 where col = 1", @@ -542,6 +565,7 @@ { "comment": "insert from reference table - query send to source table", "query": "insert into source_of_ref(x) values(4)", + "skip_e2e": true, "plan": { "QueryType": "INSERT", "Original": "insert into source_of_ref(x) values(4)", @@ -564,6 +588,7 @@ { "comment": "delete from reference table qualified with unsharded - query send to source table", "query": "delete from main.source_of_ref where col = 1", + "skip_e2e": true, "plan": { "QueryType": "DELETE", "Original": "delete from main.source_of_ref where col = 1", @@ -586,6 +611,7 @@ { "comment": "update from reference table qualified with unsharded - query send to source table", "query": "update main.source_of_ref set x = 4 where col = 1", + "skip_e2e": true, "plan": { "QueryType": "UPDATE", "Original": "update main.source_of_ref set x = 4 where col = 1", @@ -608,6 +634,7 @@ { "comment": "insert from reference table qualified with unsharded - query send to source table", "query": "insert into main.source_of_ref(x) values(4)", + "skip_e2e": true, "plan": { "QueryType": "INSERT", "Original": "insert into main.source_of_ref(x) values(4)", @@ -630,6 +657,7 @@ { "comment": "delete from reference table with another name - query send to source table", "query": "delete from user.ref_with_source where col = 1", + "skip_e2e": true, "plan": { "QueryType": "DELETE", "Original": "delete from user.ref_with_source where col = 1", @@ -652,6 +680,7 @@ { "comment": "update from reference table with another name - query send to source table", "query": "update user.ref_with_source set x = 4 where col = 1", + "skip_e2e": true, "plan": { "QueryType": "UPDATE", "Original": "update user.ref_with_source set x = 4 where col = 1", @@ -674,6 +703,7 @@ { "comment": "insert from reference table with another name - query send to source table", "query": "insert into user.ref_with_source(x) values(4)", + "skip_e2e": true, "plan": { "QueryType": "INSERT", "Original": "insert into user.ref_with_source(x) values(4)", @@ -696,6 +726,7 @@ { "comment": "select with join to reference table in sharded keyspace: should route shard-scoped", "query": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", @@ -723,6 +754,7 @@ { "comment": "select with join to reference table in unsharded keyspace: should route shard-scoped", "query": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", @@ -750,6 +782,7 @@ { "comment": "two sharded and two unsharded reference table join - all should be merged into one route", "query": "select 1 from user u join user_extra ue on u.id = ue.user_id join main.source_of_ref sr on sr.foo = ue.foo join main.rerouted_ref rr on rr.bar = sr.bar", + "skip_e2e": true, "plan": { "QueryType": "SELECT", "Original": "select 1 from user u join user_extra ue on u.id = ue.user_id join main.source_of_ref sr on sr.foo = ue.foo join main.rerouted_ref rr on rr.bar = sr.bar", @@ -771,5 +804,145 @@ "user.user_extra" ] } + }, + { + "comment": "update reference table with join on sharded table", + "query": "update main.source_of_ref as sr join main.rerouted_ref as rr on sr.id = rr.id inner join user.music as m on sr.col = m.col set sr.tt = 5 where m.user_id = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update main.source_of_ref as sr join main.rerouted_ref as rr on sr.id = rr.id inner join user.music as m on sr.col = m.col set sr.tt = 5 where m.user_id = 1", + "Instructions": { + "OperatorType": "DMLWithInput", + "TargetTabletType": "PRIMARY", + "Offset": [ + "0:[0]" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0", + "JoinVars": { + "m_col": 0 + }, + "TableName": "music_rerouted_ref, source_of_ref", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select m.col from music as m where 1 != 1", + "Query": "select m.col from music as m where m.user_id = 1 lock in share mode", + "Table": "music", + "Values": [ + "1" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select sr.id from source_of_ref as sr, rerouted_ref as rr where 1 != 1", + "Query": "select sr.id from source_of_ref as sr, rerouted_ref as rr where sr.col = :m_col and sr.id = rr.id lock in share mode", + "Table": "rerouted_ref, source_of_ref" + } + ] + }, + { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref as sr set sr.tt = 5 where sr.id in ::dml_vals", + "Table": "source_of_ref" + } + ] + }, + "TablesUsed": [ + "main.rerouted_ref", + "main.source_of_ref", + "user.music" + ] + } + }, + { + "comment": "delete from reference table with join on sharded table", + "query": "delete sr from main.source_of_ref as sr join main.rerouted_ref as rr on sr.id = rr.id inner join user.music as m on sr.col = m.col where m.user_id = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete sr from main.source_of_ref as sr join main.rerouted_ref as rr on sr.id = rr.id inner join user.music as m on sr.col = m.col where m.user_id = 1", + "Instructions": { + "OperatorType": "DMLWithInput", + "TargetTabletType": "PRIMARY", + "Offset": [ + "0:[0]" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0", + "JoinVars": { + "m_col": 0 + }, + "TableName": "music_rerouted_ref, source_of_ref", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select m.col from music as m where 1 != 1", + "Query": "select m.col from music as m where m.user_id = 1", + "Table": "music", + "Values": [ + "1" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select sr.id from source_of_ref as sr, rerouted_ref as rr where 1 != 1", + "Query": "select sr.id from source_of_ref as sr, rerouted_ref as rr where sr.col = :m_col and sr.id = rr.id", + "Table": "rerouted_ref, source_of_ref" + } + ] + }, + { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref as sr where sr.id in ::dml_vals", + "Table": "source_of_ref" + } + ] + }, + "TablesUsed": [ + "main.rerouted_ref", + "main.source_of_ref", + "user.music" + ] + } } ] diff --git a/go/vt/vtgate/planbuilder/testdata/sampledata/user.sql b/go/vt/vtgate/planbuilder/testdata/sampledata/user.sql index 044a1ee140d..ff1afd68fca 100644 --- a/go/vt/vtgate/planbuilder/testdata/sampledata/user.sql +++ b/go/vt/vtgate/planbuilder/testdata/sampledata/user.sql @@ -11,4 +11,13 @@ INSERT INTO sales_extra(colx, cola, colb, start, end) VALUES (13, 'a_3', 'b_3',1000, 1500); INSERT INTO sales_extra(colx, cola, colb, start, end) -VALUES (14, 'a_4', 'b_4',1500, 2000); \ No newline at end of file +VALUES (14, 'a_4', 'b_4',1500, 2000); + +INSERT INTO music (id, user_id, col) +VALUES (100, 1, 'foo'); + +INSERT INTO source_of_ref (id, col, tt) +VALUES (200, 'foo', 2); + +INSERT INTO rerouted_ref (id, ref_col, name) +VALUES (200, 'bar', 'baz'); \ No newline at end of file diff --git a/go/vt/vtgate/planbuilder/testdata/schemas/main.sql b/go/vt/vtgate/planbuilder/testdata/schemas/main.sql index fb03b69419b..e615871cf2b 100644 --- a/go/vt/vtgate/planbuilder/testdata/schemas/main.sql +++ b/go/vt/vtgate/planbuilder/testdata/schemas/main.sql @@ -1,26 +1,44 @@ -CREATE TABLE `unsharded` ( - `id` INT NOT NULL PRIMARY KEY, - `col` VARCHAR(255) DEFAULT NULL, - `col1` VARCHAR(255) DEFAULT NULL, - `col2` VARCHAR(255) DEFAULT NULL, - `name` VARCHAR(255) DEFAULT NULL, - `baz` INT +CREATE TABLE `unsharded` +( + `id` INT NOT NULL PRIMARY KEY, + `col` VARCHAR(255) DEFAULT NULL, + `col1` VARCHAR(255) DEFAULT NULL, + `col2` VARCHAR(255) DEFAULT NULL, + `name` VARCHAR(255) DEFAULT NULL, + `baz` INT ); -CREATE TABLE `unsharded_auto` ( +CREATE TABLE `unsharded_auto` +( `id` INT NOT NULL PRIMARY KEY, `col1` VARCHAR(255) DEFAULT NULL, `col2` VARCHAR(255) DEFAULT NULL ); -CREATE TABLE `unsharded_a` ( +CREATE TABLE `unsharded_a` +( `id` INT NOT NULL PRIMARY KEY, `col` VARCHAR(255) DEFAULT NULL, `name` VARCHAR(255) DEFAULT NULL ); -CREATE TABLE `unsharded_b` ( +CREATE TABLE `unsharded_b` +( `id` INT NOT NULL PRIMARY KEY, `col` VARCHAR(255) DEFAULT NULL, `name` VARCHAR(255) DEFAULT NULL +); + +CREATE TABLE `source_of_ref` +( + `id` INT NOT NULL PRIMARY KEY, + `col` VARCHAR(255) DEFAULT NULL, + `tt` BIGINT DEFAULT NULL +); + +CREATE TABLE `rerouted_ref` +( + `id` INT NOT NULL PRIMARY KEY, + `ref_col` VARCHAR(255) DEFAULT NULL, + `name` VARCHAR(255) DEFAULT NULL ); \ No newline at end of file diff --git a/go/vt/vtgate/planbuilder/testdata/schemas/user.sql b/go/vt/vtgate/planbuilder/testdata/schemas/user.sql index 818d2508069..10c1886a992 100644 --- a/go/vt/vtgate/planbuilder/testdata/schemas/user.sql +++ b/go/vt/vtgate/planbuilder/testdata/schemas/user.sql @@ -1,25 +1,25 @@ CREATE TABLE user ( - id INT PRIMARY KEY, - col BIGINT, - intcol BIGINT, - user_id INT, - id1 INT, - id2 INT, - id3 INT, - m INT, - bar INT, - a INT, - name VARCHAR(255), - col1 VARCHAR(255), - col2 VARCHAR(255), - costly VARCHAR(255), - predef1 VARCHAR(255), - predef2 VARCHAR(255), - textcol1 VARCHAR(255), - textcol2 VARCHAR(255), - someColumn VARCHAR(255), - foo VARCHAR(255) + id INT PRIMARY KEY, + col BIGINT, + intcol BIGINT, + user_id INT, + id1 INT, + id2 INT, + id3 INT, + m INT, + bar INT, + a INT, + name VARCHAR(255), + col1 VARCHAR(255), + col2 VARCHAR(255), + costly VARCHAR(255), + predef1 VARCHAR(255), + predef2 VARCHAR(255), + textcol1 VARCHAR(255), + textcol2 VARCHAR(255), + someColumn VARCHAR(255), + foo VARCHAR(255) ); CREATE TABLE user_metadata @@ -34,15 +34,23 @@ CREATE TABLE user_metadata CREATE TABLE music ( - user_id INT, - id INT, - col1 VARCHAR(255), - col2 VARCHAR(255), - genre VARCHAR(255), + user_id INT, + id INT, + col VARCHAR(255), + col1 VARCHAR(255), + col2 VARCHAR(255), + genre VARCHAR(255), componist VARCHAR(255), PRIMARY KEY (user_id) ); +CREATE TABLE name_user_vdx +( + name INT, + keyspace_id VARBINARY(10), + primary key (name) +); + CREATE TABLE samecolvin ( col VARCHAR(255), @@ -118,69 +126,63 @@ CREATE TABLE authoritative CREATE TABLE colb_colc_map ( - colb INT PRIMARY KEY, - colc INT, + colb INT PRIMARY KEY, + colc INT, keyspace_id VARCHAR(255) ); CREATE TABLE seq ( - id INT, - next_id BIGINT, - cache BIGINT, + id INT, + next_id BIGINT, + cache BIGINT, PRIMARY KEY (id) ) COMMENT 'vitess_sequence'; CREATE TABLE user_extra ( - id INT, - user_id INT, - extra_id INT, - col INT, - m2 INT, + id INT, + user_id INT, + extra_id INT, + col INT, + m2 INT, PRIMARY KEY (id, extra_id) ); CREATE TABLE name_user_map ( - name VARCHAR(255), - keyspace_id VARCHAR(255) -); - -CREATE TABLE name_user_vdx -( - name VARCHAR(255), - keyspace_id VARCHAR(255) + name VARCHAR(255), + keyspace_id VARCHAR(255) ); CREATE TABLE costly_map ( - costly VARCHAR(255), - keyspace_id VARCHAR(255) + costly VARCHAR(255), + keyspace_id VARCHAR(255) ); CREATE TABLE unq_binary_idx ( - id INT PRIMARY KEY, - col1 INT + id INT PRIMARY KEY, + col1 INT ); CREATE TABLE sales ( - oid INT PRIMARY KEY, - col1 VARCHAR(255) + oid INT PRIMARY KEY, + col1 VARCHAR(255) ); CREATE TABLE sales_extra ( - colx INT PRIMARY KEY, - cola VARCHAR(255), - colb VARCHAR(255), - start INT, - end INT + colx INT PRIMARY KEY, + cola VARCHAR(255), + colb VARCHAR(255), + start INT, + end INT ); CREATE TABLE ref ( - col INT PRIMARY KEY + col INT PRIMARY KEY ); \ No newline at end of file diff --git a/go/vt/vtgate/planbuilder/testdata/set_cases.json b/go/vt/vtgate/planbuilder/testdata/set_cases.json index 58cb2fffa75..02c5603a03c 100644 --- a/go/vt/vtgate/planbuilder/testdata/set_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/set_cases.json @@ -605,5 +605,28 @@ ] } } + }, + { + "comment": "set last_insert_id with agrument to user defined variable", + "query": "set @foo = last_insert_id(1)", + "plan": { + "QueryType": "SET", + "Original": "set @foo = last_insert_id(1)", + "Instructions": { + "OperatorType": "Set", + "Ops": [ + { + "Type": "UserDefinedVariable", + "Name": "foo", + "Expr": "last_insert_id(1)" + } + ], + "Inputs": [ + { + "OperatorType": "SingleRow" + } + ] + } + } } ] diff --git a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json index 9241cec595c..55329586b0e 100644 --- a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json @@ -342,7 +342,12 @@ { "comment": "reference table delete with join", "query": "delete r from user u join ref_with_source r on u.col = r.col", - "plan": "VT12001: unsupported: DELETE on reference table with join" + "plan": "VT12001: unsupported: DML on reference table with join" + }, + { + "comment": "reference table update with join", + "query": "update user u join ref_with_source r on u.col = r.col set r.col = 5", + "plan": "VT12001: unsupported: DML on reference table with join" }, { "comment": "group_concat unsupported when needs full evaluation at vtgate with more than 1 column", diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index 988932f4414..62cdc019ddf 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -174,7 +174,7 @@ func (a *analyzer) newSemTable( Direct: a.binder.direct, ExprTypes: a.typer.m, Tables: a.tables.Tables, - Targets: a.binder.targets, + DMLTargets: a.binder.targets, NotSingleRouteErr: a.notSingleRouteErr, NotUnshardedErr: a.unshardedErr, Warning: a.warning, diff --git a/go/vt/vtgate/semantics/semantic_table.go b/go/vt/vtgate/semantics/semantic_table.go index 492259427c5..30a41ba5f12 100644 --- a/go/vt/vtgate/semantics/semantic_table.go +++ b/go/vt/vtgate/semantics/semantic_table.go @@ -130,8 +130,8 @@ type ( // It doesn't recurse inside derived tables to find the original dependencies. Direct ExprDependencies - // Targets contains the TableSet of each table getting modified by the update/delete statement. - Targets TableSet + // DMLTargets contains the TableSet of each table getting modified by the update/delete statement. + DMLTargets TableSet // ColumnEqualities is used for transitive closures (e.g., if a == b and b == c, then a == c). ColumnEqualities map[columnName][]sqlparser.Expr @@ -203,7 +203,7 @@ func (st *SemTable) CopyDependencies(from, to sqlparser.Expr) { // GetChildForeignKeysForTargets gets the child foreign keys as a list for all the target tables. func (st *SemTable) GetChildForeignKeysForTargets() (fks []vindexes.ChildFKInfo) { - for _, ts := range st.Targets.Constituents() { + for _, ts := range st.DMLTargets.Constituents() { fks = append(fks, st.childForeignKeysInvolved[ts]...) } return fks @@ -211,7 +211,7 @@ func (st *SemTable) GetChildForeignKeysForTargets() (fks []vindexes.ChildFKInfo) // GetChildForeignKeysForTableSet gets the child foreign keys as a listfor the TableSet. func (st *SemTable) GetChildForeignKeysForTableSet(target TableSet) (fks []vindexes.ChildFKInfo) { - for _, ts := range st.Targets.Constituents() { + for _, ts := range st.DMLTargets.Constituents() { if target.IsSolvedBy(ts) { fks = append(fks, st.childForeignKeysInvolved[ts]...) } @@ -239,7 +239,7 @@ func (st *SemTable) GetChildForeignKeysList() []vindexes.ChildFKInfo { // GetParentForeignKeysForTargets gets the parent foreign keys as a list for all the target tables. func (st *SemTable) GetParentForeignKeysForTargets() (fks []vindexes.ParentFKInfo) { - for _, ts := range st.Targets.Constituents() { + for _, ts := range st.DMLTargets.Constituents() { fks = append(fks, st.parentForeignKeysInvolved[ts]...) } return fks @@ -247,7 +247,7 @@ func (st *SemTable) GetParentForeignKeysForTargets() (fks []vindexes.ParentFKInf // GetParentForeignKeysForTableSet gets the parent foreign keys as a list for the TableSet. func (st *SemTable) GetParentForeignKeysForTableSet(target TableSet) (fks []vindexes.ParentFKInfo) { - for _, ts := range st.Targets.Constituents() { + for _, ts := range st.DMLTargets.Constituents() { if target.IsSolvedBy(ts) { fks = append(fks, st.parentForeignKeysInvolved[ts]...) } @@ -971,7 +971,7 @@ func (st *SemTable) UpdateChildFKExpr(origUpdExpr *sqlparser.UpdateExpr, newExpr // GetTargetTableSetForTableName returns the TableSet for the given table name from the target tables. func (st *SemTable) GetTargetTableSetForTableName(name sqlparser.TableName) (TableSet, error) { - for _, target := range st.Targets.Constituents() { + for _, target := range st.DMLTargets.Constituents() { tbl, err := st.Tables[target.TableOffset()].Name() if err != nil { return "", err diff --git a/go/vt/vtgate/vindexes/vschema.go b/go/vt/vtgate/vindexes/vschema.go index 3852bbfcde3..278dd6932ae 100644 --- a/go/vt/vtgate/vindexes/vschema.go +++ b/go/vt/vtgate/vindexes/vschema.go @@ -25,10 +25,9 @@ import ( "strings" "time" - "vitess.io/vitess/go/ptr" - "vitess.io/vitess/go/json2" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/ptr" "vitess.io/vitess/go/sqlescape" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" @@ -473,6 +472,40 @@ func buildGlobalTables(source *vschemapb.SrvVSchema, vschema *VSchema) { } } +// AddAdditionalGlobalTables adds unique tables from unsharded keyspaces to the global tables. +// It is expected to be called from the schema tracking code. Note that this is called after `BuildVSchema` +// which means that the global tables are already populated with the tables from the sharded keyspaces and from +// unsharded keyspaces which have tables specified in associated vschemas. +func AddAdditionalGlobalTables(source *vschemapb.SrvVSchema, vschema *VSchema) { + newTables := make(map[string]*Table) + + // Collect valid uniquely named tables from unsharded keyspaces. + for ksname, ks := range source.Keyspaces { + ksvschema := vschema.Keyspaces[ksname] + // Ignore sharded keyspaces and those flagged for explicit routing. + if ks.RequireExplicitRouting || ks.Sharded { + continue + } + for tname, table := range ksvschema.Tables { + // Ignore tables already global (i.e. if specified in the vschema of an unsharded keyspace) or ambiguous. + if _, found := vschema.globalTables[tname]; !found { + _, ok := newTables[tname] + if !ok { + table.Keyspace = ksvschema.Keyspace + newTables[tname] = table + } else { + newTables[tname] = nil + } + } + } + } + + // Mark new tables found just once as globally routable, rest as ambiguous. + for k, v := range newTables { + vschema.globalTables[k] = v + } +} + func buildKeyspaceGlobalTables(vschema *VSchema, ksvschema *KeyspaceSchema) { for tname, t := range ksvschema.Tables { if gt, ok := vschema.globalTables[tname]; ok { diff --git a/go/vt/vtgate/vindexes/vschema_routing_test.go b/go/vt/vtgate/vindexes/vschema_routing_test.go new file mode 100644 index 00000000000..48ac9239fbb --- /dev/null +++ b/go/vt/vtgate/vindexes/vschema_routing_test.go @@ -0,0 +1,500 @@ +/* +Copyright 2025 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vindexes + +import ( + "encoding/json" + "errors" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + vschemapb "vitess.io/vitess/go/vt/proto/vschema" + "vitess.io/vitess/go/vt/sqlparser" +) + +// TestAutoGlobalRoutingExtended tests the global routing of tables across various keyspace configurations, +// including unsharded and sharded keyspaces, with and without the RequireExplicitRouting flag. +func TestAutoGlobalRoutingExtended(t *testing.T) { + isTableGloballyRoutable := func(vschema *VSchema, tableName string) (isGlobal, isAmbiguous bool) { + table, err := vschema.FindTable("", tableName) + if err != nil { + if strings.Contains(err.Error(), "ambiguous") { + return false, true + } + return false, false + } + return table != nil, false + } + type testKeySpace struct { + name string + ks *vschemapb.Keyspace + } + unsharded1 := &testKeySpace{ + name: "unsharded1", + ks: &vschemapb.Keyspace{ + Tables: map[string]*vschemapb.Table{ + "table1": {}, + "table2": {}, + "scommon1": {}, + "ucommon3": {}, + }, + }, + } + unsharded2 := &testKeySpace{ + name: "unsharded2", + ks: &vschemapb.Keyspace{ + Tables: map[string]*vschemapb.Table{ + "table3": {}, + "table4": {}, + "scommon1": {}, + "scommon2": {}, + "ucommon3": {}, + }, + }, + } + sharded1 := &testKeySpace{ + name: "sharded1", + ks: &vschemapb.Keyspace{ + Sharded: true, + Vindexes: map[string]*vschemapb.Vindex{ + "xxhash": { + Type: "xxhash", + }, + }, + Tables: map[string]*vschemapb.Table{ + "table5": {}, + "scommon1": {}, + "scommon2": {}, + }, + }, + } + sharded2 := &testKeySpace{ + name: "sharded2", + ks: &vschemapb.Keyspace{ + Sharded: true, + Vindexes: map[string]*vschemapb.Vindex{ + "xxhash": { + Type: "xxhash", + }, + }, + Tables: map[string]*vschemapb.Table{ + "table6": {}, + "scommon2": {}, + "scommon3": {}, + }, + }, + } + for _, tables := range []*vschemapb.Keyspace{sharded1.ks, sharded2.ks} { + for _, t := range tables.Tables { + t.ColumnVindexes = append(t.ColumnVindexes, &vschemapb.ColumnVindex{ + Column: "c1", + Name: "xxhash", + }) + } + } + type testCase struct { + name string + keyspaces []*testKeySpace + expGlobalTables []string + expAmbiguousTables []string + explicit []string + } + testCases := []testCase{ + { + name: "no keyspaces", + keyspaces: []*testKeySpace{}, + expGlobalTables: nil, + expAmbiguousTables: nil, + }, + { + name: "one unsharded keyspace", + keyspaces: []*testKeySpace{unsharded1}, + expGlobalTables: []string{"table1", "table2", "scommon1", "ucommon3"}, + expAmbiguousTables: nil, + }, + { + name: "two unsharded keyspaces", + keyspaces: []*testKeySpace{unsharded1, unsharded2}, + expGlobalTables: []string{"table1", "table2", "table3", "table4", "scommon2"}, + expAmbiguousTables: []string{"scommon1", "ucommon3"}, + }, + { + name: "two unsharded keyspaces, one with RequireExplicitRouting", + keyspaces: []*testKeySpace{unsharded1, unsharded2}, + explicit: []string{"unsharded1"}, + expGlobalTables: []string{"table3", "table4", "scommon1", "scommon2", "ucommon3"}, + }, + { + name: "one sharded keyspace", + keyspaces: []*testKeySpace{sharded1}, + expGlobalTables: []string{"table5", "scommon1", "scommon2"}, + }, + { + name: "two sharded keyspaces", + keyspaces: []*testKeySpace{sharded1, sharded2}, + expGlobalTables: []string{"table5", "table6", "scommon1", "scommon3"}, + expAmbiguousTables: []string{"scommon2"}, + }, + { + name: "two sharded keyspaces, one with RequireExplicitRouting", + keyspaces: []*testKeySpace{sharded1, sharded2}, + explicit: []string{"sharded2"}, + expGlobalTables: []string{"table5", "scommon1", "scommon2"}, + }, + { + name: "two sharded keyspaces, both with RequireExplicitRouting", + keyspaces: []*testKeySpace{sharded1, sharded2}, + explicit: []string{"sharded1", "sharded2"}, + expGlobalTables: nil, + }, + { + name: "two sharded keyspaces, one unsharded keyspace", + keyspaces: []*testKeySpace{sharded1, sharded2, unsharded1}, + expGlobalTables: []string{"table1", "table2", "table5", "table6", "scommon3", "ucommon3"}, + expAmbiguousTables: []string{"scommon1", "scommon2"}, + }, + { + name: "two sharded keyspaces, one unsharded keyspace, one with RequireExplicitRouting", + keyspaces: []*testKeySpace{sharded1, sharded2, unsharded1, unsharded2}, + explicit: []string{"unsharded1"}, + expGlobalTables: []string{"table3", "table4", "table5", "table6", "scommon3", "ucommon3"}, + expAmbiguousTables: []string{"scommon1", "scommon2"}, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + allTables := make(map[string]bool) + source := &vschemapb.SrvVSchema{ + Keyspaces: make(map[string]*vschemapb.Keyspace), + } + for _, ks := range tc.keyspaces { + source.Keyspaces[ks.name] = ks.ks + // set it false for all keyspaces here and later override for those requested in the test case + ks.ks.RequireExplicitRouting = false + for tname := range ks.ks.Tables { + _, ok := allTables[tname] + if !ok { + allTables[tname] = true + } + } + } + for _, ksName := range tc.explicit { + source.Keyspaces[ksName].RequireExplicitRouting = true + } + vschema := BuildVSchema(source, sqlparser.NewTestParser()) + require.NotNil(t, vschema) + AddAdditionalGlobalTables(source, vschema) + + var globalTables, ambiguousTables []string + for tname := range allTables { + isGlobal, isAmbiguous := isTableGloballyRoutable(vschema, tname) + if isGlobal { + globalTables = append(globalTables, tname) + } else if isAmbiguous { + ambiguousTables = append(ambiguousTables, tname) + } + } + sort.Strings(globalTables) + sort.Strings(ambiguousTables) + sort.Strings(tc.expGlobalTables) + sort.Strings(tc.expAmbiguousTables) + require.EqualValuesf(t, tc.expGlobalTables, globalTables, "global tables mismatch") + require.EqualValuesf(t, tc.expAmbiguousTables, ambiguousTables, "ambiguous tables mismatch") + }) + } +} + +// TestAutoGlobalRouting tests adding tables in unsharded keyspaces to global routing if they don't have +// an associated VSchema which has the RequireExplicitRouting flag set. These tables should also not be +// already part of the global routing tables via the VSchema of sharded keyspaces. +func TestAutoGlobalRoutingBasic(t *testing.T) { + // Create two unsharded keyspaces and two sharded keyspaces, each with some common tables. + unsharded1 := &vschemapb.Keyspace{ + Tables: map[string]*vschemapb.Table{ + "table1": {}, // unique, should be added to global routing + "table2": {}, // unique, should be added to global routing + "scommon1": {}, // common with sharded1, should not be added to global routing because it is already global because of sharded1 + "ucommon3": {}, // common with unsharded2, should not be added to global routing because it is ambiguous because of unsharded2 + }, + } + unsharded2 := &vschemapb.Keyspace{ + Tables: map[string]*vschemapb.Table{ + "table3": {}, // unique, should be added to global routing + "table4": {}, // unique, should be added to global routing + "scommon1": {}, // common with sharded1, should not be added to global routing because it is already global because of sharded1 + "scommon2": {}, // common with sharded1, should not be added to global routing because it is already global because of sharded1 + "ucommon3": {}, // common with unsharded1, should not be added to global routing because it is ambiguous because of unsharded1 + }, + } + sharded1 := &vschemapb.Keyspace{ + Sharded: true, + Vindexes: map[string]*vschemapb.Vindex{ + "xxhash": { + Type: "xxhash", + }, + }, + Tables: map[string]*vschemapb.Table{ + "table5": {}, // unique, should be added to global routing + "scommon1": {}, // common with unsharded1 and unsharded2, should be added to global routing because it's in the vschema + "scommon2": {}, // common with unsharded2, not ambiguous because sharded2 sets RequireExplicitRouting + }, + } + sharded2 := &vschemapb.Keyspace{ + Sharded: true, + RequireExplicitRouting: true, + Vindexes: map[string]*vschemapb.Vindex{ + "xxhash": { + Type: "xxhash", + }, + }, + // none should be considered for choice as global or ambiguous because of RequireExplicitRouting + Tables: map[string]*vschemapb.Table{ + "table6": {}, // unique + "scommon2": {}, // common with sharded1, but has RequireExplicitRouting + "scommon3": {}, // unique + }, + } + for _, ks := range []*vschemapb.Keyspace{sharded1, sharded2} { + for _, t := range ks.Tables { + t.ColumnVindexes = append(t.ColumnVindexes, &vschemapb.ColumnVindex{ + Column: "c1", + Name: "xxhash", + }) + } + } + source := &vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "sharded1": sharded1, + "sharded2": sharded2, + }, + } + + vschema := BuildVSchema(source, sqlparser.NewTestParser()) + require.NotNil(t, vschema) + + // Check table is global + mustRouteGlobally := func(t *testing.T, tname, ksName string) { + t.Helper() + _, err := vschema.FindTable("", tname) + require.NoError(t, err) + // The vtgate data structures don't always set the keyspace name, so cannot reliably check that at the moment. + _ = ksName + } + + mustNotRouteGlobally := func(t *testing.T, tname string) { + t.Helper() + _, err := vschema.FindTable("", tname) + require.Error(t, err) + } + + // Verify the global tables + ks := vschema.Keyspaces["sharded1"] + require.EqualValues(t, vschema.globalTables, map[string]*Table{ + "table5": ks.Tables["table5"], + "scommon1": ks.Tables["scommon1"], + "scommon2": ks.Tables["scommon2"], + }) + mustRouteGlobally(t, "table5", "sharded1") + mustRouteGlobally(t, "scommon1", "sharded1") + mustRouteGlobally(t, "scommon2", "sharded1") + + // Add unsharded keyspaces to SrvVSchema and build VSchema + var err error + source.Keyspaces["unsharded1"] = unsharded1 + source.Keyspaces["unsharded2"] = unsharded2 + vschema.Keyspaces["unsharded1"], err = BuildKeyspace(unsharded1, sqlparser.NewTestParser()) + require.NoError(t, err) + vschema.Keyspaces["unsharded2"], err = BuildKeyspace(unsharded2, sqlparser.NewTestParser()) + require.NoError(t, err) + + // Verify the global tables don't change + mustRouteGlobally(t, "table5", "sharded1") + mustRouteGlobally(t, "scommon1", "sharded1") + mustRouteGlobally(t, "scommon2", "sharded1") + + // Add additional global tables and then verify that the unsharded global tables are added + AddAdditionalGlobalTables(source, vschema) + + mustRouteGlobally(t, "table1", "unsharded1") + mustRouteGlobally(t, "table2", "unsharded1") + + mustRouteGlobally(t, "table3", "unsharded2") + mustRouteGlobally(t, "table4", "unsharded2") + mustNotRouteGlobally(t, "ucommon3") + + mustRouteGlobally(t, "scommon1", "sharded1") + mustRouteGlobally(t, "scommon2", "sharded1") + mustRouteGlobally(t, "table5", "sharded1") + + mustNotRouteGlobally(t, "table6") + mustNotRouteGlobally(t, "scommon3") +} + +func TestVSchemaRoutingRules(t *testing.T) { + input := vschemapb.SrvVSchema{ + RoutingRules: &vschemapb.RoutingRules{ + Rules: []*vschemapb.RoutingRule{{ + FromTable: "rt1", + ToTables: []string{"ks1.t1", "ks2.t2"}, + }, { + FromTable: "rt2", + ToTables: []string{"ks2.t2"}, + }, { + FromTable: "escaped", + ToTables: []string{"`ks2`.`t2`"}, + }, { + FromTable: "dup", + ToTables: []string{"ks1.t1"}, + }, { + FromTable: "dup", + ToTables: []string{"ks1.t1"}, + }, { + FromTable: "badname", + ToTables: []string{"t1.t2.t3"}, + }, { + FromTable: "unqualified", + ToTables: []string{"t1"}, + }, { + FromTable: "badkeyspace", + ToTables: []string{"ks3.t1"}, + }, { + FromTable: "notfound", + ToTables: []string{"ks1.t2"}, + }}, + }, + Keyspaces: map[string]*vschemapb.Keyspace{ + "ks1": { + Sharded: true, + ForeignKeyMode: vschemapb.Keyspace_unmanaged, + Vindexes: map[string]*vschemapb.Vindex{ + "stfu1": { + Type: "stfu", + }, + }, + Tables: map[string]*vschemapb.Table{ + "t1": { + ColumnVindexes: []*vschemapb.ColumnVindex{ + { + Column: "c1", + Name: "stfu1", + }, + }, + }, + }, + }, + "ks2": { + ForeignKeyMode: vschemapb.Keyspace_managed, + Tables: map[string]*vschemapb.Table{ + "t2": {}, + }, + }, + }, + } + got := BuildVSchema(&input, sqlparser.NewTestParser()) + ks1 := &Keyspace{ + Name: "ks1", + Sharded: true, + } + ks2 := &Keyspace{ + Name: "ks2", + } + vindex1 := &stFU{ + name: "stfu1", + } + t1 := &Table{ + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: ks1, + ColumnVindexes: []*ColumnVindex{{ + Columns: []sqlparser.IdentifierCI{sqlparser.NewIdentifierCI("c1")}, + Type: "stfu", + Name: "stfu1", + Vindex: vindex1, + isUnique: vindex1.IsUnique(), + cost: vindex1.Cost(), + }}, + } + t1.Ordered = []*ColumnVindex{ + t1.ColumnVindexes[0], + } + t2 := &Table{ + Name: sqlparser.NewIdentifierCS("t2"), + Keyspace: ks2, + } + want := &VSchema{ + MirrorRules: map[string]*MirrorRule{}, + RoutingRules: map[string]*RoutingRule{ + "rt1": { + Error: errors.New("table rt1 has more than one target: [ks1.t1 ks2.t2]"), + }, + "rt2": { + Tables: []*Table{t2}, + }, + "escaped": { + Tables: []*Table{t2}, + }, + "dup": { + Error: errors.New("duplicate rule for entry dup"), + }, + "badname": { + Error: errors.New("invalid table name: 't1.t2.t3', it must be of the qualified form . (dots are not allowed in either name)"), + }, + "unqualified": { + Error: errors.New("invalid table name: 't1', it must be of the qualified form . (dots are not allowed in either name)"), + }, + "badkeyspace": { + Error: errors.New("VT05003: unknown database 'ks3' in vschema"), + }, + "notfound": { + Error: errors.New("table t2 not found"), + }, + }, + globalTables: map[string]*Table{ + "t1": t1, + "t2": t2, + }, + uniqueVindexes: map[string]Vindex{ + "stfu1": vindex1, + }, + Keyspaces: map[string]*KeyspaceSchema{ + "ks1": { + Keyspace: ks1, + ForeignKeyMode: vschemapb.Keyspace_unmanaged, + Tables: map[string]*Table{ + "t1": t1, + }, + Vindexes: map[string]Vindex{ + "stfu1": vindex1, + }, + }, + "ks2": { + ForeignKeyMode: vschemapb.Keyspace_managed, + Keyspace: ks2, + Tables: map[string]*Table{ + "t2": t2, + }, + Vindexes: map[string]Vindex{}, + }, + }, + } + gotb, _ := json.MarshalIndent(got, "", " ") + wantb, _ := json.MarshalIndent(want, "", " ") + assert.Equal(t, string(wantb), string(gotb), string(gotb)) +} diff --git a/go/vt/vtgate/vindexes/vschema_test.go b/go/vt/vtgate/vindexes/vschema_test.go index f9bcf43ddaa..70e70745e9b 100644 --- a/go/vt/vtgate/vindexes/vschema_test.go +++ b/go/vt/vtgate/vindexes/vschema_test.go @@ -748,157 +748,6 @@ func TestShardedVSchemaOwnerInfo(t *testing.T) { } } -func TestVSchemaRoutingRules(t *testing.T) { - input := vschemapb.SrvVSchema{ - RoutingRules: &vschemapb.RoutingRules{ - Rules: []*vschemapb.RoutingRule{{ - FromTable: "rt1", - ToTables: []string{"ks1.t1", "ks2.t2"}, - }, { - FromTable: "rt2", - ToTables: []string{"ks2.t2"}, - }, { - FromTable: "escaped", - ToTables: []string{"`ks2`.`t2`"}, - }, { - FromTable: "dup", - ToTables: []string{"ks1.t1"}, - }, { - FromTable: "dup", - ToTables: []string{"ks1.t1"}, - }, { - FromTable: "badname", - ToTables: []string{"t1.t2.t3"}, - }, { - FromTable: "unqualified", - ToTables: []string{"t1"}, - }, { - FromTable: "badkeyspace", - ToTables: []string{"ks3.t1"}, - }, { - FromTable: "notfound", - ToTables: []string{"ks1.t2"}, - }}, - }, - Keyspaces: map[string]*vschemapb.Keyspace{ - "ks1": { - Sharded: true, - ForeignKeyMode: vschemapb.Keyspace_unmanaged, - Vindexes: map[string]*vschemapb.Vindex{ - "stfu1": { - Type: "stfu", - }, - }, - Tables: map[string]*vschemapb.Table{ - "t1": { - ColumnVindexes: []*vschemapb.ColumnVindex{ - { - Column: "c1", - Name: "stfu1", - }, - }, - }, - }, - }, - "ks2": { - ForeignKeyMode: vschemapb.Keyspace_managed, - Tables: map[string]*vschemapb.Table{ - "t2": {}, - }, - }, - }, - } - got := BuildVSchema(&input, sqlparser.NewTestParser()) - ks1 := &Keyspace{ - Name: "ks1", - Sharded: true, - } - ks2 := &Keyspace{ - Name: "ks2", - } - vindex1 := &stFU{ - name: "stfu1", - } - t1 := &Table{ - Name: sqlparser.NewIdentifierCS("t1"), - Keyspace: ks1, - ColumnVindexes: []*ColumnVindex{{ - Columns: []sqlparser.IdentifierCI{sqlparser.NewIdentifierCI("c1")}, - Type: "stfu", - Name: "stfu1", - Vindex: vindex1, - isUnique: vindex1.IsUnique(), - cost: vindex1.Cost(), - }}, - } - t1.Ordered = []*ColumnVindex{ - t1.ColumnVindexes[0], - } - t2 := &Table{ - Name: sqlparser.NewIdentifierCS("t2"), - Keyspace: ks2, - } - want := &VSchema{ - MirrorRules: map[string]*MirrorRule{}, - RoutingRules: map[string]*RoutingRule{ - "rt1": { - Error: errors.New("table rt1 has more than one target: [ks1.t1 ks2.t2]"), - }, - "rt2": { - Tables: []*Table{t2}, - }, - "escaped": { - Tables: []*Table{t2}, - }, - "dup": { - Error: errors.New("duplicate rule for entry dup"), - }, - "badname": { - Error: errors.New("invalid table name: 't1.t2.t3', it must be of the qualified form . (dots are not allowed in either name)"), - }, - "unqualified": { - Error: errors.New("invalid table name: 't1', it must be of the qualified form . (dots are not allowed in either name)"), - }, - "badkeyspace": { - Error: errors.New("VT05003: unknown database 'ks3' in vschema"), - }, - "notfound": { - Error: errors.New("table t2 not found"), - }, - }, - globalTables: map[string]*Table{ - "t1": t1, - "t2": t2, - }, - uniqueVindexes: map[string]Vindex{ - "stfu1": vindex1, - }, - Keyspaces: map[string]*KeyspaceSchema{ - "ks1": { - Keyspace: ks1, - ForeignKeyMode: vschemapb.Keyspace_unmanaged, - Tables: map[string]*Table{ - "t1": t1, - }, - Vindexes: map[string]Vindex{ - "stfu1": vindex1, - }, - }, - "ks2": { - ForeignKeyMode: vschemapb.Keyspace_managed, - Keyspace: ks2, - Tables: map[string]*Table{ - "t2": t2, - }, - Vindexes: map[string]Vindex{}, - }, - }, - } - gotb, _ := json.MarshalIndent(got, "", " ") - wantb, _ := json.MarshalIndent(want, "", " ") - assert.Equal(t, string(wantb), string(gotb), string(gotb)) -} - func TestVSchemaMirrorRules(t *testing.T) { input := vschemapb.SrvVSchema{ MirrorRules: &vschemapb.MirrorRules{ diff --git a/go/vt/vtgate/vschema_manager.go b/go/vt/vtgate/vschema_manager.go index 62ea2cd3455..290971c45ed 100644 --- a/go/vt/vtgate/vschema_manager.go +++ b/go/vt/vtgate/vschema_manager.go @@ -194,6 +194,10 @@ func (vm *VSchemaManager) buildAndEnhanceVSchema(v *vschemapb.SrvVSchema) *vinde // We mark the keyspaces that have foreign key management in Vitess and have cyclic foreign keys // to have an error. This makes all queries against them to fail. markErrorIfCyclesInFk(vschema) + // Add tables from schema tracking into globally routable tables, if they are not already present. + // We need to skip if already present, to handle the case where MoveTables has switched traffic + // and removed the source vschema but not from the source database because user asked to --keep-data + vindexes.AddAdditionalGlobalTables(v, vschema) } return vschema } diff --git a/go/vt/vtorc/inst/analysis_dao.go b/go/vt/vtorc/inst/analysis_dao.go index fc91c28b021..7837955c541 100644 --- a/go/vt/vtorc/inst/analysis_dao.go +++ b/go/vt/vtorc/inst/analysis_dao.go @@ -311,7 +311,7 @@ func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAna a.AnalyzedInstancePrimaryAlias = topoproto.TabletAliasString(primaryTablet.Alias) a.AnalyzedInstanceBinlogCoordinates = BinlogCoordinates{ LogFile: m.GetString("binary_log_file"), - LogPos: m.GetUint32("binary_log_pos"), + LogPos: m.GetUint64("binary_log_pos"), Type: BinaryLog, } isStaleBinlogCoordinates := m.GetBool("is_stale_binlog_coordinates") diff --git a/go/vt/vtorc/inst/binlog.go b/go/vt/vtorc/inst/binlog.go index 9c115e4e457..b4abf34ac7e 100644 --- a/go/vt/vtorc/inst/binlog.go +++ b/go/vt/vtorc/inst/binlog.go @@ -40,22 +40,23 @@ const ( // BinlogCoordinates described binary log coordinates in the form of log file & log position. type BinlogCoordinates struct { LogFile string - LogPos uint32 + LogPos uint64 Type BinlogType } -// ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306 +// ParseBinlogCoordinates will parse a string representation such as "mysql-bin.000001:12345" +// into a BinlogCoordinates struct. func ParseBinlogCoordinates(logFileLogPos string) (*BinlogCoordinates, error) { tokens := strings.SplitN(logFileLogPos, ":", 2) if len(tokens) != 2 { return nil, fmt.Errorf("ParseBinlogCoordinates: Cannot parse BinlogCoordinates from %s. Expected format is file:pos", logFileLogPos) } - logPos, err := strconv.ParseUint(tokens[1], 10, 32) + logPos, err := strconv.ParseUint(tokens[1], 10, 64) if err != nil { return nil, fmt.Errorf("ParseBinlogCoordinates: invalid pos: %s", tokens[1]) } - return &BinlogCoordinates{LogFile: tokens[0], LogPos: uint32(logPos)}, nil + return &BinlogCoordinates{LogFile: tokens[0], LogPos: logPos}, nil } // DisplayString returns a user-friendly string representation of these coordinates @@ -177,6 +178,6 @@ func (binlogCoordinates *BinlogCoordinates) ExtractDetachedCoordinates() (isDeta } detachedCoordinates.LogFile = detachedCoordinatesSubmatch[1] logPos, _ := strconv.ParseUint(detachedCoordinatesSubmatch[2], 10, 32) - detachedCoordinates.LogPos = uint32(logPos) + detachedCoordinates.LogPos = logPos return true, detachedCoordinates } diff --git a/go/vt/vtorc/inst/binlog_test.go b/go/vt/vtorc/inst/binlog_test.go index bc0110e981c..1f73f3c0029 100644 --- a/go/vt/vtorc/inst/binlog_test.go +++ b/go/vt/vtorc/inst/binlog_test.go @@ -41,7 +41,7 @@ func TestPreviousFileCoordinates(t *testing.T) { require.NoError(t, err) require.Equal(t, previous.LogFile, "mysql-bin.000009") - require.Equal(t, previous.LogPos, uint32(0)) + require.Equal(t, previous.LogPos, uint64(0)) } func TestNextFileCoordinates(t *testing.T) { @@ -49,7 +49,7 @@ func TestNextFileCoordinates(t *testing.T) { require.NoError(t, err) require.Equal(t, next.LogFile, "mysql-bin.000011") - require.Equal(t, next.LogPos, uint32(0)) + require.Equal(t, next.LogPos, uint64(0)) } func TestBinlogCoordinates(t *testing.T) { diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 66aef7c8a78..9198514d6ed 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -548,14 +548,14 @@ func readInstanceRow(m sqlutils.RowMap) *Instance { instance.GtidPurged = m.GetString("gtid_purged") instance.GtidErrant = m.GetString("gtid_errant") instance.SelfBinlogCoordinates.LogFile = m.GetString("binary_log_file") - instance.SelfBinlogCoordinates.LogPos = m.GetUint32("binary_log_pos") + instance.SelfBinlogCoordinates.LogPos = m.GetUint64("binary_log_pos") instance.ReadBinlogCoordinates.LogFile = m.GetString("source_log_file") - instance.ReadBinlogCoordinates.LogPos = m.GetUint32("read_source_log_pos") + instance.ReadBinlogCoordinates.LogPos = m.GetUint64("read_source_log_pos") instance.ExecBinlogCoordinates.LogFile = m.GetString("relay_source_log_file") - instance.ExecBinlogCoordinates.LogPos = m.GetUint32("exec_source_log_pos") + instance.ExecBinlogCoordinates.LogPos = m.GetUint64("exec_source_log_pos") instance.IsDetached, _ = instance.ExecBinlogCoordinates.ExtractDetachedCoordinates() instance.RelaylogCoordinates.LogFile = m.GetString("relay_log_file") - instance.RelaylogCoordinates.LogPos = m.GetUint32("relay_log_pos") + instance.RelaylogCoordinates.LogPos = m.GetUint64("relay_log_pos") instance.RelaylogCoordinates.Type = RelayLog instance.LastSQLError = m.GetString("last_sql_error") instance.LastIOError = m.GetString("last_io_error") diff --git a/go/vt/vttablet/tabletserver/connpool/pool.go b/go/vt/vttablet/tabletserver/connpool/pool.go index 14fcc6d0f2e..141d8257062 100644 --- a/go/vt/vttablet/tabletserver/connpool/pool.go +++ b/go/vt/vttablet/tabletserver/connpool/pool.go @@ -69,6 +69,7 @@ func NewPool(env tabletenv.Env, name string, cfg tabletenv.ConnPoolConfig) *Pool config := smartconnpool.Config[*Conn]{ Capacity: int64(cfg.Size), IdleTimeout: cfg.IdleTimeout, + MaxIdleCount: int64(cfg.MaxIdleCount), MaxLifetime: cfg.MaxLifetime, RefreshInterval: mysqlctl.PoolDynamicHostnameResolution, } diff --git a/go/vt/vttablet/tabletserver/connpool/pool_test.go b/go/vt/vttablet/tabletserver/connpool/pool_test.go index 8cf27cbb327..c305c61b7b4 100644 --- a/go/vt/vttablet/tabletserver/connpool/pool_test.go +++ b/go/vt/vttablet/tabletserver/connpool/pool_test.go @@ -55,10 +55,10 @@ func TestConnPoolTimeout(t *testing.T) { defer db.Close() cfg := tabletenv.ConnPoolConfig{ - Size: 1, + Size: 1, + Timeout: time.Second, + IdleTimeout: 10 * time.Second, } - cfg.Timeout = time.Second - cfg.IdleTimeout = 10 * time.Second connPool := NewPool(tabletenv.NewEnv(vtenv.NewTestEnv(), nil, "PoolTest"), "TestPool", cfg) params := dbconfigs.New(db.ConnParams()) connPool.Open(params, params, params) @@ -135,6 +135,58 @@ func TestConnPoolSetCapacity(t *testing.T) { } } +// TestConnPoolMaxIdleCount tests the max idle count for the pool. +// The pool should close the idle connections if the idle count is more than the allowed idle count. +// Changing the pool capacity will affect the idle count allowed for that pool. +func TestConnPoolMaxIdleCount(t *testing.T) { + db := fakesqldb.New(t) + defer db.Close() + + cfg := tabletenv.ConnPoolConfig{ + Size: 5, + MaxIdleCount: 2, + } + connPool := NewPool(tabletenv.NewEnv(vtenv.NewTestEnv(), nil, "PoolTest"), "TestPool", cfg) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) + defer connPool.Close() + + assert.EqualValues(t, 5, connPool.Capacity(), "pool capacity should be 5") + assert.EqualValues(t, 2, connPool.IdleCount(), "pool idle count should be 2") + + var conns []*PooledConn + for i := 0; i < 3; i++ { + conn, err := connPool.Get(context.Background(), nil) + require.NoError(t, err) + conns = append(conns, conn) + } + + // after recycle - 1 idle connection + conns[0].Recycle() + assert.Zero(t, connPool.Metrics.IdleClosed(), "pool idle closed should be 0") + + // after recycle - 2 idle connection + conns[1].Recycle() + assert.Zero(t, connPool.Metrics.IdleClosed(), "pool idle closed should be 0") + + // after recycle - 3 idle connection, 1 will be closed + conns[2].Recycle() + assert.EqualValues(t, 1, connPool.Metrics.IdleClosed(), "pool idle closed should be 1") + + // changing the pool capacity will affect the idle count allowed for that pool. + // If setting the capacity to lower value than max idle count. + + err := connPool.SetCapacity(context.Background(), 4) + require.NoError(t, err) + assert.EqualValues(t, 4, connPool.Capacity(), "pool capacity should be 4") + assert.EqualValues(t, 2, connPool.IdleCount(), "pool idle count should be 2") + + err = connPool.SetCapacity(context.Background(), 1) + require.NoError(t, err) + assert.EqualValues(t, 1, connPool.Capacity(), "pool capacity should be 1") + assert.EqualValues(t, 1, connPool.IdleCount(), "pool idle count should be changed to 1") +} + func TestConnPoolStatJSON(t *testing.T) { db := fakesqldb.New(t) defer db.Close() diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 994999f2368..42cc300f92d 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -143,6 +143,9 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) { fs.DurationVar(¤tConfig.OltpReadPool.Timeout, "queryserver-config-query-pool-timeout", defaultConfig.OltpReadPool.Timeout, "query server query pool timeout, it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead.") fs.DurationVar(¤tConfig.OlapReadPool.Timeout, "queryserver-config-stream-pool-timeout", defaultConfig.OlapReadPool.Timeout, "query server stream pool timeout, it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout.") fs.DurationVar(¤tConfig.TxPool.Timeout, "queryserver-config-txpool-timeout", defaultConfig.TxPool.Timeout, "query server transaction pool timeout, it is how long vttablet waits if tx pool is full") + fs.IntVar(¤tConfig.OltpReadPool.MaxIdleCount, "queryserver-config-query-pool-max-idle-count", defaultConfig.OltpReadPool.MaxIdleCount, "query server query pool - maximum number of idle connections to retain in the pool. Use this to balance between faster response times during traffic bursts and resource efficiency during low-traffic periods.") + fs.IntVar(¤tConfig.OlapReadPool.MaxIdleCount, "queryserver-config-stream-pool-max-idle-count", defaultConfig.OlapReadPool.MaxIdleCount, "query server stream pool - maximum number of idle connections to retain in the pool. Use this to balance between faster response times during traffic bursts and resource efficiency during low-traffic periods.") + fs.IntVar(¤tConfig.TxPool.MaxIdleCount, "queryserver-config-txpool-max-idle-count", defaultConfig.TxPool.MaxIdleCount, "query server transaction pool - maximum number of idle connections to retain in the pool. Use this to balance between faster response times during traffic bursts and resource efficiency during low-traffic periods.") fs.DurationVar(¤tConfig.OltpReadPool.IdleTimeout, "queryserver-config-idle-timeout", defaultConfig.OltpReadPool.IdleTimeout, "query server idle timeout, vttablet manages various mysql connection pools. This config means if a connection has not been used in given idle timeout, this connection will be removed from pool. This effectively manages number of connection objects and optimize the pool performance.") fs.DurationVar(¤tConfig.OltpReadPool.MaxLifetime, "queryserver-config-pool-conn-max-lifetime", defaultConfig.OltpReadPool.MaxLifetime, "query server connection max lifetime, vttablet manages various mysql connection pools. This config means if a connection has lived at least this long, it connection will be removed from pool upon the next time it is returned to the pool.") @@ -424,6 +427,7 @@ type ConnPoolConfig struct { Size int `json:"size,omitempty"` Timeout time.Duration `json:"timeoutSeconds,omitempty"` IdleTimeout time.Duration `json:"idleTimeoutSeconds,omitempty"` + MaxIdleCount int `json:"maxIdleCount,omitempty"` MaxLifetime time.Duration `json:"maxLifetimeSeconds,omitempty"` PrefillParallelism int `json:"prefillParallelism,omitempty"` } @@ -433,9 +437,10 @@ func (cfg *ConnPoolConfig) MarshalJSON() ([]byte, error) { tmp := struct { Proxy - Timeout string `json:"timeoutSeconds,omitempty"` - IdleTimeout string `json:"idleTimeoutSeconds,omitempty"` - MaxLifetime string `json:"maxLifetimeSeconds,omitempty"` + Timeout string `json:"timeoutSeconds,omitempty"` + IdleTimeout string `json:"idleTimeoutSeconds,omitempty"` + MaxIdleCount int `json:"maxIdleCount,omitempty"` + MaxLifetime string `json:"maxLifetimeSeconds,omitempty"` }{ Proxy: Proxy(*cfg), } @@ -460,6 +465,7 @@ func (cfg *ConnPoolConfig) UnmarshalJSON(data []byte) (err error) { Size int `json:"size,omitempty"` Timeout string `json:"timeoutSeconds,omitempty"` IdleTimeout string `json:"idleTimeoutSeconds,omitempty"` + MaxIdleCount int `json:"maxIdleCount,omitempty"` MaxLifetime string `json:"maxLifetimeSeconds,omitempty"` PrefillParallelism int `json:"prefillParallelism,omitempty"` } @@ -490,6 +496,7 @@ func (cfg *ConnPoolConfig) UnmarshalJSON(data []byte) (err error) { } cfg.Size = tmp.Size + cfg.MaxIdleCount = tmp.MaxIdleCount cfg.PrefillParallelism = tmp.PrefillParallelism return nil diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go index d16b6276964..9ae653bafb9 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go @@ -28,13 +28,12 @@ import ( "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/mysqlctl" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/throttler" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/yaml2" - - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) func TestConfigParse(t *testing.T) { @@ -49,10 +48,11 @@ func TestConfigParse(t *testing.T) { }, }, OltpReadPool: ConnPoolConfig{ - Size: 16, - Timeout: 10 * time.Second, - IdleTimeout: 20 * time.Second, - MaxLifetime: 50 * time.Second, + Size: 16, + Timeout: 10 * time.Second, + IdleTimeout: 20 * time.Second, + MaxLifetime: 50 * time.Second, + MaxIdleCount: 8, }, RowStreamer: RowStreamerConfig{ MaxInnoDBTrxHistLen: 1000, @@ -113,6 +113,7 @@ txPool: {} oltpReadPool: size: 16 idleTimeoutSeconds: 20s + maxIdleCount: 8 maxLifetimeSeconds: 50s `) gotCfg := cfg diff --git a/go/vt/vttest/vtprocess.go b/go/vt/vttest/vtprocess.go index 6371811a60e..1719fafd8a7 100644 --- a/go/vt/vttest/vtprocess.go +++ b/go/vt/vttest/vtprocess.go @@ -205,7 +205,7 @@ func VtcomboProcess(environment Environment, args *Config, mysql MySQLManager) ( if args.VtComboBindAddress != "" { vtcomboBindAddress = args.VtComboBindAddress } - grpcBindAddress := "127.0.0.1" + grpcBindAddress := "" if servenv.GRPCBindAddress() != "" { grpcBindAddress = servenv.GRPCBindAddress() } diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 984ff93095e..3251c7c8e3d 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -133,7 +133,8 @@ func TestEmergencyReparentShard(t *testing.T) { }, }, }) - goodReplica1RelayLogPos, _ := replication.ParseFilePosGTIDSet("relay-bin.000004:455") + goodReplica1RelayLogPos, err := replication.ParseFilePosGTIDSet("relay-bin.003222:18321744073709551612") // Requires all 64 bits or uint64 + require.NoError(t, err) goodReplica1.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{ GTIDSet: goodReplica1RelayLogPos, } @@ -182,7 +183,7 @@ func TestEmergencyReparentShard(t *testing.T) { // run EmergencyReparentShard waitReplicaTimeout := time.Second * 2 - err := vp.Run([]string{"EmergencyReparentShard", "--wait_replicas_timeout", waitReplicaTimeout.String(), newPrimary.Tablet.Keyspace + "/" + newPrimary.Tablet.Shard, + err = vp.Run([]string{"EmergencyReparentShard", "--wait_replicas_timeout", waitReplicaTimeout.String(), newPrimary.Tablet.Keyspace + "/" + newPrimary.Tablet.Shard, topoproto.TabletAliasString(newPrimary.Tablet.Alias)}) require.NoError(t, err) // check what was run diff --git a/java/example/pom.xml b/java/example/pom.xml index fa3220f51bd..fabab27dd77 100644 --- a/java/example/pom.xml +++ b/java/example/pom.xml @@ -32,7 +32,7 @@ mysql mysql-connector-java - 8.0.28 + 8.0.33 false diff --git a/java/grpc-client/src/test/resources/ca.config b/java/grpc-client/src/test/resources/ca.config index e0955f28ccf..c5758831e06 100644 --- a/java/grpc-client/src/test/resources/ca.config +++ b/java/grpc-client/src/test/resources/ca.config @@ -2,6 +2,7 @@ default_bits = 1024 default_keyfile = keyfile.pem distinguished_name = req_distinguished_name + x509_extensions = v3_ca attributes = req_attributes prompt = no output_password = mypass @@ -15,3 +16,5 @@ emailAddress = test@email.address [ req_attributes ] challengePassword = A challenge password +[ v3_ca ] + basicConstraints = CA:TRUE diff --git a/java/pom.xml b/java/pom.xml index 6742258a6b8..e68bbd5574e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -72,8 +72,8 @@ 4.1.110.Final 2.0.65.Final - 4.28.3 - 3.24.3 + 3.25.5 + 3.25.5 3.0.0 2.24.1 diff --git a/test/config.json b/test/config.json index c08097e2a99..dfc0910f10e 100644 --- a/test/config.json +++ b/test/config.json @@ -604,7 +604,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": ["upgrade_downgrade_query_serving_queries"] + "Tags": ["upgrade_downgrade_query_serving_queries_2"] }, "vtgate_queries_subquery": { "File": "unused.go", @@ -613,7 +613,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": ["upgrade_downgrade_query_serving_queries"] + "Tags": ["upgrade_downgrade_query_serving_queries_2"] }, "vtgate_queries_union": { "File": "unused.go", @@ -622,7 +622,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": ["upgrade_downgrade_query_serving_queries"] + "Tags": ["upgrade_downgrade_query_serving_queries_2"] }, "vtgate_queries_insert": { "File": "unused.go", @@ -631,7 +631,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": ["upgrade_downgrade_query_serving_queries"] + "Tags": ["upgrade_downgrade_query_serving_queries_2"] }, "vtgate_queries_vexplain": { "File": "unused.go", @@ -640,7 +640,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": ["upgrade_downgrade_query_serving_queries"] + "Tags": ["upgrade_downgrade_query_serving_queries_2"] }, "vtgate_queries_reference": { "File": "unused.go", @@ -649,7 +649,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 1, - "Tags": ["upgrade_downgrade_query_serving_queries"] + "Tags": ["upgrade_downgrade_query_serving_queries_2"] }, "vtgate_queries_random": { "File": "unused.go", @@ -658,7 +658,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 1, - "Tags": ["upgrade_downgrade_query_serving_queries"] + "Tags": ["upgrade_downgrade_query_serving_queries_2"] }, "vtgate_kill": { "File": "unused.go", @@ -667,7 +667,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 1, - "Tags": ["upgrade_downgrade_query_serving_queries"] + "Tags": ["upgrade_downgrade_query_serving_queries_2"] }, "vtgate_concurrentdml": { "File": "unused.go", @@ -1310,6 +1310,15 @@ "RetryMax": 1, "Tags": [] }, + "global_routing": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestGlobalRouting", "-timeout", "30m"], + "Command": [], + "Manual": false, + "Shard": "vreplication_v2", + "RetryMax": 1, + "Tags": [] + }, "vreplication_fk": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestFKWorkflow"], diff --git a/test/local_example.sh b/test/local_example.sh index 391e75a9224..27f512a34eb 100755 --- a/test/local_example.sh +++ b/test/local_example.sh @@ -98,5 +98,11 @@ mysql --table < ../common/select_customer80-_data.sql ./306_down_shard_0.sh -./401_teardown.sh +./401_backup.sh + +./402_list_backup.sh + +./403_restore_from_backup.sh + +./501_teardown.sh diff --git a/web/vtadmin/package-lock.json b/web/vtadmin/package-lock.json index 8ad7c67a5b4..5439928837c 100644 --- a/web/vtadmin/package-lock.json +++ b/web/vtadmin/package-lock.json @@ -28,6 +28,7 @@ "react": "^17.0.2", "react-dom": "^17.0.2", "react-flow-renderer": "^10.3.17", + "react-json-tree": "^0.19.0", "react-query": "^3.5.9", "react-router-dom": "^5.3.4", "react-tiny-popover": "^6.0.5", @@ -4828,8 +4829,7 @@ "node_modules/@types/lodash": { "version": "4.17.0", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz", - "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==", - "dev": true + "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==" }, "node_modules/@types/lodash-es": { "version": "4.17.12", @@ -6633,6 +6633,19 @@ "node": ">=6" } }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -6649,6 +6662,16 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "license": "MIT", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/colord": { "version": "2.9.3", "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", @@ -15231,6 +15254,18 @@ "node": ">=0.10.0" } }, + "node_modules/react-base16-styling": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/react-base16-styling/-/react-base16-styling-0.10.0.tgz", + "integrity": "sha512-H1k2eFB6M45OaiRru3PBXkuCcn2qNmx+gzLb4a9IPMR7tMH8oBRXU5jGbPDYG1Hz+82d88ED0vjR8BmqU3pQdg==", + "license": "MIT", + "dependencies": { + "@types/lodash": "^4.17.0", + "color": "^4.2.3", + "csstype": "^3.1.3", + "lodash-es": "^4.17.21" + } + }, "node_modules/react-dom": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-17.0.2.tgz", @@ -15288,6 +15323,20 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==" }, + "node_modules/react-json-tree": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/react-json-tree/-/react-json-tree-0.19.0.tgz", + "integrity": "sha512-PqT1WRVcWP+RROsZPQfNEKIC1iM/ZMfY4g5jN6oDnXp5593PPRAYgoHcgYCDjflAHQMtxl8XGdlTwIBdEGUXvw==", + "license": "MIT", + "dependencies": { + "@types/lodash": "^4.17.0", + "react-base16-styling": "^0.10.0" + }, + "peerDependencies": { + "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, "node_modules/react-query": { "version": "3.39.3", "resolved": "https://registry.npmjs.org/react-query/-/react-query-3.39.3.tgz", @@ -16161,6 +16210,21 @@ "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "license": "MIT" + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", diff --git a/web/vtadmin/package.json b/web/vtadmin/package.json index 5d4d5dc787a..3d850fd1cc9 100644 --- a/web/vtadmin/package.json +++ b/web/vtadmin/package.json @@ -27,6 +27,7 @@ "react": "^17.0.2", "react-dom": "^17.0.2", "react-flow-renderer": "^10.3.17", + "react-json-tree": "^0.19.0", "react-query": "^3.5.9", "react-router-dom": "^5.3.4", "react-tiny-popover": "^6.0.5", diff --git a/web/vtadmin/src/components/jsonViewTree/JSONViewTree.tsx b/web/vtadmin/src/components/jsonViewTree/JSONViewTree.tsx new file mode 100644 index 00000000000..af4e71db38a --- /dev/null +++ b/web/vtadmin/src/components/jsonViewTree/JSONViewTree.tsx @@ -0,0 +1,95 @@ +/** + * Copyright 2025 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { useState } from 'react'; +import { JSONTree } from 'react-json-tree'; + +const vtAdminTheme = { + scheme: 'vtadmin', + author: 'custom', + base00: '#ffffff', + base01: '#f6f8fa', + base02: '#e6e8eb', + base03: '#8c8c8c', + base04: '#3c3c3c', + base05: '#2c2c2c', + base06: '#0057b8', + base07: '#000000', + base08: '#00875a', + base09: '#2c2c2c', + base0A: '#e44d26', + base0B: '#2c2c2c', + base0C: '#1a73e8', + base0D: '#3d5afe', + base0E: '#3cba54', + base0F: '#ff6f61', +}; + +interface JSONViewTreeProps { + data: any; +} + +const JSONViewTree: React.FC = ({ data }) => { + const [expandAll, setExpandAll] = useState(false); + const [treeKey, setTreeKey] = useState(0); + + const handleExpand = () => { + setExpandAll(true); + setTreeKey((prev) => prev + 1); + }; + + const handleCollapse = () => { + setExpandAll(false); + setTreeKey((prev) => prev + 1); + }; + + const getItemString = (type: string, data: any) => { + if (Array.isArray(data)) { + return `${type}[${data.length}]`; + } + return type; + }; + + if (!data) return null; + return ( +
+
+ + +
+ expandAll} + /> +
+ ); +}; + +export default JSONViewTree; diff --git a/web/vtadmin/src/components/routes/keyspace/Keyspace.tsx b/web/vtadmin/src/components/routes/keyspace/Keyspace.tsx index 61bf52bb189..068f9cea2cd 100644 --- a/web/vtadmin/src/components/routes/keyspace/Keyspace.tsx +++ b/web/vtadmin/src/components/routes/keyspace/Keyspace.tsx @@ -19,7 +19,6 @@ import { Link, Redirect, Route } from 'react-router-dom'; import { useKeyspace } from '../../../hooks/api'; import { useDocumentTitle } from '../../../hooks/useDocumentTitle'; import { isReadOnlyMode } from '../../../util/env'; -import { Code } from '../../Code'; import { ContentContainer } from '../../layout/ContentContainer'; import { NavCrumbs } from '../../layout/NavCrumbs'; import { WorkspaceHeader } from '../../layout/WorkspaceHeader'; @@ -32,6 +31,8 @@ import { Advanced } from './Advanced'; import style from './Keyspace.module.scss'; import { KeyspaceShards } from './KeyspaceShards'; import { KeyspaceVSchema } from './KeyspaceVSchema'; +import JSONViewTree from '../../jsonViewTree/JSONViewTree'; +import { Code } from '../../Code'; interface RouteParams { clusterID: string; @@ -94,6 +95,7 @@ export const Keyspace = () => { + @@ -114,6 +116,11 @@ export const Keyspace = () => { + + + + + {!isReadOnlyMode() && ( diff --git a/web/vtadmin/src/components/routes/shard/Shard.tsx b/web/vtadmin/src/components/routes/shard/Shard.tsx index 686dc089d8f..8f713be9b13 100644 --- a/web/vtadmin/src/components/routes/shard/Shard.tsx +++ b/web/vtadmin/src/components/routes/shard/Shard.tsx @@ -24,12 +24,13 @@ import { WorkspaceTitle } from '../../layout/WorkspaceTitle'; import { ContentContainer } from '../../layout/ContentContainer'; import { Tab } from '../../tabs/Tab'; import { TabContainer } from '../../tabs/TabContainer'; -import { Code } from '../../Code'; import { useDocumentTitle } from '../../../hooks/useDocumentTitle'; import { KeyspaceLink } from '../../links/KeyspaceLink'; import { useKeyspace } from '../../../hooks/api'; import { ShardTablets } from './ShardTablets'; import Advanced from './Advanced'; +import JSONViewTree from '../../jsonViewTree/JSONViewTree'; +import { Code } from '../../Code'; interface RouteParams { clusterID: string; @@ -114,6 +115,7 @@ export const Shard = () => { + @@ -123,6 +125,7 @@ export const Shard = () => { {shard && } + {shard && } diff --git a/web/vtadmin/src/components/routes/stream/Stream.tsx b/web/vtadmin/src/components/routes/stream/Stream.tsx index 45e9846315a..6143ac3e302 100644 --- a/web/vtadmin/src/components/routes/stream/Stream.tsx +++ b/web/vtadmin/src/components/routes/stream/Stream.tsx @@ -13,17 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Link, useParams } from 'react-router-dom'; +import { Link, Redirect, Route, Switch, useParams, useRouteMatch } from 'react-router-dom'; import { useWorkflow } from '../../../hooks/api'; import { useDocumentTitle } from '../../../hooks/useDocumentTitle'; import { formatStreamKey, getStream } from '../../../util/workflows'; -import { Code } from '../../Code'; import { ContentContainer } from '../../layout/ContentContainer'; import { NavCrumbs } from '../../layout/NavCrumbs'; import { WorkspaceHeader } from '../../layout/WorkspaceHeader'; import { WorkspaceTitle } from '../../layout/WorkspaceTitle'; import style from './Stream.module.scss'; +import JSONViewTree from '../../jsonViewTree/JSONViewTree'; +import { TabContainer } from '../../tabs/TabContainer'; +import { Tab } from '../../tabs/Tab'; +import { Code } from '../../Code'; interface RouteParams { clusterID: string; @@ -36,6 +39,7 @@ interface RouteParams { export const Stream = () => { const params = useParams(); + const { path, url } = useRouteMatch(); const { data: workflow } = useWorkflow({ clusterID: params.clusterID, keyspace: params.keyspace, @@ -72,7 +76,17 @@ export const Stream = () => { - + + + + + + + {stream && } + {stream && } + + + ); diff --git a/web/vtadmin/src/components/routes/tablet/Tablet.tsx b/web/vtadmin/src/components/routes/tablet/Tablet.tsx index b0181251fd0..f4a9ca20248 100644 --- a/web/vtadmin/src/components/routes/tablet/Tablet.tsx +++ b/web/vtadmin/src/components/routes/tablet/Tablet.tsx @@ -19,7 +19,6 @@ import { useExperimentalTabletDebugVars, useTablet } from '../../../hooks/api'; import { useDocumentTitle } from '../../../hooks/useDocumentTitle'; import { isReadOnlyMode } from '../../../util/env'; import { formatDisplayType, formatState } from '../../../util/tablets'; -import { Code } from '../../Code'; import { ContentContainer } from '../../layout/ContentContainer'; import { NavCrumbs } from '../../layout/NavCrumbs'; import { WorkspaceHeader } from '../../layout/WorkspaceHeader'; @@ -34,6 +33,8 @@ import style from './Tablet.module.scss'; import { TabletCharts } from './TabletCharts'; import { env } from '../../../util/env'; import FullStatus from './FullStatus'; +import JSONViewTree from '../../jsonViewTree/JSONViewTree'; +import { Code } from '../../Code'; interface RouteParams { alias: string; @@ -108,6 +109,7 @@ export const Tablet = () => { + @@ -128,6 +130,14 @@ export const Tablet = () => { + +
+ + + {env().VITE_ENABLE_EXPERIMENTAL_TABLET_DEBUG_VARS && } +
+ + {tablet && } {!isReadOnlyMode() && ( diff --git a/web/vtadmin/src/components/routes/workflow/Workflow.tsx b/web/vtadmin/src/components/routes/workflow/Workflow.tsx index 939d1c6d386..7b3f0df32a5 100644 --- a/web/vtadmin/src/components/routes/workflow/Workflow.tsx +++ b/web/vtadmin/src/components/routes/workflow/Workflow.tsx @@ -30,11 +30,12 @@ import { ContentContainer } from '../../layout/ContentContainer'; import { TabContainer } from '../../tabs/TabContainer'; import { Tab } from '../../tabs/Tab'; import { getStreams } from '../../../util/workflows'; -import { Code } from '../../Code'; import { ShardLink } from '../../links/ShardLink'; import { WorkflowVDiff } from './WorkflowVDiff'; import { Select } from '../../inputs/Select'; import { formatDateTimeShort } from '../../../util/time'; +import JSONViewTree from '../../jsonViewTree/JSONViewTree'; +import { Code } from '../../Code'; interface RouteParams { clusterID: string; @@ -168,6 +169,7 @@ export const Workflow = () => { + @@ -192,6 +194,10 @@ export const Workflow = () => { + + + +