diff --git a/.github/workflows/static_checks_etc.yml b/.github/workflows/static_checks_etc.yml index b7f14688b95..8af08fbb9eb 100644 --- a/.github/workflows/static_checks_etc.yml +++ b/.github/workflows/static_checks_etc.yml @@ -197,7 +197,7 @@ jobs: - name: Install golangci-lint if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' - run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.2 + run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.5 - name: Clean Env if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml index 9fc4c00e18e..6363811f5b5 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml @@ -74,7 +74,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.5 + go-version: 1.24.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml index 7ca2c3ebbea..6175883c1b7 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml @@ -78,7 +78,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.5 + go-version: 1.24.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml index 03dd28c2e2a..91062f46211 100644 --- a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml +++ b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml @@ -86,7 +86,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.5 + go-version: 1.24.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml index e155be9f389..17b73a9dca4 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml @@ -78,7 +78,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.5 + go-version: 1.24.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml index 2d58d4c8fae..adff191250c 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml @@ -78,7 +78,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.5 + go-version: 1.24.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml index 8c82838594d..41861a063c9 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml @@ -78,7 +78,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.5 + go-version: 1.24.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml index b3e2cf6451e..6ed7293a1d4 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml @@ -78,7 +78,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.5 + go-version: 1.24.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml index fe0702b2fff..17467c5d57c 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml @@ -78,7 +78,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.5 + go-version: 1.24.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_semi_sync.yml b/.github/workflows/upgrade_downgrade_test_semi_sync.yml index cb950d43e4e..50863df04d2 100644 --- a/.github/workflows/upgrade_downgrade_test_semi_sync.yml +++ b/.github/workflows/upgrade_downgrade_test_semi_sync.yml @@ -74,7 +74,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.5 + go-version: 1.24.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.golangci.yml b/.golangci.yml index e907f56743b..63c547fcee9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,5 @@ run: - go: 1.23 + go: 1.24 timeout: 10m linters-settings: @@ -191,4 +191,4 @@ issues: # https://github.com/golangci/golangci/wiki/Configuration service: - golangci-lint-version: 1.52.2 # use the fixed version to not introduce new linters unexpectedly + golangci-lint-version: 1.64.5 # use the fixed version to not introduce new linters unexpectedly diff --git a/Makefile b/Makefile index aeb1955bf18..2ed645315c1 100644 --- a/Makefile +++ b/Makefile @@ -286,7 +286,7 @@ $(PROTO_GO_OUTS): minimaltools install_protoc-gen-go proto/*.proto # This rule builds the bootstrap images for all flavors. DOCKER_IMAGES_FOR_TEST = mysql80 mysql84 percona80 DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST) -BOOTSTRAP_VERSION=41 +BOOTSTRAP_VERSION=42 ensure_bootstrap_version: find docker/ -type f -exec sed -i "s/^\(ARG bootstrap_version\)=.*/\1=${BOOTSTRAP_VERSION}/" {} \; sed -i 's/\(^.*flag.String(\"bootstrap-version\",\) *\"[^\"]\+\"/\1 \"${BOOTSTRAP_VERSION}\"/' test.go diff --git a/build.env b/build.env index 6818975103f..9f513ec8f54 100755 --- a/build.env +++ b/build.env @@ -17,7 +17,7 @@ source ./tools/shell_functions.inc go version >/dev/null 2>&1 || fail "Go is not installed or is not in \$PATH. See https://vitess.io/contributing/build-from-source for install instructions." -goversion_min 1.23.5 || echo "Go version reported: `go version`. Version 1.23.5+ recommended. See https://vitess.io/contributing/build-from-source for install instructions." +goversion_min 1.24.0 || echo "Go version reported: `go version`. Version 1.24.0+ recommended. See https://vitess.io/contributing/build-from-source for install instructions." mkdir -p dist mkdir -p bin diff --git a/docker/bootstrap/CHANGELOG.md b/docker/bootstrap/CHANGELOG.md index 9728dffbab8..176540c0430 100644 --- a/docker/bootstrap/CHANGELOG.md +++ b/docker/bootstrap/CHANGELOG.md @@ -162,4 +162,7 @@ List of changes between bootstrap image versions. ## [41] - 2025-01-15 ### Changes - Update base image to bookworm -- Add MySQL84 image \ No newline at end of file +- Add MySQL84 image + +## [42] - 2025-02-14 +- Update build to golang 1.24.0 diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index e0866fe4bda..fcbe8b0c629 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 golang:1.23.5-bookworm +FROM --platform=linux/amd64 golang:1.24.0-bookworm # Install Vitess build dependencies RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile index c75a0a5ad4c..70d4787686c 100644 --- a/docker/lite/Dockerfile +++ b/docker/lite/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=linux/amd64 golang:1.23.5-bookworm AS builder +FROM --platform=linux/amd64 golang:1.24.0-bookworm AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mysql84 b/docker/lite/Dockerfile.mysql84 index e2836ca3047..047e3d1f90c 100644 --- a/docker/lite/Dockerfile.mysql84 +++ b/docker/lite/Dockerfile.mysql84 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=linux/amd64 golang:1.23.5-bookworm AS builder +FROM --platform=linux/amd64 golang:1.24.0-bookworm AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index 16e1da78627..011f20c4022 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=linux/amd64 golang:1.23.5-bookworm AS builder +FROM --platform=linux/amd64 golang:1.24.0-bookworm AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/vttestserver/Dockerfile.mysql80 b/docker/vttestserver/Dockerfile.mysql80 index 989aebe8740..ec01244e7a7 100644 --- a/docker/vttestserver/Dockerfile.mysql80 +++ b/docker/vttestserver/Dockerfile.mysql80 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=linux/amd64 golang:1.23.5-bookworm AS builder +FROM --platform=linux/amd64 golang:1.24.0-bookworm AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/vttestserver/Dockerfile.mysql84 b/docker/vttestserver/Dockerfile.mysql84 index ad944f500a0..9ddc78b640f 100644 --- a/docker/vttestserver/Dockerfile.mysql84 +++ b/docker/vttestserver/Dockerfile.mysql84 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=linux/amd64 golang:1.23.5-bookworm AS builder +FROM --platform=linux/amd64 golang:1.24.0-bookworm AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/go.mod b/go.mod index d4e98fda37a..3e73e0a74ef 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module vitess.io/vitess -go 1.23.5 +go 1.24.0 require ( cloud.google.com/go/storage v1.50.0 diff --git a/go/hack/ensure_swiss_map.go b/go/hack/ensure_swiss_map.go new file mode 100644 index 00000000000..b09bde1121c --- /dev/null +++ b/go/hack/ensure_swiss_map.go @@ -0,0 +1,39 @@ +//go:build !goexperiment.swissmap + +/* +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// This is invalid Go code, and it will fail to compile if you disable +// Swiss maps when building Vitess. Our runtime memory accounting system +// expects the map implementation in Go 1.24 to be Swiss. + +package hack + +var EnsureSwissMapIsAlwaysEnabled = [-1]struct{}{} diff --git a/go/hack/hack_test.go b/go/hack/hack_test.go index 9f71d82cf11..094bd14f5df 100644 --- a/go/hack/hack_test.go +++ b/go/hack/hack_test.go @@ -19,6 +19,7 @@ limitations under the License. package hack import ( + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -47,3 +48,48 @@ func TestStringToByte(t *testing.T) { b = StringBytes(s) assert.Nil(t, b) } + +func testMapSize[K comparable, V any](t *testing.T, gen func(i int) (K, V)) { + for _, size := range []int{16 * 1024, 128 * 1024, 256 * 1024, 512 * 1024, 1024 * 1024} { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + + m := make(map[K]V, size) + for i := 0; i < size; i++ { + k, v := gen(i) + m[k] = v + } + + runtime.GC() + runtime.ReadMemStats(&after) + + heapDiff := after.HeapAlloc - before.HeapAlloc + calcSize := RuntimeMapSize(m) + + assert.InEpsilonf(t, heapDiff, calcSize, 0.1, "%Tx%v heapDiff = %v, calcSize = %v", m, size, heapDiff, calcSize) + runtime.KeepAlive(m) + } +} + +func TestMapSize(t *testing.T) { + testMapSize(t, func(i int) (int, int) { + return i, i + }) + + testMapSize(t, func(i int) (uint32, uint32) { + return uint32(i), uint32(i) + }) + + testMapSize(t, func(i int) ([32]uint32, uint32) { + return [32]uint32{0: uint32(i)}, uint32(i) + }) + + testMapSize(t, func(i int) (uint32, [32]uint32) { + return uint32(i), [32]uint32{0: uint32(i)} + }) + + testMapSize(t, func(i int) ([32]uint32, [32]uint32) { + return [32]uint32{0: uint32(i)}, [32]uint32{0: uint32(i)} + }) +} diff --git a/go/hack/msize.go b/go/hack/msize.go index aead790e627..04024a2a282 100644 --- a/go/hack/msize.go +++ b/go/hack/msize.go @@ -32,6 +32,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package hack +import ( + "unsafe" +) + const ( _MaxSmallSize = 32768 smallSizeDiv = 8 @@ -74,3 +78,101 @@ func roundupsize(size uintptr) uintptr { func RuntimeAllocSize(size int64) int64 { return int64(roundupsize(uintptr(size))) } + +// RuntimeMapSize returns the size of the internal data structures of a Map. +// This is an internal implementation detail based on the Swiss Map implementation +// as of Go 1.24 and needs to be kept up-to-date after every Go release. +func RuntimeMapSize[K comparable, V any](themap map[K]V) (internalSize int64) { + // The following are copies of internal data structures as of Go 1.24 + // THEY MUST BE KEPT UP TO DATE + + // internal/abi/type.go + const SizeofType = 48 + + // internal/abi/map_swiss.go + type SwissMapType struct { + pad [SizeofType]uint8 + Key unsafe.Pointer + Elem unsafe.Pointer + Group unsafe.Pointer + Hasher func(unsafe.Pointer, uintptr) uintptr + GroupSize uintptr // == Group.Size_ + SlotSize uintptr // size of key/elem slot + ElemOff uintptr // offset of elem in key/elem slot + Flags uint32 + } + + // internal/runtime/maps/map.go + type Map struct { + used uint64 + seed uintptr + dirPtr unsafe.Pointer + dirLen int + globalDepth uint8 + globalShift uint8 + writing uint8 + clearSeq uint64 + } + + // internal/runtime/maps/groups.go + type groupsReference struct { + data unsafe.Pointer // data *[length]typ.Group + lengthMask uint64 + } + + // internal/runtime/maps/table.go + type table struct { + used uint16 + capacity uint16 + growthLeft uint16 + localDepth uint8 + index int + groups groupsReference + } + + directoryAt := func(m *Map, i uintptr) *table { + const PtrSize = 4 << (^uintptr(0) >> 63) + return *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + PtrSize*i)) + } + + // Converting the map to an interface will result in two ptr-sized words; + // like all interfaces, the first word points to an *abi.Type instance, + // which is specialized as SwissMapType for maps, and to the actual + // memory allocation, which is `*Map` + type MapInterface struct { + Type *SwissMapType + Data *Map + } + + mapiface := any(themap) + iface := (*MapInterface)(unsafe.Pointer(&mapiface)) + m := iface.Data + + var groupSize = int64(iface.Type.GroupSize) + internalSize = int64(unsafe.Sizeof(Map{})) + + if m.dirLen <= 0 { + // Small map optimization: we don't allocate tables at all if all the + // entries in a map fit in a single group + if m.dirPtr != nil { + internalSize += RuntimeAllocSize(groupSize) + } + } else { + // For normal maps, we iterate each table and add the size of all the + // groups it contains. + // See: internal/runtime/maps/export_test.go + var lastTab *table + for i := range m.dirLen { + t := directoryAt(m, uintptr(i)) + if t == lastTab { + continue + } + lastTab = t + + gc := int64(t.groups.lengthMask + 1) + internalSize += RuntimeAllocSize(groupSize * gc) + } + } + + return internalSize +} diff --git a/go/mysql/collations/cached_size.go b/go/mysql/collations/cached_size.go index 630bf41230a..407c02c4ee9 100644 --- a/go/mysql/collations/cached_size.go +++ b/go/mysql/collations/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,13 +17,7 @@ limitations under the License. package collations -import ( - "math" - "reflect" - "unsafe" - - hack "vitess.io/vitess/go/hack" -) +import hack "vitess.io/vitess/go/hack" //go:nocheckptr func (cached *Environment) CachedSize(alloc bool) int64 { @@ -36,28 +30,14 @@ func (cached *Environment) CachedSize(alloc bool) int64 { } // field byName map[string]vitess.io/vitess/go/mysql/collations.ID if cached.byName != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.byName) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 160)) - if len(cached.byName) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 160)) - } + size += hack.RuntimeMapSize(cached.byName) for k := range cached.byName { size += hack.RuntimeAllocSize(int64(len(k))) } } // field byCharset map[string]*vitess.io/vitess/go/mysql/collations.colldefaults if cached.byCharset != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.byCharset) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.byCharset) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.byCharset) for k, v := range cached.byCharset { size += hack.RuntimeAllocSize(int64(len(k))) if v != nil { @@ -67,42 +47,21 @@ func (cached *Environment) CachedSize(alloc bool) int64 { } // field byCharsetName map[vitess.io/vitess/go/mysql/collations.ID]string if cached.byCharsetName != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.byCharsetName) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 160)) - if len(cached.byCharsetName) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 160)) - } + size += hack.RuntimeMapSize(cached.byCharsetName) for _, v := range cached.byCharsetName { size += hack.RuntimeAllocSize(int64(len(v))) } } // field unsupported map[string]vitess.io/vitess/go/mysql/collations.ID if cached.unsupported != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.unsupported) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 160)) - if len(cached.unsupported) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 160)) - } + size += hack.RuntimeMapSize(cached.unsupported) for k := range cached.unsupported { size += hack.RuntimeAllocSize(int64(len(k))) } } // field byID map[vitess.io/vitess/go/mysql/collations.ID]string if cached.byID != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.byID) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 160)) - if len(cached.byID) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 160)) - } + size += hack.RuntimeMapSize(cached.byID) for _, v := range cached.byID { size += hack.RuntimeAllocSize(int64(len(v))) } diff --git a/go/mysql/collations/colldata/cached_size.go b/go/mysql/collations/colldata/cached_size.go index 190e1731651..a82cacc073b 100644 --- a/go/mysql/collations/colldata/cached_size.go +++ b/go/mysql/collations/colldata/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/mysql/decimal/cached_size.go b/go/mysql/decimal/cached_size.go index 87f6c201b80..10c5b15381d 100644 --- a/go/mysql/decimal/cached_size.go +++ b/go/mysql/decimal/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/mysql/json/cached_size.go b/go/mysql/json/cached_size.go index 27fd511dafc..0d57d9000d8 100644 --- a/go/mysql/json/cached_size.go +++ b/go/mysql/json/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/pools/smartconnpool/cached_size.go b/go/pools/smartconnpool/cached_size.go index 8c985349db3..2c794c46eca 100644 --- a/go/pools/smartconnpool/cached_size.go +++ b/go/pools/smartconnpool/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/sqltypes/cached_size.go b/go/sqltypes/cached_size.go index 53bc407278d..67bf017ed83 100644 --- a/go/sqltypes/cached_size.go +++ b/go/sqltypes/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/tools/sizegen/integration/cached_size.go b/go/tools/sizegen/integration/cached_size.go index 6023b29b7e4..619e77ad817 100644 --- a/go/tools/sizegen/integration/cached_size.go +++ b/go/tools/sizegen/integration/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,13 +17,7 @@ limitations under the License. package integration -import ( - "math" - "reflect" - "unsafe" - - hack "vitess.io/vitess/go/hack" -) +import hack "vitess.io/vitess/go/hack" type cachedObject interface { CachedSize(alloc bool) int64 @@ -89,14 +83,7 @@ func (cached *Map1) CachedSize(alloc bool) int64 { } // field field1 map[uint8]uint8 if cached.field1 != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.field1) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 32)) - if len(cached.field1) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 32)) - } + size += hack.RuntimeMapSize(cached.field1) } return size } @@ -112,14 +99,7 @@ func (cached *Map2) CachedSize(alloc bool) int64 { } // field field1 map[uint64]vitess.io/vitess/go/tools/sizegen/integration.A if cached.field1 != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.field1) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.field1) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.field1) } return size } @@ -135,14 +115,7 @@ func (cached *Map3) CachedSize(alloc bool) int64 { } // field field1 map[uint64]vitess.io/vitess/go/tools/sizegen/integration.B if cached.field1 != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.field1) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.field1) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.field1) for _, v := range cached.field1 { if cc, ok := v.(cachedObject); ok { size += cc.CachedSize(true) diff --git a/go/tools/sizegen/integration/integration_test.go b/go/tools/sizegen/integration/integration_test.go index f394f019b5a..cc2b269924e 100644 --- a/go/tools/sizegen/integration/integration_test.go +++ b/go/tools/sizegen/integration/integration_test.go @@ -20,6 +20,8 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/hack" ) @@ -27,9 +29,6 @@ func TestTypeSizes(t *testing.T) { var PtrSize = hack.RuntimeAllocSize(8) var SliceHeaderSize = hack.RuntimeAllocSize(3 * PtrSize) var FatPointerSize = hack.RuntimeAllocSize(2 * PtrSize) - var BucketHeaderSize = hack.RuntimeAllocSize(8) - var BucketSize = hack.RuntimeAllocSize(8) - var HashMapHeaderSize = hack.RuntimeAllocSize(48) cases := []struct { obj cachedObject @@ -60,17 +59,17 @@ func TestTypeSizes(t *testing.T) { {&Slice3{field1: []*Bimpl{{}, {}, {}, {}}}, SliceHeaderSize + PtrSize*4 + 8*4}, {&Map1{field1: nil}, PtrSize}, - {&Map1{field1: map[uint8]uint8{}}, PtrSize + HashMapHeaderSize}, - {&Map1{field1: map[uint8]uint8{0: 0}}, PtrSize + HashMapHeaderSize + BucketHeaderSize + 1*BucketSize + 1*BucketSize + PtrSize}, + {&Map1{field1: map[uint8]uint8{}}, 56}, + {&Map1{field1: map[uint8]uint8{0: 0}}, 80}, {&Map2{field1: nil}, PtrSize}, - {&Map2{field1: map[uint64]A{}}, PtrSize + HashMapHeaderSize}, - {&Map2{field1: map[uint64]A{0: {}}}, PtrSize + HashMapHeaderSize + BucketHeaderSize + 8*BucketSize + 16*BucketSize + PtrSize}, + {&Map2{field1: map[uint64]A{}}, 56}, + {&Map2{field1: map[uint64]A{0: {}}}, 264}, {&Map3{field1: nil}, PtrSize}, - {&Map3{field1: map[uint64]B{}}, PtrSize + HashMapHeaderSize}, - {&Map3{field1: map[uint64]B{0: &Bimpl{}}}, PtrSize + HashMapHeaderSize + BucketHeaderSize + 8*BucketSize + FatPointerSize*BucketSize + PtrSize + 8}, - {&Map3{field1: map[uint64]B{0: nil}}, PtrSize + HashMapHeaderSize + BucketHeaderSize + 8*BucketSize + FatPointerSize*BucketSize + PtrSize}, + {&Map3{field1: map[uint64]B{}}, 56}, + {&Map3{field1: map[uint64]B{0: &Bimpl{}}}, 272}, + {&Map3{field1: map[uint64]B{0: nil}}, 264}, {&String1{}, hack.RuntimeAllocSize(PtrSize*2 + 8)}, {&String1{field1: "1234"}, hack.RuntimeAllocSize(PtrSize*2+8) + hack.RuntimeAllocSize(4)}, @@ -79,9 +78,7 @@ func TestTypeSizes(t *testing.T) { for _, tt := range cases { t.Run(fmt.Sprintf("sizeof(%T)", tt.obj), func(t *testing.T) { size := tt.obj.CachedSize(true) - if size != tt.size { - t.Errorf("expected %T to be %d bytes, got %d", tt.obj, tt.size, size) - } + assert.Equalf(t, tt.size, size, "expected %T to be %d bytes, got %d", tt.obj, tt.size, size) }) } } diff --git a/go/tools/sizegen/integration/types.go b/go/tools/sizegen/integration/types.go index ff2a676b998..f38e66e690c 100644 --- a/go/tools/sizegen/integration/types.go +++ b/go/tools/sizegen/integration/types.go @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +//go:generate go run ../sizegen.go --in . --gen vitess.io/vitess/go/tools/sizegen/integration.* + // nolint package integration diff --git a/go/tools/sizegen/sizegen.go b/go/tools/sizegen/sizegen.go index cc733c8826d..32a22ea186f 100644 --- a/go/tools/sizegen/sizegen.go +++ b/go/tools/sizegen/sizegen.go @@ -34,7 +34,7 @@ import ( "vitess.io/vitess/go/tools/codegen" ) -const licenseFileHeader = `Copyright 2021 The Vitess Authors. +const licenseFileHeader = `Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -340,7 +340,7 @@ func (sizegen *sizegen) sizeImplForSlice(name *types.TypeName, st *types.Slice) } func (sizegen *sizegen) sizeImplForMap(name *types.TypeName, st *types.Map) (jen.Code, codeFlag) { - stmt := sizegen.sizeStmtForMap(jen.Op("*").Add(jen.Id("cached")), st) + stmt := sizegen.sizeStmtForMap(jen.Op("*").Add(jen.Id("cached"))) f := jen.Func() f.Params(jen.Id("cached").Op("*").Id(name.Name())) @@ -388,49 +388,9 @@ func (sizegen *sizegen) sizeImplForSignature(name *types.TypeName, _ *types.Sign return f, 0 } -func (sizegen *sizegen) sizeStmtForMap(fieldName *jen.Statement, m *types.Map) []jen.Code { - const bucketCnt = 8 - const sizeofHmap = int64(6 * 8) - - /* - type bmap struct { - // tophash generally contains the top byte of the hash value - // for each key in this bucket. If tophash[0] < minTopHash, - // tophash[0] is a bucket evacuation state instead. - tophash [bucketCnt]uint8 - // Followed by bucketCnt keys and then bucketCnt elems. - // NOTE: packing all the keys together and then all the elems together makes the - // code a bit more complicated than alternating key/elem/key/elem/... but it allows - // us to eliminate padding which would be needed for, e.g., map[int64]int8. - // Followed by an overflow pointer. - } - */ - sizeOfBucket := int( - bucketCnt + // tophash - bucketCnt*sizegen.sizes.Sizeof(m.Key()) + - bucketCnt*sizegen.sizes.Sizeof(m.Elem()) + - 8, // overflow pointer - ) - +func (sizegen *sizegen) sizeStmtForMap(fieldName *jen.Statement) []jen.Code { return []jen.Code{ - jen.Id("size").Op("+=").Lit(hack.RuntimeAllocSize(sizeofHmap)), - - jen.Id("hmap").Op(":=").Qual("reflect", "ValueOf").Call(fieldName), - - jen.Id("numBuckets").Op(":=").Id("int").Call( - jen.Qual("math", "Pow").Call(jen.Lit(2), jen.Id("float64").Call( - jen.Parens(jen.Op("*").Parens(jen.Op("*").Id("uint8")).Call( - jen.Qual("unsafe", "Pointer").Call(jen.Id("hmap").Dot("Pointer").Call(). - Op("+").Id("uintptr").Call(jen.Lit(9)))))))), - - jen.Id("numOldBuckets").Op(":=").Parens(jen.Op("*").Parens(jen.Op("*").Id("uint16")).Call( - jen.Qual("unsafe", "Pointer").Call( - jen.Id("hmap").Dot("Pointer").Call().Op("+").Id("uintptr").Call(jen.Lit(10))))), - - jen.Id("size").Op("+=").Do(mallocsize(jen.Int64().Call(jen.Id("numOldBuckets").Op("*").Lit(sizeOfBucket)))), - - jen.If(jen.Id("len").Call(fieldName).Op(">").Lit(0).Op("||").Id("numBuckets").Op(">").Lit(1)).Block( - jen.Id("size").Op("+=").Do(mallocsize(jen.Int64().Call(jen.Id("numBuckets").Op("*").Lit(sizeOfBucket))))), + jen.Id("size").Op("+=").Qual("vitess.io/vitess/go/hack", "RuntimeMapSize").Call(fieldName), } } @@ -512,11 +472,16 @@ func (sizegen *sizegen) sizeStmtForType(fieldName *jen.Statement, field types.Ty return nil, 0 case *types.Map: - keySize, keyFlag := sizegen.sizeStmtForType(jen.Id("k"), node.Key(), false) - valSize, valFlag := sizegen.sizeStmtForType(jen.Id("v"), node.Elem(), false) + const ( + SwissMapMaxKeyBytes = 128 + SwissMapMaxElemBytes = 128 + ) + + keySize, keyFlag := sizegen.sizeStmtForType(jen.Id("k"), node.Key(), sizegen.sizes.Sizeof(node.Key()) > SwissMapMaxKeyBytes) + valSize, valFlag := sizegen.sizeStmtForType(jen.Id("v"), node.Elem(), sizegen.sizes.Sizeof(node.Elem()) > SwissMapMaxElemBytes) return jen.If(fieldName.Clone().Op("!=").Nil()).BlockFunc(func(block *jen.Group) { - for _, stmt := range sizegen.sizeStmtForMap(fieldName, node) { + for _, stmt := range sizegen.sizeStmtForMap(fieldName) { block.Add(stmt) } diff --git a/go/vt/key/cached_size.go b/go/vt/key/cached_size.go index ef4e6e79896..1f20a8404d9 100644 --- a/go/vt/key/cached_size.go +++ b/go/vt/key/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/proto/query/cached_size.go b/go/vt/proto/query/cached_size.go index 58054d3c445..1f3d69513aa 100644 --- a/go/vt/proto/query/cached_size.go +++ b/go/vt/proto/query/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/proto/topodata/cached_size.go b/go/vt/proto/topodata/cached_size.go index 165c9ee4226..0fda6448975 100644 --- a/go/vt/proto/topodata/cached_size.go +++ b/go/vt/proto/topodata/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/proto/vttime/cached_size.go b/go/vt/proto/vttime/cached_size.go index f2b69dbefae..42252ec9d2c 100644 --- a/go/vt/proto/vttime/cached_size.go +++ b/go/vt/proto/vttime/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/schema/cached_size.go b/go/vt/schema/cached_size.go index 5ed67c01696..6f82d741c38 100644 --- a/go/vt/schema/cached_size.go +++ b/go/vt/schema/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/sqlparser/cached_size.go b/go/vt/sqlparser/cached_size.go index 4f17041bdbe..36bc371c0c8 100644 --- a/go/vt/sqlparser/cached_size.go +++ b/go/vt/sqlparser/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,13 +17,7 @@ limitations under the License. package sqlparser -import ( - "math" - "reflect" - "unsafe" - - hack "vitess.io/vitess/go/hack" -) +import hack "vitess.io/vitess/go/hack" type cachedObject interface { CachedSize(alloc bool) int64 @@ -814,14 +808,7 @@ func (cached *CommentDirectives) CachedSize(alloc bool) int64 { } // field m map[string]string if cached.m != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.m) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 272)) - if len(cached.m) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 272)) - } + size += hack.RuntimeMapSize(cached.m) for k, v := range cached.m { size += hack.RuntimeAllocSize(int64(len(k))) size += hack.RuntimeAllocSize(int64(len(v))) diff --git a/go/vt/srvtopo/cached_size.go b/go/vt/srvtopo/cached_size.go index 03dd1ceb0da..06a908fee70 100644 --- a/go/vt/srvtopo/cached_size.go +++ b/go/vt/srvtopo/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/tableacl/cached_size.go b/go/vt/tableacl/cached_size.go index 0bc5cc6c042..aa8c1511d54 100644 --- a/go/vt/tableacl/cached_size.go +++ b/go/vt/tableacl/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/vtenv/cached_size.go b/go/vt/vtenv/cached_size.go index 808cc4cdca3..44a982db830 100644 --- a/go/vt/vtenv/cached_size.go +++ b/go/vt/vtenv/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 50d3a4b6bbf..23412990b0a 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,13 +17,7 @@ limitations under the License. package engine -import ( - "math" - "reflect" - "unsafe" - - hack "vitess.io/vitess/go/hack" -) +import hack "vitess.io/vitess/go/hack" type cachedObject interface { CachedSize(alloc bool) int64 @@ -102,14 +96,7 @@ func (cached *Concatenate) CachedSize(alloc bool) int64 { } // field NoNeedToTypeCheck map[int]any if cached.NoNeedToTypeCheck != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.NoNeedToTypeCheck) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.NoNeedToTypeCheck) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.NoNeedToTypeCheck) } return size } @@ -222,14 +209,7 @@ func (cached *DMLWithInput) CachedSize(alloc bool) int64 { size += hack.RuntimeAllocSize(int64(cap(cached.BVList)) * int64(8)) for _, elem := range cached.BVList { if elem != nil { - size += int64(48) - hmap := reflect.ValueOf(elem) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(elem) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(elem) for k := range elem { size += hack.RuntimeAllocSize(int64(len(k))) } @@ -589,14 +569,7 @@ func (cached *Join) CachedSize(alloc bool) int64 { } // field Vars map[string]int if cached.Vars != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.Vars) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.Vars) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.Vars) for k := range cached.Vars { size += hack.RuntimeAllocSize(int64(len(k))) } @@ -881,14 +854,7 @@ func (cached *RecurseCTE) CachedSize(alloc bool) int64 { } // field Vars map[string]int if cached.Vars != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.Vars) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.Vars) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.Vars) for k := range cached.Vars { size += hack.RuntimeAllocSize(int64(len(k))) } @@ -1002,14 +968,7 @@ func (cached *RoutingParameters) CachedSize(alloc bool) int64 { } // field SysTableTableName map[string]vitess.io/vitess/go/vt/vtgate/evalengine.Expr if cached.SysTableTableName != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.SysTableTableName) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 272)) - if len(cached.SysTableTableName) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 272)) - } + size += hack.RuntimeMapSize(cached.SysTableTableName) for k, v := range cached.SysTableTableName { size += hack.RuntimeAllocSize(int64(len(k))) if cc, ok := v.(cachedObject); ok { @@ -1124,14 +1083,7 @@ func (cached *SemiJoin) CachedSize(alloc bool) int64 { } // field Vars map[string]int if cached.Vars != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.Vars) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.Vars) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.Vars) for k := range cached.Vars { size += hack.RuntimeAllocSize(int64(len(k))) } @@ -1381,14 +1333,7 @@ func (cached *Update) CachedSize(alloc bool) int64 { size += cached.DML.CachedSize(true) // field ChangedVindexValues map[string]*vitess.io/vitess/go/vt/vtgate/engine.VindexValues if cached.ChangedVindexValues != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.ChangedVindexValues) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.ChangedVindexValues) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.ChangedVindexValues) for k, v := range cached.ChangedVindexValues { size += hack.RuntimeAllocSize(int64(len(k))) size += v.CachedSize(true) @@ -1570,14 +1515,7 @@ func (cached *VindexValues) CachedSize(alloc bool) int64 { } // field EvalExprMap map[string]vitess.io/vitess/go/vt/vtgate/evalengine.Expr if cached.EvalExprMap != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.EvalExprMap) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 272)) - if len(cached.EvalExprMap) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 272)) - } + size += hack.RuntimeMapSize(cached.EvalExprMap) for k, v := range cached.EvalExprMap { size += hack.RuntimeAllocSize(int64(len(k))) if cc, ok := v.(cachedObject); ok { @@ -1635,14 +1573,7 @@ func (cached *shardRoute) CachedSize(alloc bool) int64 { size += cached.rs.CachedSize(true) // field bv map[string]*vitess.io/vitess/go/vt/proto/query.BindVariable if cached.bv != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.bv) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.bv) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.bv) for k, v := range cached.bv { size += hack.RuntimeAllocSize(int64(len(k))) size += v.CachedSize(true) diff --git a/go/vt/vtgate/evalengine/cached_size.go b/go/vt/vtgate/evalengine/cached_size.go index b953afda95c..fc7a02e84c1 100644 --- a/go/vt/vtgate/evalengine/cached_size.go +++ b/go/vt/vtgate/evalengine/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/vtgate/vindexes/cached_size.go b/go/vt/vtgate/vindexes/cached_size.go index ac68887c00d..94f6daa0c05 100644 --- a/go/vt/vtgate/vindexes/cached_size.go +++ b/go/vt/vtgate/vindexes/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,13 +17,7 @@ limitations under the License. package vindexes -import ( - "math" - "reflect" - "unsafe" - - hack "vitess.io/vitess/go/hack" -) +import hack "vitess.io/vitess/go/hack" type cachedObject interface { CachedSize(alloc bool) int64 @@ -327,14 +321,7 @@ func (cached *MultiCol) CachedSize(alloc bool) int64 { size += hack.RuntimeAllocSize(int64(len(cached.name))) // field columnVdx map[int]vitess.io/vitess/go/vt/vtgate/vindexes.Hashing if cached.columnVdx != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.columnVdx) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.columnVdx) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.columnVdx) for _, v := range cached.columnVdx { if cc, ok := v.(cachedObject); ok { size += cc.CachedSize(true) @@ -343,14 +330,7 @@ func (cached *MultiCol) CachedSize(alloc bool) int64 { } // field columnBytes map[int]int if cached.columnBytes != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.columnBytes) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 144)) - if len(cached.columnBytes) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 144)) - } + size += hack.RuntimeMapSize(cached.columnBytes) } return size } @@ -400,14 +380,7 @@ func (cached *NumericLookupTable) CachedSize(alloc bool) int64 { if alloc { size += int64(8) } - size += int64(48) - hmap := reflect.ValueOf(*cached) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 144)) - if len(*cached) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 144)) - } + size += hack.RuntimeMapSize(*cached) return size } @@ -428,14 +401,7 @@ func (cached *NumericStaticMap) CachedSize(alloc bool) int64 { } // field lookup vitess.io/vitess/go/vt/vtgate/vindexes.NumericLookupTable if cached.lookup != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.lookup) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 144)) - if len(cached.lookup) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 144)) - } + size += hack.RuntimeMapSize(cached.lookup) } // field unknownParams []string { @@ -479,14 +445,7 @@ func (cached *RegionJSON) CachedSize(alloc bool) int64 { size += hack.RuntimeAllocSize(int64(len(cached.name))) // field regionMap vitess.io/vitess/go/vt/vtgate/vindexes.RegionMap if cached.regionMap != nil { - size += int64(48) - hmap := reflect.ValueOf(cached.regionMap) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(cached.regionMap) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(cached.regionMap) for k := range cached.regionMap { size += hack.RuntimeAllocSize(int64(len(k))) } @@ -508,14 +467,7 @@ func (cached *RegionMap) CachedSize(alloc bool) int64 { if alloc { size += int64(8) } - size += int64(48) - hmap := reflect.ValueOf(*cached) - numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) - numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) - size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) - if len(*cached) > 0 || numBuckets > 1 { - size += hack.RuntimeAllocSize(int64(numBuckets * 208)) - } + size += hack.RuntimeMapSize(*cached) return size } func (cached *ReverseBits) CachedSize(alloc bool) int64 { diff --git a/go/vt/vttablet/tabletserver/cached_size.go b/go/vt/vttablet/tabletserver/cached_size.go index 96d09826356..5d357738d7b 100644 --- a/go/vt/vttablet/tabletserver/cached_size.go +++ b/go/vt/vttablet/tabletserver/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/vttablet/tabletserver/planbuilder/cached_size.go b/go/vt/vttablet/tabletserver/planbuilder/cached_size.go index 7e7f2f3ef4b..e10fce93cb6 100644 --- a/go/vt/vttablet/tabletserver/planbuilder/cached_size.go +++ b/go/vt/vttablet/tabletserver/planbuilder/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/vttablet/tabletserver/rules/cached_size.go b/go/vt/vttablet/tabletserver/rules/cached_size.go index d06a31ab8cb..5ea45661e27 100644 --- a/go/vt/vttablet/tabletserver/rules/cached_size.go +++ b/go/vt/vttablet/tabletserver/rules/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/vttablet/tabletserver/schema/cached_size.go b/go/vt/vttablet/tabletserver/schema/cached_size.go index 4db9f313644..13132e6acb2 100644 --- a/go/vt/vttablet/tabletserver/schema/cached_size.go +++ b/go/vt/vttablet/tabletserver/schema/cached_size.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2025 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/vt/vttablet/tabletserver/schema/schema.go b/go/vt/vttablet/tabletserver/schema/schema.go index e800477da3b..d5d1a6d29bc 100644 --- a/go/vt/vttablet/tabletserver/schema/schema.go +++ b/go/vt/vttablet/tabletserver/schema/schema.go @@ -17,12 +17,11 @@ limitations under the License. package schema import ( + "fmt" "sync" "time" "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/sqlparser" querypb "vitess.io/vitess/go/vt/proto/query" @@ -85,10 +84,10 @@ func (seq *SequenceInfo) Reset() { seq.LastVal = 0 } -func (seq *SequenceInfo) String() { +func (seq *SequenceInfo) String() string { seq.Lock() defer seq.Unlock() - log.Infof("SequenceInfo: NextVal: %d, LastVal: %d", seq.NextVal, seq.LastVal) + return fmt.Sprintf("SequenceInfo: NextVal: %d, LastVal: %d", seq.NextVal, seq.LastVal) } // MessageInfo contains info specific to message tables. @@ -131,6 +130,10 @@ type MessageInfo struct { IDType sqltypes.Type } +func (mi *MessageInfo) String() string { + return fmt.Sprintf("MessageInfo: AckWaitDuration: %v, PurgeAfterDuration: %v, BatchSize: %v, CacheSize: %v, PollInterval: %v, MinBackoff: %v, MaxBackoff: %v, IDType: %v", mi.AckWaitDuration, mi.PurgeAfterDuration, mi.BatchSize, mi.CacheSize, mi.PollInterval, mi.MinBackoff, mi.MaxBackoff, mi.IDType) +} + // NewTable creates a new Table. func NewTable(name string, tableType int) *Table { return &Table{ diff --git a/go/vt/vttablet/tabletserver/schema/schemaz_test.go b/go/vt/vttablet/tabletserver/schema/schemaz_test.go index 0127c796f67..c41d34bc82d 100644 --- a/go/vt/vttablet/tabletserver/schema/schemaz_test.go +++ b/go/vt/vttablet/tabletserver/schema/schemaz_test.go @@ -28,7 +28,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestSchamazHandler1(t *testing.T) { +func TestSchemazHandler1(t *testing.T) { resp := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/schemaz", nil) tables := initialSchema() @@ -50,7 +50,7 @@ func TestSchamazHandler1(t *testing.T) { `id: INT32
next_id: INT64
cache: INT64
increment: INT64
`, `id
`, `sequence`, - `{{0 0} 0 0}<nil>`, + `SequenceInfo: NextVal: 0, LastVal: 0<nil>`, } matched, err = regexp.Match(strings.Join(seq, `\s*`), body) require.NoError(t, err) diff --git a/misc/git/hooks/golangci-lint b/misc/git/hooks/golangci-lint index 21313316a12..3fd30f65533 100755 --- a/misc/git/hooks/golangci-lint +++ b/misc/git/hooks/golangci-lint @@ -14,7 +14,7 @@ # limitations under the License. # Required version of golangci-lint -REQUIRED_VERSION="v1.60.2" +REQUIRED_VERSION="v1.64.5" # Function to compare versions in pure Bash version_greater_or_equal() { @@ -67,4 +67,4 @@ gopackages=$(echo "$gofiles" | xargs -n1 dirname | sort -u | paste -sd ' ' -) # Lint the Go packages echo "Linting $gopackages" -golangci-lint run $gopackages \ No newline at end of file +golangci-lint run $gopackages diff --git a/test.go b/test.go index 247541009a6..e3d9222a96d 100755 --- a/test.go +++ b/test.go @@ -77,7 +77,7 @@ For example: // Flags var ( flavor = flag.String("flavor", "mysql80", "comma-separated bootstrap flavor(s) to run against (when using Docker mode). Available flavors: all,"+flavors) - bootstrapVersion = flag.String("bootstrap-version", "41", "the version identifier to use for the docker images") + bootstrapVersion = flag.String("bootstrap-version", "42", "the version identifier to use for the docker images") runCount = flag.Int("runs", 1, "run each test this many times") retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests") logPass = flag.Bool("log-pass", false, "log test output even if it passes") diff --git a/test/templates/dockerfile.tpl b/test/templates/dockerfile.tpl index 733377418f8..b3138a30f08 100644 --- a/test/templates/dockerfile.tpl +++ b/test/templates/dockerfile.tpl @@ -1,4 +1,4 @@ -ARG bootstrap_version=41 +ARG bootstrap_version=42 ARG image="vitess/bootstrap:${bootstrap_version}-{{.Platform}}" FROM "${image}"