From f382d1faaf473dd54d5a8d1ecf99013dbcf77008 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 12 Feb 2025 11:53:01 +0000 Subject: [PATCH] Add e2e downgrade automatic cancellation test See https://github.com/etcd-io/etcd/pull/19365#issuecomment-2649299898 Signed-off-by: Chun-Hung Tseng --- tests/e2e/cluster_downgrade_test.go | 111 ++++++++++++++++++++++++++-- tests/framework/e2e/downgrade.go | 28 ++++--- 2 files changed, 123 insertions(+), 16 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 3533adfff4c..e9aed2aae9e 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -145,7 +145,9 @@ func testDowngradeUpgrade(t *testing.T, numberOfMembersToDowngrade int, clusterS t.Logf("Cancelling downgrade before enabling") e2e.DowngradeCancel(t, epc) t.Log("Downgrade cancelled, validating if cluster is in the right state") - e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersion)) + identicalVersions, err := generateIdenticalVersions(clusterSize, currentVersion) + require.NoError(t, err) + e2e.ValidateMemberVersions(t, epc, identicalVersions) return // No need to perform downgrading, end the test here } @@ -154,7 +156,9 @@ func testDowngradeUpgrade(t *testing.T, numberOfMembersToDowngrade int, clusterS t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)") e2e.DowngradeCancel(t, epc) t.Log("Downgrade cancelled, validating if cluster is in the right state") - e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersion)) + identicalVersions, err := generateIdenticalVersions(clusterSize, currentVersion) + require.NoError(t, err) + e2e.ValidateMemberVersions(t, epc, identicalVersions) return // No need to perform downgrading, end the test here } @@ -207,6 +211,92 @@ func testDowngradeUpgrade(t *testing.T, numberOfMembersToDowngrade int, clusterS assert.Equal(t, beforeMembers.Members, afterMembers.Members) } +func TestAutomaticDowngradeCancellationAfterCompletingDowngradingInClusterOf3(t *testing.T) { + clusterSize := 3 + + currentEtcdBinary := e2e.BinPath.Etcd + lastReleaseBinary := e2e.BinPath.EtcdLastRelease + if !fileutil.Exist(lastReleaseBinary) { + t.Skipf("%q does not exist", lastReleaseBinary) + } + + currentVersion, err := e2e.GetVersionFromBinary(currentEtcdBinary) + require.NoError(t, err) + // wipe any pre-release suffix like -alpha.0 we see commonly in builds + currentVersion.PreRelease = "" + + lastVersion, err := e2e.GetVersionFromBinary(lastReleaseBinary) + require.NoError(t, err) + + require.Equalf(t, lastVersion.Minor, currentVersion.Minor-1, "unexpected minor version difference") + currentVersionStr := currentVersion.String() + lastVersionStr := lastVersion.String() + + lastClusterVersion := semver.New(lastVersionStr) + lastClusterVersion.Patch = 0 + + e2e.BeforeTest(t) + + t.Logf("Create cluster with version %s", currentVersionStr) + var snapshotCount uint64 = 10 + epc := newCluster(t, clusterSize, snapshotCount) + for i := 0; i < len(epc.Procs); i++ { + e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{ + Cluster: currentVersionStr, + Server: version.Version, + Storage: currentVersionStr, + }) + } + cc := epc.Etcdctl() + t.Logf("Cluster created") + if len(epc.Procs) > 1 { + t.Log("Waiting health interval to required to make membership changes") + time.Sleep(etcdserver.HealthInterval) + } + + t.Log("Adding member to test membership, but a learner avoid breaking quorum") + resp, err := cc.MemberAddAsLearner(context.Background(), "fake1", []string{"http://127.0.0.1:1001"}) + require.NoError(t, err) + t.Log("Removing learner to test membership") + _, err = cc.MemberRemove(context.Background(), resp.Member.ID) + require.NoError(t, err) + beforeMembers, beforeKV := getMembersAndKeys(t, cc) + + e2e.DowngradeEnable(t, epc, lastVersion) + + t.Logf("Starting downgrade process for node 0 to %q", lastVersionStr) + err = e2e.DowngradeUpgradeMembersByID(t, nil, epc, []int{0}, currentVersion, lastClusterVersion) + require.NoError(t, err) + + e2e.DowngradeCancel(t, epc) + t.Log("Downgrade cancelled, validating if cluster is in the right state") + e2e.ValidateMemberVersions(t, epc, generatePartialCancellationVersions(clusterSize, []int{0}, lastClusterVersion)) + + t.Logf("Starting downgrade process for node 1 and 2 to %q", lastVersionStr) + err = e2e.DowngradeUpgradeMembersByID(t, nil, epc, []int{1, 2}, currentVersion, lastClusterVersion) + require.NoError(t, err) + + t.Log("Downgrade complete, validating if cluster is in the right state") + identicalVersions, err := generateIdenticalVersions(clusterSize, lastClusterVersion) + require.NoError(t, err) + e2e.ValidateMemberVersions(t, epc, identicalVersions) + + afterMembers, afterKV := getMembersAndKeys(t, cc) + assert.Equal(t, beforeKV.Kvs, afterKV.Kvs) + assert.Equal(t, beforeMembers.Members, afterMembers.Members) + + if len(epc.Procs) > 1 { + t.Log("Waiting health interval to required to make membership changes") + time.Sleep(etcdserver.HealthInterval) + } + + e2e.DowngradeAutoCancelCheck(t, epc) + t.Log("Downgrade cancellation is automatically cancelled since the cluster has been downgraded, validating if cluster is in the right state") + identicalVersions, err = generateIdenticalVersions(clusterSize, lastClusterVersion) + require.NoError(t, err) + e2e.ValidateMemberVersions(t, epc, identicalVersions) +} + func newCluster(t *testing.T, clusterSize int, snapshotCount uint64) *e2e.EtcdProcessCluster { epc, err := e2e.NewEtcdProcessCluster(context.TODO(), t, e2e.WithClusterSize(clusterSize), @@ -298,18 +388,29 @@ func getMembersAndKeys(t *testing.T, cc *e2e.EtcdctlV3) (*clientv3.MemberListRes return members, kvs } -func generateIdenticalVersions(clusterSize int, ver *semver.Version) []*version.Versions { +func generateIdenticalVersions(clusterSize int, ver *semver.Version) ([]*version.Versions, error) { ret := make([]*version.Versions, clusterSize) + // storage version string is non-empty starting from 3.6.0 + ver3_6_0, err := semver.NewVersion("3.6.0") + if err != nil { + return nil, err + } + + storage_str := ver.String() + if ver.Compare(*ver3_6_0) == -1 { + storage_str = "" + } + for i := range clusterSize { ret[i] = &version.Versions{ Cluster: ver.String(), Server: ver.String(), - Storage: ver.String(), + Storage: storage_str, } } - return ret + return ret, nil } func generatePartialCancellationVersions(clusterSize int, membersToChange []int, ver *semver.Version) []*version.Versions { diff --git a/tests/framework/e2e/downgrade.go b/tests/framework/e2e/downgrade.go index a0a270dd182..d6bd78f75db 100644 --- a/tests/framework/e2e/downgrade.go +++ b/tests/framework/e2e/downgrade.go @@ -82,6 +82,19 @@ func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster) { t.Log("Cluster downgrade cancellation is completed") } +func DowngradeAutoCancelCheck(t *testing.T, epc *EtcdProcessCluster) { + c := epc.Etcdctl() + + var err error + testutils.ExecuteWithTimeout(t, 1*time.Minute, func() { + t.Logf("etcdctl downgrade cancel") + err = c.DowngradeCancel(context.TODO()) + require.Error(t, err, "no inflight downgrade job") + }) + + t.Log("Cluster downgrade is completed, thus downgrade cancellation is automatically cancelled") +} + func DowngradeUpgradeMembers(t *testing.T, lg *zap.Logger, clus *EtcdProcessCluster, numberOfMembersToChange int, currentVersion, targetVersion *semver.Version) error { membersToChange := rand.Perm(len(clus.Procs))[:numberOfMembersToChange] t.Logf("Elect members for operations on members: %v", membersToChange) @@ -120,17 +133,10 @@ func DowngradeUpgradeMembersByID(t *testing.T, lg *zap.Logger, clus *EtcdProcess lg.Info("Validating versions") for _, memberID := range membersToChange { member := clus.Procs[memberID] - if isDowngrade || len(membersToChange) == len(clus.Procs) { - ValidateVersion(t, clus.Cfg, member, version.Versions{ - Cluster: targetVersion.String(), - Server: targetVersion.String(), - }) - } else { - ValidateVersion(t, clus.Cfg, member, version.Versions{ - Cluster: currentVersion.String(), - Server: targetVersion.String(), - }) - } + ValidateVersion(t, clus.Cfg, member, version.Versions{ + Cluster: targetVersion.String(), + Server: targetVersion.String(), + }) } return nil }