Add e2e downgrade automatic cancellation test

See #19365 (comment)

Signed-off-by: Chun-Hung Tseng <henrytseng@google.com>

tests/e2e/cluster_downgrade_test.go

@@ -145,7 +145,9 @@ func testDowngradeUpgrade(t *testing.T, numberOfMembersToDowngrade int, clusterS
 		t.Logf("Cancelling downgrade before enabling")
 		e2e.DowngradeCancel(t, epc)
 		t.Log("Downgrade cancelled, validating if cluster is in the right state")
-		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersion))
+		identicalVersions, err := generateIdenticalVersions(clusterSize, currentVersion)
+		require.NoError(t, err)
+		e2e.ValidateMemberVersions(t, epc, identicalVersions)
 
 		return // No need to perform downgrading, end the test here
 	}
@@ -154,7 +156,9 @@ func testDowngradeUpgrade(t *testing.T, numberOfMembersToDowngrade int, clusterS
 		t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)")
 		e2e.DowngradeCancel(t, epc)
 		t.Log("Downgrade cancelled, validating if cluster is in the right state")
-		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersion))
+		identicalVersions, err := generateIdenticalVersions(clusterSize, currentVersion)
+		require.NoError(t, err)
+		e2e.ValidateMemberVersions(t, epc, identicalVersions)
 		return // No need to perform downgrading, end the test here
 	}
 
@@ -207,6 +211,92 @@ func testDowngradeUpgrade(t *testing.T, numberOfMembersToDowngrade int, clusterS
 	assert.Equal(t, beforeMembers.Members, afterMembers.Members)
 }
 
+func TestAutomaticDowngradeCancellationAfterCompletingDowngradingInClusterOf3(t *testing.T) {
+	clusterSize := 3
+
+	currentEtcdBinary := e2e.BinPath.Etcd
+	lastReleaseBinary := e2e.BinPath.EtcdLastRelease
+	if !fileutil.Exist(lastReleaseBinary) {
+		t.Skipf("%q does not exist", lastReleaseBinary)
+	}
+
+	currentVersion, err := e2e.GetVersionFromBinary(currentEtcdBinary)
+	require.NoError(t, err)
+	// wipe any pre-release suffix like -alpha.0 we see commonly in builds
+	currentVersion.PreRelease = ""
+
+	lastVersion, err := e2e.GetVersionFromBinary(lastReleaseBinary)
+	require.NoError(t, err)
+
+	require.Equalf(t, lastVersion.Minor, currentVersion.Minor-1, "unexpected minor version difference")
+	currentVersionStr := currentVersion.String()
+	lastVersionStr := lastVersion.String()
+
+	lastClusterVersion := semver.New(lastVersionStr)
+	lastClusterVersion.Patch = 0
+
+	e2e.BeforeTest(t)
+
+	t.Logf("Create cluster with version %s", currentVersionStr)
+	var snapshotCount uint64 = 10
+	epc := newCluster(t, clusterSize, snapshotCount)
+	for i := 0; i < len(epc.Procs); i++ {
+		e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
+			Cluster: currentVersionStr,
+			Server:  version.Version,
+			Storage: currentVersionStr,
+		})
+	}
+	cc := epc.Etcdctl()
+	t.Logf("Cluster created")
+	if len(epc.Procs) > 1 {
+		t.Log("Waiting health interval to required to make membership changes")
+		time.Sleep(etcdserver.HealthInterval)
+	}
+
+	t.Log("Adding member to test membership, but a learner avoid breaking quorum")
+	resp, err := cc.MemberAddAsLearner(context.Background(), "fake1", []string{"http://127.0.0.1:1001"})
+	require.NoError(t, err)
+	t.Log("Removing learner to test membership")
+	_, err = cc.MemberRemove(context.Background(), resp.Member.ID)
+	require.NoError(t, err)
+	beforeMembers, beforeKV := getMembersAndKeys(t, cc)
+
+	e2e.DowngradeEnable(t, epc, lastVersion)
+
+	t.Logf("Starting downgrade process for node 0 to %q", lastVersionStr)
+	err = e2e.DowngradeUpgradeMembersByID(t, nil, epc, []int{0}, currentVersion, lastClusterVersion)
+	require.NoError(t, err)
+
+	e2e.DowngradeCancel(t, epc)
+	t.Log("Downgrade cancelled, validating if cluster is in the right state")
+	e2e.ValidateMemberVersions(t, epc, generatePartialCancellationVersions(clusterSize, []int{0}, lastClusterVersion))
+
+	t.Logf("Starting downgrade process for node 1 and 2 to %q", lastVersionStr)
+	err = e2e.DowngradeUpgradeMembersByID(t, nil, epc, []int{1, 2}, currentVersion, lastClusterVersion)
+	require.NoError(t, err)
+
+	t.Log("Downgrade complete, validating if cluster is in the right state")
+	identicalVersions, err := generateIdenticalVersions(clusterSize, lastClusterVersion)
+	require.NoError(t, err)
+	e2e.ValidateMemberVersions(t, epc, identicalVersions)
+
+	afterMembers, afterKV := getMembersAndKeys(t, cc)
+	assert.Equal(t, beforeKV.Kvs, afterKV.Kvs)
+	assert.Equal(t, beforeMembers.Members, afterMembers.Members)
+
+	if len(epc.Procs) > 1 {
+		t.Log("Waiting health interval to required to make membership changes")
+		time.Sleep(etcdserver.HealthInterval)
+	}
+
+	e2e.DowngradeAutoCancelCheck(t, epc)
+	t.Log("Downgrade cancellation is automatically cancelled since the cluster has been downgraded, validating if cluster is in the right state")
+	identicalVersions, err = generateIdenticalVersions(clusterSize, lastClusterVersion)
+	require.NoError(t, err)
+	e2e.ValidateMemberVersions(t, epc, identicalVersions)
+}
+
 func newCluster(t *testing.T, clusterSize int, snapshotCount uint64) *e2e.EtcdProcessCluster {
 	epc, err := e2e.NewEtcdProcessCluster(context.TODO(), t,
 		e2e.WithClusterSize(clusterSize),
@@ -298,18 +388,29 @@ func getMembersAndKeys(t *testing.T, cc *e2e.EtcdctlV3) (*clientv3.MemberListRes
 	return members, kvs
 }
 
-func generateIdenticalVersions(clusterSize int, ver *semver.Version) []*version.Versions {
+func generateIdenticalVersions(clusterSize int, ver *semver.Version) ([]*version.Versions, error) {
 	ret := make([]*version.Versions, clusterSize)
 
+	// storage version string is non-empty starting from 3.6.0
+	ver3_6_0, err := semver.NewVersion("3.6.0")
+	if err != nil {
+		return nil, err
+	}
+
+	storage_str := ver.String()
+	if ver.Compare(*ver3_6_0) == -1 {
+		storage_str = ""
+	}
+
 	for i := range clusterSize {
 		ret[i] = &version.Versions{
 			Cluster: ver.String(),
 			Server:  ver.String(),
-			Storage: ver.String(),
+			Storage: storage_str,
 		}
 	}
 
-	return ret
+	return ret, nil
 }
 
 func generatePartialCancellationVersions(clusterSize int, membersToChange []int, ver *semver.Version) []*version.Versions {

tests/framework/e2e/downgrade.go

@@ -82,6 +82,19 @@ func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster) {
 	t.Log("Cluster downgrade cancellation is completed")
 }
 
+func DowngradeAutoCancelCheck(t *testing.T, epc *EtcdProcessCluster) {
+	c := epc.Etcdctl()
+
+	var err error
+	testutils.ExecuteWithTimeout(t, 1*time.Minute, func() {
+		t.Logf("etcdctl downgrade cancel")
+		err = c.DowngradeCancel(context.TODO())
+		require.Error(t, err, "no inflight downgrade job")
+	})
+
+	t.Log("Cluster downgrade is completed, thus downgrade cancellation is automatically cancelled")
+}
+
 func DowngradeUpgradeMembers(t *testing.T, lg *zap.Logger, clus *EtcdProcessCluster, numberOfMembersToChange int, currentVersion, targetVersion *semver.Version) error {
 	membersToChange := rand.Perm(len(clus.Procs))[:numberOfMembersToChange]
 	t.Logf("Elect members for operations on members: %v", membersToChange)
@@ -120,17 +133,10 @@ func DowngradeUpgradeMembersByID(t *testing.T, lg *zap.Logger, clus *EtcdProcess
 	lg.Info("Validating versions")
 	for _, memberID := range membersToChange {
 		member := clus.Procs[memberID]
-		if isDowngrade || len(membersToChange) == len(clus.Procs) {
-			ValidateVersion(t, clus.Cfg, member, version.Versions{
-				Cluster: targetVersion.String(),
-				Server:  targetVersion.String(),
-			})
-		} else {
-			ValidateVersion(t, clus.Cfg, member, version.Versions{
-				Cluster: currentVersion.String(),
-				Server:  targetVersion.String(),
-			})
-		}
+		ValidateVersion(t, clus.Cfg, member, version.Versions{
+			Cluster: targetVersion.String(),
+			Server:  targetVersion.String(),
+		})
 	}
 	return nil
 }