Separate robustness test scenarios and increase number of times we ru…

…n exploratory tests in nightly

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>

.github/workflows/robustness-nightly.yaml

@@ -16,39 +16,44 @@ jobs:
     uses: ./.github/workflows/robustness-template.yaml
     with:
       etcdBranch: main
-      count: 80
+      count: 150
       testTimeout: 200m
       artifactName: main
       runs-on: "['ubuntu-latest-8-cores']"
+      scenario: TestRobustnessExploratory
   main-arm64:
     uses: ./.github/workflows/robustness-template.yaml
     with:
       etcdBranch: main
-      count: 80
+      count: 150
       testTimeout: 200m
       artifactName: main-arm64
       runs-on: "['actuated-arm64-8cpu-8gb']"
+      scenario: TestRobustnessExploratory
   release-35:
     uses: ./.github/workflows/robustness-template.yaml
     with:
       etcdBranch: release-3.5
-      count: 80
+      count: 150
       testTimeout: 200m
       artifactName: release-35
       runs-on: "['ubuntu-latest-8-cores']"
+      scenario: TestRobustnessExploratory
   release-35-arm64:
     uses: ./.github/workflows/robustness-template.yaml
     with:
       etcdBranch: release-3.5
-      count: 100
+      count: 150
       testTimeout: 200m
       artifactName: release-35-arm64
       runs-on: "['actuated-arm64-8cpu-8gb']"
+      scenario: TestRobustnessExploratory
   release-34:
     uses: ./.github/workflows/robustness-template.yaml
     with:
       etcdBranch: release-3.4
-      count: 80
+      count: 150
       testTimeout: 200m
       artifactName: release-34
       runs-on: "['ubuntu-latest-8-cores']"
+      scenario: TestRobustnessExploratory

.github/workflows/robustness-template.yaml

@@ -20,6 +20,9 @@ on:
         required: false
         type: string
         default: "['ubuntu-latest']"
+      scenario:
+        required: true
+        type: string
 permissions: read-all
 
 jobs:
@@ -45,7 +48,7 @@ jobs:
           sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf
           make install-lazyfs
           # Use --failfast to avoid overriding report generated by failed test
-          GO_TEST_FLAGS="-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestRobustness"
+          GO_TEST_FLAGS="-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run ${{ inputs.scenario }}"
           case "${ETCD_BRANCH}" in
             release-3.5)
               EXPECT_DEBUG=true GO_TEST_FLAGS=${GO_TEST_FLAGS} RESULTS_DIR=/tmp/results make test-robustness-release-3.5

.github/workflows/robustness.yaml

@@ -11,6 +11,7 @@ jobs:
       testTimeout: 30m
       artifactName: main
       runs-on: "['ubuntu-latest-8-cores']"
+      scenario: TestRobustness
   main-arm64:
     uses: ./.github/workflows/robustness-template.yaml
     with:
@@ -19,3 +20,4 @@ jobs:
       testTimeout: 30m
       artifactName: main-arm64
       runs-on: "['actuated-arm64-8cpu-8gb']"
+      scenario: TestRobustness

tests/robustness/main_test.go

@@ -39,9 +39,21 @@ func TestMain(m *testing.M) {
 	testRunner.TestMain(m)
 }
 
-func TestRobustness(t *testing.T) {
+func TestRobustnessExploratory(t *testing.T) {
 	testRunner.BeforeTest(t)
-	for _, scenario := range scenarios(t) {
+	for _, scenario := range exploratoryScenarios(t) {
+		t.Run(scenario.name, func(t *testing.T) {
+			lg := zaptest.NewLogger(t)
+			scenario.cluster.Logger = lg
+			ctx := context.Background()
+			testRobustness(ctx, t, lg, scenario)
+		})
+	}
+}
+
+func TestRobustnessRegression(t *testing.T) {
+	testRunner.BeforeTest(t)
+	for _, scenario := range regressionScenarios(t) {
 		t.Run(scenario.name, func(t *testing.T) {
 			lg := zaptest.NewLogger(t)
 			scenario.cluster.Logger = lg

tests/robustness/makefile.mk

@@ -12,22 +12,22 @@ test-robustness-release-3.4: /tmp/etcd-release-3.4-failpoints/bin
 
 .PHONY: test-robustness-issue14370
 test-robustness-issue14370: /tmp/etcd-v3.5.4-failpoints/bin
-	GO_TEST_FLAGS='-v --run=TestRobustness/Issue14370 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.4-failpoints/bin' make test-robustness && \
+	GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue14370 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.4-failpoints/bin' make test-robustness && \
 	 echo "Failed to reproduce" || echo "Successful reproduction"
 
 .PHONY: test-robustness-issue13766
 test-robustness-issue13766: /tmp/etcd-v3.5.2-failpoints/bin
-	GO_TEST_FLAGS='-v --run=TestRobustness/Issue13766 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.2-failpoints/bin' make test-robustness && \
+	GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue13766 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.2-failpoints/bin' make test-robustness && \
 	 echo "Failed to reproduce" || echo "Successful reproduction"
 
 .PHONY: test-robustness-issue14685
 test-robustness-issue14685: /tmp/etcd-v3.5.5-failpoints/bin
-	GO_TEST_FLAGS='-v --run=TestRobustness/Issue14685 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.5-failpoints/bin' make test-robustness && \
+	GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue14685 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.5-failpoints/bin' make test-robustness && \
 	 echo "Failed to reproduce" || echo "Successful reproduction"
 
 .PHONY: test-robustness-issue15271
 test-robustness-issue15271: /tmp/etcd-v3.5.7-failpoints/bin
-	GO_TEST_FLAGS='-v --run=TestRobustness/Issue15271 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.7-failpoints/bin' make test-robustness && \
+	GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue15271 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.7-failpoints/bin' make test-robustness && \
 	 echo "Failed to reproduce" || echo "Successful reproduction"
 
 # Failpoints

tests/robustness/scenarios.go

@@ -59,7 +59,7 @@ type testScenario struct {
 	watch     watchConfig
 }
 
-func scenarios(t *testing.T) []testScenario {
+func exploratoryScenarios(t *testing.T) []testScenario {
 	v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd)
 	if err != nil {
 		t.Fatalf("Failed checking etcd version binary, binary: %q, err: %v", e2e.BinPath.Etcd, err)
@@ -112,13 +112,22 @@ func scenarios(t *testing.T) []testScenario {
 			cluster: *e2e.NewConfig(clusterOfSize3Options...),
 		})
 	}
+	return scenarios
+}
+
+func regressionScenarios(t *testing.T) []testScenario {
+	v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd)
+	if err != nil {
+		t.Fatalf("Failed checking etcd version binary, binary: %q, err: %v", e2e.BinPath.Etcd, err)
+	}
+
+	scenarios := []testScenario{}
 	scenarios = append(scenarios, testScenario{
 		name:      "Issue14370",
 		failpoint: failpoint.RaftBeforeSavePanic,
 		profile:   traffic.LowTraffic,
 		traffic:   traffic.EtcdPutDeleteLease,
 		cluster: *e2e.NewConfig(
-			options.WithSubsetOptions(randomizableOptions...),
 			e2e.WithClusterSize(1),
 			e2e.WithGoFailEnabled(true),
 		),
@@ -129,7 +138,6 @@ func scenarios(t *testing.T) []testScenario {
 		profile:   traffic.LowTraffic,
 		traffic:   traffic.EtcdPutDeleteLease,
 		cluster: *e2e.NewConfig(
-			options.WithSubsetOptions(randomizableOptions...),
 			e2e.WithClusterSize(1),
 			e2e.WithGoFailEnabled(true),
 		),
@@ -151,7 +159,6 @@ func scenarios(t *testing.T) []testScenario {
 		profile: traffic.LowTraffic,
 		traffic: traffic.EtcdPutDeleteLease,
 		cluster: *e2e.NewConfig(
-			options.WithSubsetOptions(randomizableOptions...),
 			e2e.WithClusterSize(1),
 		),
 	})
@@ -163,7 +170,6 @@ func scenarios(t *testing.T) []testScenario {
 			profile:   traffic.HighTrafficProfile,
 			traffic:   traffic.EtcdPut,
 			cluster: *e2e.NewConfig(
-				options.WithSubsetOptions(randomizableOptions...),
 				e2e.WithSnapshotCatchUpEntries(100),
 				e2e.WithSnapshotCount(100),
 				e2e.WithPeerProxy(true),