UPSTREAM: <carry>: remove "SKIPPED" annotations, and replace with inc…

…lude/exclude CEL to make use of environment flags
openshift · Feb 26, 2025 · db6b8e5 · db6b8e5
1 parent 1daa2c4
commit db6b8e5
Show file tree

Hide file tree

Showing 2 changed files with 143 additions and 169 deletions.
diff --git a/openshift-hack/cmd/k8s-tests-ext/k8s-tests.go b/openshift-hack/cmd/k8s-tests-ext/k8s-tests.go
@@ -13,7 +13,7 @@ import (
 	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
 	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
-	"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
 	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
 	v "github.com/openshift-eng/openshift-tests-extension/pkg/version"
 
@@ -97,12 +97,153 @@ func main() {
 	//		  the environmental skip code from the enhancement once its implemented.
 	//		- Make sure to account for test renames that occur because of removal of these
 	//		  annotations
-	specs.Walk(func(spec *extensiontests.ExtensionTestSpec) {
+	specs.Walk(func(spec *et.ExtensionTestSpec) {
 		if annotations, ok := generated.Annotations[spec.Name]; ok {
 			spec.Name += annotations
 		}
 	})
 
+	// Tests that don't pass on disconnected, either due to requiring
+	// internet access for GitHub (e.g. many of the s2i builds), or
+	// because of pullthrough not supporting ICSP (https://bugzilla.redhat.com/show_bug.cgi?id=1918376)
+	specs.Select(et.NameContains("[sig-network] Networking should provide Internet connection for containers")).
+		Exclude(et.ExternalConnectivityEquals("Disconnected"))
+
+	// LoadBalancer tests in 1.31 require explicit platform-specific skips
+	// https://issues.redhat.com/browse/OCPBUGS-38840
+	specs.Select(et.NameContains("[Feature:LoadBalancer]")).
+		Exclude(et.Or(
+			et.PlatformEquals("alibabacloud"),
+			et.PlatformEquals("baremetal"),
+			et.PlatformEquals("ibmcloud"),
+			et.PlatformEquals("kubevirt"),
+			et.PlatformEquals("nutanix"),
+			et.PlatformEquals("openstack"),
+			et.PlatformEquals("ovirt"),
+			et.PlatformEquals("vsphere"),
+		))
+
+	// LoadBalancer tests in 1.31 require explicit platform-specific skips
+	// https://issues.redhat.com/browse/OCPBUGS-38840
+	specs.SelectAny([]et.SelectFunction{
+		et.NameContainsAll("[sig-network] LoadBalancers [Feature:LoadBalancer]", "UDP"),
+		et.NameContainsAll("[sig-network] LoadBalancers [Feature:LoadBalancer]", "session affinity"),
+	}).Exclude(et.PlatformEquals("aws"))
+
+	specs.SelectAny([]et.SelectFunction{
+		et.NameContains("Networking should provide Internet connection for containers"), // Azure does not allow ICMP traffic to internet.
+		// Azure CSI migration changed how we treat regions without zones.
+		// See https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=2066865
+		et.NameContains("[sig-storage] In-tree Volumes [Driver: azure-disk] [Testpattern: Dynamic PV (immediate binding)] topology should provision a volume and schedule a pod with AllowedTopologies"),
+		et.NameContains("[sig-storage] In-tree Volumes [Driver: azure-disk] [Testpattern: Dynamic PV (delayed binding)] topology should provision a volume and schedule a pod with AllowedTopologies"),
+	}).Exclude(et.PlatformEquals("azure"))
+
+	specs.SelectAny([]et.SelectFunction{
+		// Requires creation of a different compute instance in a different zone and is not compatible with volumeBindingMode of WaitForFirstConsumer which we use in 4.x
+		et.NameContains("[sig-storage] Multi-AZ Cluster Volumes should only be allowed to provision PDs in zones where nodes exist"),
+		// The following tests try to ssh directly to a node. None of our nodes have external IPs
+		et.NameContains("[k8s.io] [sig-node] crictl should be able to run crictl on the node"),
+		et.NameContains("[sig-storage] Flexvolumes should be mountable"),
+		et.NameContains("[sig-storage] Detaching volumes should not work when mount is in progress"),
+
+		// We are using ovn-kubernetes to conceal metadata
+		et.NameContains("[sig-auth] Metadata Concealment should run a check-metadata-concealment job to completion"),
+
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1740959
+		et.NameContains("[sig-api-machinery] AdmissionWebhook should be able to deny pod and configmap creation"),
+
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1745720
+		et.NameContains("[sig-storage] CSI Volumes [Driver: pd.csi.storage.gke.io]"),
+
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1749882
+		et.NameContains("[sig-storage] CSI Volumes CSI Topology test using GCE PD driver [Serial]"),
+
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1751367
+		et.NameContains("gce-localssd-scsi-fs"),
+
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1750851
+		// should be serial if/when it's re-enabled
+		et.NameContains("[HPA] Horizontal pod autoscaling (scale resource: Custom Metrics from Stackdriver)"),
+		et.NameContains("[Feature:CustomMetricsAutoscaling]"),
+	}).Exclude(et.PlatformEquals("gce"))
+
+	// These tests are skipped when openshift-tests needs to use a proxy to reach the
+	// cluster -- either because the test won't work while proxied, or because the test
+	// itself is testing a functionality using it's own proxy.
+	specs.SelectAny([]et.SelectFunction{
+		// These tests setup their own proxy, which won't work when we need to access the
+		// cluster through a proxy.
+		et.NameContains("[sig-cli] Kubectl client Simple pod should support exec through an HTTP proxy"),
+		et.NameContains("[sig-cli] Kubectl client Simple pod should support exec through kubectl proxy"),
+
+		// Kube currently uses the x/net/websockets pkg, which doesn't work with proxies.
+		// See: https://github.com/kubernetes/kubernetes/pull/103595
+		et.NameContains("[sig-node] Pods should support retrieving logs from the container over websockets"),
+		et.NameContains("[sig-cli] Kubectl Port forwarding With a server listening on localhost should support forwarding over websockets"),
+		et.NameContains("[sig-cli] Kubectl Port forwarding With a server listening on 0.0.0.0 should support forwarding over websockets"),
+		et.NameContains("[sig-node] Pods should support remote command execution over websockets"),
+
+		// These tests are flacky and require internet access
+		// See https://bugzilla.redhat.com/show_bug.cgi?id=2019375
+		et.NameContains("[sig-network] DNS should resolve DNS of partial qualified names for services"),
+		et.NameContains("[sig-network] DNS should provide DNS for the cluster"),
+		// This test does not work when using in-proxy cluster, see https://bugzilla.redhat.com/show_bug.cgi?id=2084560
+		et.NameContains("[sig-network] Networking should provide Internet connection for containers"),
+	}).Exclude(et.ExternalConnectivityEquals("Proxy"))
+
+	specs.SelectAny([]et.SelectFunction{
+		et.NameContains("[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance]"),
+		et.NameContains("[sig-node] NoExecuteTaintManager Single Pod [Serial] doesn't evict pod with tolerations from tainted nodes"),
+		et.NameContains("[sig-node] NoExecuteTaintManager Single Pod [Serial] eventually evict pod with finite tolerations from tainted nodes"),
+		et.NameContains("[sig-node] NoExecuteTaintManager Single Pod [Serial] evicts pods from tainted nodes"),
+		et.NameContains("[sig-node] NoExecuteTaintManager Single Pod [Serial] removing taint cancels eviction [Disruptive] [Conformance]"),
+		et.NameContains("[sig-node] NoExecuteTaintManager Single Pod [Serial] pods evicted from tainted nodes have pod disruption condition"),
+		et.NameContains("[sig-node] NoExecuteTaintManager Multiple Pods [Serial] evicts pods with minTolerationSeconds [Disruptive] [Conformance]"),
+		et.NameContains("[sig-node] NoExecuteTaintManager Multiple Pods [Serial] only evicts pods without tolerations from tainted nodes"),
+		et.NameContains("[sig-cli] Kubectl client Kubectl taint [Serial] should remove all the taints with the same key off a node"),
+		et.NameContains("[sig-network] LoadBalancers should be able to preserve UDP traffic when server pod cycles for a LoadBalancer service on different nodes"),
+		et.NameContains("[sig-network] LoadBalancers should be able to preserve UDP traffic when server pod cycles for a LoadBalancer service on the same nodes"),
+		et.NameContains("[sig-architecture] Conformance Tests should have at least two untainted nodes"),
+	}).Exclude(et.TopologyEquals("SingleReplicaTopology"))
+
+	// Tests which can't be run/don't make sense to run against a cluster with all optional capabilities disabled
+	specs.SelectAny([]et.SelectFunction{
+		// Requires CSISnapshot capability
+		et.NameContains("[Feature:VolumeSnapshotDataSource]"),
+		// Requires Storage capability
+		et.NameContains("[Driver: aws]"),
+		et.NameContains("[Feature:StorageProvider]"),
+	}).Exclude(et.NoOptionalCapabilitiesExist())
+
+	specs.SelectAll([]et.SelectFunction{
+		// ovn-kubernetes does not support named ports
+		et.NameContains("NetworkPolicy"),
+		et.NameContains("named port"),
+	}).Exclude(et.NetworkEquals("OVNKubernetes"))
+
+	specs.SelectAny([]et.SelectFunction{
+		// Calico is allowing the request to timeout instead of returning 'REFUSED'
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1825021 - ROKS: calico SDN results in a request timeout when accessing services with no endpoints
+		et.NameContains("[sig-network] Services should be rejected when no endpoints exist"),
+
+		// Nodes in ROKS have access to secrets in the cluster to handle encryption
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1825013 - ROKS: worker nodes have access to secrets in the cluster
+		et.NameContains("[sig-auth] [Feature:NodeAuthorizer] Getting a non-existent configmap should exit with the Forbidden error, not a NotFound error"),
+		et.NameContains("[sig-auth] [Feature:NodeAuthorizer] Getting a non-existent secret should exit with the Forbidden error, not a NotFound error"),
+		et.NameContains("[sig-auth] [Feature:NodeAuthorizer] Getting a secret for a workload the node has access to should succeed"),
+		et.NameContains("[sig-auth] [Feature:NodeAuthorizer] Getting an existing configmap should exit with the Forbidden error"),
+		et.NameContains("[sig-auth] [Feature:NodeAuthorizer] Getting an existing secret should exit with the Forbidden error"),
+
+		// Access to node external address is blocked from pods within a ROKS cluster by Calico
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1825016 - e2e: NodeAuthenticator tests use both external and internal addresses for node
+		et.NameContains("[sig-auth] [Feature:NodeAuthenticator] The kubelet's main port 10250 should reject requests with no credentials"),
+		et.NameContains("[sig-auth] [Feature:NodeAuthenticator] The kubelet can delegate ServiceAccount tokens to the API server"),
+
+		// Mode returned by RHEL7 worker contains an extra character not expected by the test: dgtrwx vs dtrwx
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1825024 - e2e: Failing test - HostPath should give a volume the correct mode
+		et.NameContains("[sig-storage] HostPath should give a volume the correct mode"),
+	}).Exclude(et.PlatformEquals("ibmroks"))
+
 	kubeTestsExtension.AddSpecs(specs)
 
 	// Cobra stuff