Skip to content

Commit

Permalink
test(e2e): refactor e2e test case for test coverage (#144)
Browse files Browse the repository at this point in the history
* refactor: remove deprecated e2e configuration and test files

* test(e2e): add kerberos e2e test case

* test(e2e): add oidc loggin test case

* test(e2e): add oidc e2e test case

* test(e2e): add HDFS access and installation configuration

* test(e2e): add smoke tests for HDFS cluster operations and override PDB

* refactor: remove the static redirect URL, add cookie secure for dynamic http redirect URL

* feat(discovery): retrieve listener name from pod labels in getListenerAddress

* ci: add product version matrix and update resource limits

* chore: update chainsaw configuration with commented namespace and skipDelete options
  • Loading branch information
lwpk110 authored Jan 12, 2025
1 parent 95e03b2 commit 95b5474
Show file tree
Hide file tree
Showing 37 changed files with 948 additions and 383 deletions.
3 changes: 2 additions & 1 deletion .chainsaw.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ kind: Configuration
metadata:
name: custom-config
spec:
# namespace: chainsaw
timeouts:
apply: 120s
assert: 400s
cleanup: 240s
delete: 240s
error: 200s
exec: 200s
skipDelete: false
# skipDelete: true
failFast: true
parallel: 1
3 changes: 3 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ jobs:
strategy:
matrix:
k8s-version: ['1.26.15', '1.27.16']
product-version: ['3.3.6','3.4.0']
max-parallel: 1
steps:
- name: Clone the code
uses: actions/checkout@v4
Expand Down Expand Up @@ -61,4 +63,5 @@ jobs:
KINDTEST_K8S_VERSION: ${{ matrix.k8s-version }}
KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }}
KIND_KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }}
PRODUCT_VERSION: ${{ matrix.product-version }}
run: make chainsaw-test
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,7 @@ endif
# Tool Versions
KINDTEST_K8S_VERSION ?= 1.26.15
CHAINSAW_VERSION ?= v0.2.11
PRODUCT_VERSION ?= 3.3.6

KIND_IMAGE ?= kindest/node:v${KINDTEST_K8S_VERSION}
KIND_KUBECONFIG ?= ./kind-kubeconfig-$(KINDTEST_K8S_VERSION)
Expand Down Expand Up @@ -314,7 +315,7 @@ chainsaw-setup: ## Run the chainsaw setup

.PHONY: chainsaw-test
chainsaw-test: chainsaw ## Run the chainsaw test
KUBECONFIG=$(KIND_KUBECONFIG) $(CHAINSAW) test --cluster cluster-1=$(KIND_KUBECONFIG) --test-dir ./test/e2e/
echo "product_version: $(PRODUCT_VERSION)" | KUBECONFIG=$(KIND_KUBECONFIG) $(CHAINSAW) test --cluster cluster-1=$(KIND_KUBECONFIG) --test-dir ./test/e2e/ --values -

.PHONY: chainsaw-cleanup
chainsaw-cleanup: ## Run the chainsaw cleanup
Expand Down
12 changes: 8 additions & 4 deletions internal/common/oidc.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,10 +186,6 @@ func (o *OidcContainerBuilder) ContainerEnv() []corev1.EnvVar {
Name: "OAUTH2_PROXY_HTTP_ADDRESS",
Value: "0.0.0.0:4180",
},
{
Name: "OAUTH2_PROXY_REDIRECT_URL",
Value: "http://localhost:4180/oauth2/callback",
},
{
Name: "OAUTH2_PROXY_CODE_CHALLENGE_METHOD",
Value: "S256",
Expand All @@ -198,6 +194,14 @@ func (o *OidcContainerBuilder) ContainerEnv() []corev1.EnvVar {
Name: "OAUTH2_PROXY_EMAIL_DOMAINS",
Value: "*",
},
{
Name: "OAUTH2_PROXY_COOKIE_SECURE", // https://github.com/oauth2-proxy/oauth2-proxy/blob/c64ec1251b8366b48c6c445bbeb307b18fcb314f/oauthproxy.go#L1091
Value: "false",
},
{
Name: "OAUTH2_PROXY_WHITELIST_DOMAINS",
Value: "*",
},
}

}
Expand Down
35 changes: 33 additions & 2 deletions internal/controller/discovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,13 @@ func (d *Discovery) getListenerAddress(
cache *map[string]*listenerv1alpha1.IngressAddressSpec,
ctx context.Context,
podName string) (*listenerv1alpha1.IngressAddressSpec, error) {
cacheKey := podName

listenerName, err := d.getListenerNameByPodName(ctx, podName)
if err != nil {
discoveryLog.Info("failed to get listener name by pod name", "podName", podName, "namespace", d.Instance.Namespace)
return nil, err
}
cacheKey := listenerName
cacheObj := *cache
if address, ok := cacheObj[cacheKey]; ok {
return address, nil
Expand All @@ -244,7 +250,7 @@ func (d *Discovery) getListenerAddress(
},
}
resourceClient := common.NewResourceClient(ctx, d.Client, d.Instance.Namespace)
err := resourceClient.Get(listener)
err = resourceClient.Get(listener)
if err != nil {
discoveryLog.Info("failed to get listener", "cacheKey", cacheKey)
return nil, ErrListenerNotFound
Expand All @@ -260,3 +266,28 @@ func (d *Discovery) getListenerAddress(
cacheObj[cacheKey] = address
return address, nil
}

// get listener name from pod's lable
// label pattern: "listeners.kubedoop.dev/mnt.{listener_uid}: {listener_name}"
// the pod can be fetched by pod name,namespaces
func (d *Discovery) getListenerNameByPodName(ctx context.Context, podName string) (string, error) {
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: d.Instance.Namespace,
},
}
resourceClient := common.NewResourceClient(ctx, d.Client, d.Instance.Namespace)
err := resourceClient.Get(pod)
if err != nil {
discoveryLog.Info("failed to get pod", "podName", podName)
return "", err
}

for key, value := range pod.Labels {
if strings.HasPrefix(key, "listeners.kubedoop.dev/mnt") {
return value, nil
}
}
return "", errors.New("not found listener name by pod name")
}
40 changes: 0 additions & 40 deletions test/e2e/default/chainsaw-test.yaml

This file was deleted.

92 changes: 92 additions & 0 deletions test/e2e/kerberos/chainsaw-test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: kerberos
spec:
bindings:
- name: relam
value: KUBEDOOP.DEV # kerberos relam, should be uppercase, see hdfs also
- name: kadminPassword
value: kubedoopdev
- name: kadminKeytabSecret
value: kadmin-keytab
steps:
- name: install krb5 kdc
try:
- apply: # create a krb5 deployment and service, both named "krb5"
file: krb5-install.yaml
- assert:
file: krb5-assert.yaml
- script:
env:
- name: RELAM
value: ($relam)
- name: NAMESPACE
value: ($namespace)
- name: DEPLOYMENT_NAME
value: krb5
- name: KADMIN_KEYTAB_SECRET
value: ($kadminKeytabSecret)
content: |
# debug node info
kubectl get node
# get pod of deploment by label
POD_NAME=$(kubectl get pod -n $NAMESPACE -l app=$DEPLOYMENT_NAME -o jsonpath="{.items[0].metadata.name}")
# check pod exist
if [ -z "$POD_NAME" ]; then
echo "pod not found"
exit 1
fi
# get kadmin keytab
KADMIN_KEYTAB=$(kubectl exec -n $NAMESPACE $POD_NAME -- sh -c "cat /var/kerberos/krb5kdc/kadmin.keytab | base64 -w 0")
# create secret, name is $KADMIN_KEYTAB_SECRET
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Secret
metadata:
name: $KADMIN_KEYTAB_SECRET
namespace: $NAMESPACE
data:
keytab: $KADMIN_KEYTAB
- assert: # assert kadmin keytab secret
resource:
apiVersion: v1
kind: Secret
metadata:
name: ($kadminKeytabSecret)
data:
(keytab != ""): true
- apply:
file: krb5-secretclass.yaml
- name: install zookeeper
try:
- apply:
file: ../setup/zookeeper.yaml
- assert:
file: ../setup/zookeeper-assert.yaml
cleanup:
- sleep:
duration: 30s
- name: install hdfs
try:
- apply:
file: krb5-secretclass.yaml
- apply:
file: hdfs.yaml
- assert:
file: hdfs-assert.yaml
- name: access hdfs
try:
- apply:
file: krb5-hdfs-access-script.yaml
- apply:
file: ../setup/access-hdfs-install.yaml
- assert:
file: ../setup/access-hdfs-assert.yaml
catch:
- podLogs:
name: access-hdfs
container: access-hdfs
49 changes: 49 additions & 0 deletions test/e2e/kerberos/hdfs-access-assert.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
---
# hdfs access assert script configmap
apiVersion: v1
kind: ConfigMap
metadata:
name: krb5-hdfs-access
data:
hdfs-access-assert.sh: |
#!/bin/bash
set -ex

echo "Running HDFS access test"

echo "list credential in krb5 keytab"
klist -k /kubedoop/kerberos/keytab

# Extract unique principals from keytab
principals=$(klist -k /kubedoop/kerberos/keytab | grep -v "Keytab name:" | awk '{print $2}' | sort -u)

for principal in $principals; do
echo "Testing with principal: $principal"

echo "Authenticating with keytab"
kdestroy
kinit -kt /kubedoop/kerberos/keytab "$principal"

# Test HDFS operations
TEST_DIR="/tmp/test-$(date +%s)"
TEST_FILE="$TEST_DIR/test.txt"

echo "Creating test directory"
bin/hdfs dfs -mkdir -p "$TEST_DIR"

echo "Writing test data"
echo "Hello HDFS" | bin/hdfs dfs -put - "$TEST_FILE"

echo "Reading test data"
bin/hdfs dfs -cat "$TEST_FILE"

echo "Listing directory"
bin/hdfs dfs -ls "$TEST_DIR"

echo "Cleaning up"
bin/hdfs dfs -rm -r "$TEST_DIR"

echo "Test completed for $principal"
done

echo "All HDFS access tests completed successfully"
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,11 @@ kind: StatefulSet
metadata:
name: hdfscluster-sample-default-datanode
status:
availableReplicas: 2
replicas: 2
availableReplicas: 1
replicas: 1
---
# hdfs configmap discovery
apiVersion: v1
kind: ConfigMap
metadata:
name: hdfscluster-sample
16 changes: 9 additions & 7 deletions test/e2e/default/hdfs.yaml → test/e2e/kerberos/hdfs.yaml
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
apiVersion: hdfs.kubedoop.dev/v1alpha1
kind: HdfsCluster
metadata:
labels:
app.kubernetes.io/name: hdfscluster
app.kubernetes.io/instance: hdfscluster-sample
app.kubernetes.io/part-of: hdfs-operator
app.kubernetes.io/managed-by: kustomize
app.kubernetes.io/created-by: hdfs-operator
name: hdfscluster-sample
spec:
image:
productVersion: ($values.product_version)
clusterConfig:
zookeeperConfigMapName: sample-hdfs-znode
authentication:
kerberos:
secretClass: kerberos
tls:
jksPassword: changeit
secretClass: tls
nameNode:
roleGroups:
default:
Expand Down Expand Up @@ -52,7 +54,7 @@ spec:
dataNode:
roleGroups:
default:
replicas: 2
replicas: 1
config:
logging:
datanode:
Expand Down
14 changes: 14 additions & 0 deletions test/e2e/kerberos/krb5-assert.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: krb5
status:
availableReplicas: 1
readyReplicas: 1
replicas: 1
---
apiVersion: v1
kind: Service
metadata:
name: krb5
Loading

0 comments on commit 95b5474

Please sign in to comment.