diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..fd652f92b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +*.env +*.kube-config \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..3445b8947 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,68 @@ +name: docker + +on: + push: + tags: + - "*" + +env: + REGISTRY: ghcr.io + IMAGE_NAME: datadog/kubehound + +permissions: + contents: read + +jobs: + docker-build-push: + runs-on: ubuntu-latest + strategy: + # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs + matrix: + include: + - dockerfile: ./Dockerfile + component: core + - dockerfile: deployments/kubehound/janusgraph/Dockerfile + image: janusgraph + permissions: + contents: read + packages: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@128a63446a954579617e875aaab7d2978154e969 + with: + egress-policy: block + allowed-endpoints: > + auth.docker.io:443 + dl-cdn.alpinelinux.org:443 + ghcr.io:443 + github.com:443 + pipelines.actions.githubusercontent.com:443 + pkg-containers.githubusercontent.com:443 + production.cloudflare.docker.com:443 + proxy.golang.org:443 + registry-1.docker.io:443 + storage.googleapis.com:443 + + - name: Checkout + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab + with: + fetch-depth: 0 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: . + file: ${{ matrix.dockerfile }} + push: true + build-args: | + VERSION=${{ github.ref_name }} + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image }}:${{ github.ref_name }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image }}:latest \ No newline at end of file diff --git a/.github/workflows/system-test.yml b/.github/workflows/system-test.yml index d3e1fc936..2a33b53dc 100644 --- a/.github/workflows/system-test.yml +++ b/.github/workflows/system-test.yml @@ -17,13 +17,10 @@ jobs: config: test/setup/test-cluster/cluster.yaml wait: 5m env: - KUBECONFIG: ./test/setup/.kube/config + KUBECONFIG: ./test/setup/.kube-config - name: Create K8s resources - working-directory: test/setup/ - run: bash create-cluster-resources.sh - env: - KUBECONFIG: .kube/config + run: make local-cluster-config-deploy - name: Setup Golang uses: actions/setup-go@v4 @@ -31,6 +28,4 @@ jobs: go-version: "1.20" - name: Run integration Tests - run: make system-test - env: - KUBECONFIG: .kube/config \ No newline at end of file + run: make system-test \ No newline at end of file diff --git a/.gitignore b/.gitignore index 48a9f2a6f..0ad85da74 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ test/setup/.kube # binary for the autogen of fixtures test/system/generator/generator *.env +*.kube-config \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..aa17ff695 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM golang:1.20-alpine AS build + +RUN apk update && \ + apk add make && \ + rm -rf /var/cache/apt/* && \ + go install github.com/vektra/mockery/v2@v2.30.1 + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY Makefile ./ +COPY cmd ./cmd/ +COPY pkg ./pkg/ + +RUN make build BUILD_VERSION=${VERSION} + +FROM scratch +LABEL org.opencontainers.image.source="https://github.com/DataDog/kubehound/" + +WORKDIR / +COPY --from=build /app/bin/kubehound /kubehound +COPY deployments/kubehound/kubehound.yaml /etc/kubehound.yaml + +ENTRYPOINT [ "/kubehound" ] \ No newline at end of file diff --git a/Makefile b/Makefile index ae5ccb039..c2200046c 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,9 @@ BUILD_VERSION=dev-snapshot MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) ROOT_DIR := $(dir $(MAKEFILE_PATH)) -DOCKER_COMPOSE_FILE_PATH := -f test/system/docker-compose.yaml -f test/system/docker-compose.local.yaml -DOCKER_COMPOSE_ENV_FILE_PATH := test/system/.env - -# https://docs.github.com/en/actions/learn-github-actions/variables -ifeq (${CI},true) - DOCKER_COMPOSE_FILE_PATH := -f test/system/docker-compose.yaml -endif +DOCKER_COMPOSE_FILE_PATH := -f deployments/kubehound/docker-compose.yaml +DOCKER_COMPOSE_ENV_FILE_PATH := deployments/kubehound/.env +DEV_ENV_FILE_PATH := test/setup/.env.local # Loading docker .env file if present ifneq (,$(wildcard $(DOCKER_COMPOSE_ENV_FILE_PATH))) @@ -17,16 +13,37 @@ ifneq (,$(wildcard $(DOCKER_COMPOSE_ENV_FILE_PATH))) export endif -# No API key is being set -ifeq (${DD_API_KEY},) - DOCKER_COMPOSE_FILE_PATH := -f test/system/docker-compose.yaml +# Loading docker .env file if present +ifneq (,$(wildcard $(DEV_ENV_FILE_PATH))) + include $(DEV_ENV_FILE_PATH) + export endif +ifeq (${KUBEHOUND_ENV}, prod) + DOCKER_COMPOSE_FILE_PATH += -f deployments/kubehound/docker-compose.prod.yaml +else ifeq (${KUBEHOUND_ENV}, dev) + DOCKER_COMPOSE_FILE_PATH += -f deployments/kubehound/docker-compose.dev.yaml +endif + + +# No API key is being set +# ifeq (${DD_API_KEY},) +ifneq (${DD_API_KEY},) + DOCKER_COMPOSE_FILE_PATH += -f deployments/kubehound/docker-compose.datadog.yaml +endif -DOCKER_CMD = docker UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Linux) - DOCKER_CMD = sudo docker +ifndef DOCKER_CMD + ifeq ($(UNAME_S),Linux) + # https://docs.github.com/en/actions/learn-github-actions/variables + ifneq (${CI},true) + DOCKER_CMD := sudo docker + endif + else + DOCKER_CMD := docker + endif +else + DOCKER_CMD := ${DOCKER_CMD} endif all: build @@ -57,9 +74,7 @@ test: ## Run the full suite of unit tests system-test: ## Run the system tests $(MAKE) infra-rm $(MAKE) infra-up - # we print the KUBECONFIG envvar here to make it easier to see what is actively used - sleep 10 - cd test/system && export KUBECONFIG=$(ROOT_DIR)/test/setup/.kube/config && bash -c "printenv KUBECONFIG" && go test -v -timeout "60s" -count=1 ./... + cd test/system && export KUBECONFIG=$(ROOT_DIR)/test/setup/${KIND_KUBECONFIG} && go test -v -timeout "60s" -count=1 ./... .PHONY: local-cluster-reset local-cluster-reset: ## Destroy the current kind cluster and creates a new one diff --git a/deployments/kubehound/.env.tpl b/deployments/kubehound/.env.tpl new file mode 100644 index 000000000..bab2483d2 --- /dev/null +++ b/deployments/kubehound/.env.tpl @@ -0,0 +1,4 @@ +DD_API_KEY= +DD_SITE=api.datadoghq.com +KUBEHOUND_ENV=prod +BUILDKIT_PROGRESS=auto # (default) auto, plain, tty \ No newline at end of file diff --git a/deployments/kubehound/docker-compose.datadog.yaml b/deployments/kubehound/docker-compose.datadog.yaml new file mode 100644 index 000000000..df4acead4 --- /dev/null +++ b/deployments/kubehound/docker-compose.datadog.yaml @@ -0,0 +1,17 @@ +version: "3.8" + +services: + datadog: + image: gcr.io/datadoghq/agent:7 + restart: unless-stopped + container_name: datadog-agent + ports: + - "127.0.0.1:8125:8125" + environment: + - DD_API_KEY="${DD_API_KEY:?error}" + - DD_SITE=datadoghq.com + networks: + - kubenet + +networks: + kubenet: \ No newline at end of file diff --git a/deployments/kubehound/docker-compose.dev.yaml b/deployments/kubehound/docker-compose.dev.yaml new file mode 100644 index 000000000..282a17991 --- /dev/null +++ b/deployments/kubehound/docker-compose.dev.yaml @@ -0,0 +1,42 @@ +version: "3.8" +name: kubehound-dev +services: + mongodb: + volumes: + - mongodb_data:/data/db + + janusgraph: + volumes: + - janusgraph_data:/data + + mongo-express: + image: mongo-express:0.54.0 + restart: unless-stopped + depends_on: + - mongodb + ports: + - "127.0.0.1:8081:8081" + environment: + ME_CONFIG_MONGODB_URL: mongodb://mongodb:27017/ + + kubehound: + container_name: ${COMPOSE_PROJECT_NAME}-core + restart: unless-stopped + build: + context: ../../ + dockerfile: Dockerfile + environment: + - KUBECONFIG=/tmp/.kube/config + volumes: + - ./.kube-config:/tmp/.kube/config + networks: + - kubenet + - kind + +volumes: + mongodb_data: + janusgraph_data: + +networks: + kind: + external: true \ No newline at end of file diff --git a/deployments/kubehound/docker-compose.prod.yaml b/deployments/kubehound/docker-compose.prod.yaml new file mode 100644 index 000000000..40b6496a3 --- /dev/null +++ b/deployments/kubehound/docker-compose.prod.yaml @@ -0,0 +1,26 @@ +version: "3.8" +name: kubehound-prod +services: + mongodb: + volumes: + - mongodb_data:/data/db + + janusgraph: + image: ghcr.io/datadog/kubehound/kubehound-janusgraph:latest + volumes: + - janusgraph_data:/data + + kubehound: + container_name: ${COMPOSE_PROJECT_NAME}-core + restart: unless-stopped + image: ghcr.io/datadog/kubehound/kubehound-core:latest + environment: + - KUBECONFIG=/tmp/.kube/config + volumes: + - ./.kube-config:/tmp/.kube/config + networks: + - kubenet + +volumes: + mongodb_data: + janusgraph_data: \ No newline at end of file diff --git a/deployments/kubehound/docker-compose.yaml b/deployments/kubehound/docker-compose.yaml new file mode 100644 index 000000000..4bc97b151 --- /dev/null +++ b/deployments/kubehound/docker-compose.yaml @@ -0,0 +1,36 @@ +version: "3.7" +name: kubehound-testing +services: + mongodb: + image: mongo:6.0.6 + restart: unless-stopped + container_name: ${COMPOSE_PROJECT_NAME}-storedb + ports: + - "127.0.0.1:27017:27017" + networks: + - kubenet + depends_on: + janusgraph: + condition: service_healthy + + janusgraph: + build: ./janusgraph/ + restart: unless-stopped + container_name: ${COMPOSE_PROJECT_NAME}-graphdb + ports: + - "127.0.0.1:8182:8182" + networks: + - kubenet + environment: + # Enforce strict schema constrains as per https://docs.janusgraph.org/configs/configuration-reference/#schema + - janusgraph.schema.constraints=true + - janusgraph.schema.default=none + healthcheck: + test: ["CMD", "/opt/janusgraph/bin/gremlin.sh", "-e", "/opt/janusgraph/scripts/health-check.groovy"] + interval: 60s + timeout: 30s + retries: 1 + start_period: 15s + +networks: + kubenet: \ No newline at end of file diff --git a/deployments/kubehound/janusgraph/Dockerfile b/deployments/kubehound/janusgraph/Dockerfile index 5517f4657..b0ad8e320 100644 --- a/deployments/kubehound/janusgraph/Dockerfile +++ b/deployments/kubehound/janusgraph/Dockerfile @@ -1,4 +1,5 @@ -FROM janusgraph/janusgraph:latest +FROM janusgraph/janusgraph:0.6.3 +LABEL org.opencontainers.image.source="https://github.com/DataDog/kubehound/" # Add our initialization script for the database schema to the startup directory # See https://github.com/JanusGraph/janusgraph-docker#initialization diff --git a/deployments/kubehound/kubehound.yaml b/deployments/kubehound/kubehound.yaml new file mode 100644 index 000000000..486e44934 --- /dev/null +++ b/deployments/kubehound/kubehound.yaml @@ -0,0 +1,18 @@ +storage: + retry_delay: 10s + retry: 5 +collector: + type: live-k8s-api-collector + live: + page_size: 500 + page_buffer_size: 10 + rate_limit_per_second: 100 +mongodb: + url: "mongodb://mongodb:27017" + connection_timeout: 5s +janusgraph: + url: "ws://janusgraph:8182/gremlin" + connection_timeout: 5s +telemetry: + statsd: + url: "datadog:8125" \ No newline at end of file diff --git a/deployments/kubehound/start.sh b/deployments/kubehound/start.sh deleted file mode 100755 index 4405ed30c..000000000 --- a/deployments/kubehound/start.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e - -# Spin up the compose with neo4j, mongodb, etc -docker-compose -f ./docker-compose-dev.yaml up --force-recreate --build \ No newline at end of file diff --git a/deployments/kubehound/wipe-data.sh b/deployments/kubehound/wipe-data.sh deleted file mode 100755 index 964ce4851..000000000 --- a/deployments/kubehound/wipe-data.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -rm -r ./data/mongodb/ -rm -r ./data/janus/ - diff --git a/test/setup/.env.local b/test/setup/.env.local index ef3f60cfb..a450f00cc 100644 --- a/test/setup/.env.local +++ b/test/setup/.env.local @@ -1,3 +1,5 @@ CLUSTER_NAME=kubehound.test.local CONFIG_DIR=./test-cluster -KUBECONFIG=./test/setup/.kube/config \ No newline at end of file +KIND_KUBECONFIG=./.kube-config +KIND_KUBECONFIG_INTERNAL=./../../deployments/kubehound/.kube-config +DOCKER_CMD=docker \ No newline at end of file diff --git a/test/setup/create-cluster-resources.sh b/test/setup/create-cluster-resources.sh deleted file mode 100755 index 969364ac3..000000000 --- a/test/setup/create-cluster-resources.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e - -CLUSTER_NAME=kubehound.test.local -CONFIG_DIR=./test-cluster -export KUBECONFIG=.kube/config - -echo "[*] Deploying test resources via kubectl apply" -for attack in ${CONFIG_DIR}/attacks/*.yaml; do - [ -e "$attack" ] || continue - - kubectl apply -f "$attack" --context "kind-${CLUSTER_NAME}" -done - -echo "[*] Deployments complete" \ No newline at end of file diff --git a/test/setup/destroy-cluster.sh b/test/setup/destroy-cluster.sh deleted file mode 100755 index 04397692c..000000000 --- a/test/setup/destroy-cluster.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -CLUSTER_NAME=kubehound.test.local -export KUBECONFIG=.kube/config - -echo "[*] Destroying test cluster "${CLUSTER_NAME}" via kind" -kind delete cluster --name "${CLUSTER_NAME}" \ No newline at end of file diff --git a/test/setup/manage-cluster-resources.sh b/test/setup/manage-cluster-resources.sh index fe5240cec..d25e78718 100755 --- a/test/setup/manage-cluster-resources.sh +++ b/test/setup/manage-cluster-resources.sh @@ -13,7 +13,7 @@ function handle_resources(){ _printf_warn "$2 test resources via kubectl apply" for attack in ${SCRIPT_DIR}/${CONFIG_DIR}/attacks/*.yaml; do [ -e "$attack" ] || continue - echo "$attack" + _printf_ok "$attack" # since deletion can take some times, || true to be able to retry in case of C-C kubectl $1 -f "$attack" --context "kind-${CLUSTER_NAME}" || true done diff --git a/test/setup/manage-cluster.sh b/test/setup/manage-cluster.sh index 6be1046cd..bd08d0f1e 100755 --- a/test/setup/manage-cluster.sh +++ b/test/setup/manage-cluster.sh @@ -10,20 +10,33 @@ source $SCRIPT_DIR/util.sh PROJECT_MAN="options: [create | destroy]" function create_cluster(){ - echo "[*] Creating test cluster "${CLUSTER_NAME}" via kind" - $KIND create cluster \ + _printf_ok "Creating test cluster "${CLUSTER_NAME}" via kind" + $KIND_CMD create cluster \ --name "${CLUSTER_NAME}" \ - --config "${CONFIG_DIR}/cluster.yaml" \ + --config "${SCRIPT_DIR}/${CONFIG_DIR}/cluster.yaml" \ - echo "Using KUBECONFIG: $(printenv KUBECONFIG)" + _printf_warn "Using KUBECONFIG: $(printenv KUBECONFIG)" kubectl cluster-info --context "kind-${CLUSTER_NAME}" + dump_config_file echo "[*] Cluster ${CLUSTER_NAME} configuration complete" } function destroy_cluster(){ - echo "[*] Destroying test cluster "${CLUSTER_NAME}" via kind" - $KIND delete cluster --name "${CLUSTER_NAME}" + _printf_ok "Destroying test cluster "${CLUSTER_NAME}" via kind" + $KIND_CMD delete cluster --name "${CLUSTER_NAME}" +} + +function remove_config_files(){ + _printf_ok "Removing config files for kind cluster "${CLUSTER_NAME}"" + rm -f ${SCRIPT_DIR}/${KIND_KUBECONFIG} + rm -f ${SCRIPT_DIR}/${KIND_KUBECONFIG_INTERNAL} +} + +function dump_config_file(){ + _printf_ok "Dump kind cluster "${CLUSTER_NAME}" via kind for Docker env" + $KIND_CMD get kubeconfig --name "${CLUSTER_NAME}" > ${SCRIPT_DIR}/${KIND_KUBECONFIG} + $KIND_CMD get kubeconfig --internal --name "${CLUSTER_NAME}" > ${SCRIPT_DIR}/${KIND_KUBECONFIG_INTERNAL} } case $SCRIPT_ACTION in @@ -32,6 +45,7 @@ create) ;; destroy) destroy_cluster + remove_config_files ;; *) echo "$PROJECT_MAN" diff --git a/test/setup/setup-cluster.sh b/test/setup/setup-cluster.sh deleted file mode 100755 index 6eddca9b7..000000000 --- a/test/setup/setup-cluster.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e - -CLUSTER_NAME=kubehound.test.local -CONFIG_DIR=./test-cluster -export KUBECONFIG=.kube/config - -echo "[*] Creating test cluster "${CLUSTER_NAME}" via kind" -kind create cluster \ - --name "${CLUSTER_NAME}" \ - --config "${CONFIG_DIR}/cluster.yaml" \ - -kubectl cluster-info --context "kind-${CLUSTER_NAME}" - -echo "[*] Cluster ${CLUSTER_NAME} configuration complete" \ No newline at end of file diff --git a/test/setup/util.sh b/test/setup/util.sh index 064953688..55ba0dc46 100644 --- a/test/setup/util.sh +++ b/test/setup/util.sh @@ -28,7 +28,7 @@ function load_env(){ _printf_warn "Loading env vars from $SCRIPT_DIR/.env.local ..." if [ -f $SCRIPT_DIR/.env.local ]; then set -a - source $SCRIPT_DIR/.env + source $SCRIPT_DIR/.env.local set +a fi } @@ -36,13 +36,12 @@ function load_env(){ load_env # post load env -KIND=kind -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - KIND="sudo kind" -fi - -KIND="$KIND --kubeconfig $KUBECONFIG" -if [ -f $KUBECONFIG ]; then - sudo chown $USER:$USER $KUBECONFIG -fi -echo "Using KUBECONFIG: $(printenv KUBECONFIG)" \ No newline at end of file +# Set configuration for linux - https://docs.github.com/en/actions/learn-github-actions/variables +if [ -z $KIND_CMD ]; then + if [[ "$OSTYPE" == "linux-gnu"* && "$CI" != "true" ]]; then + _printf_warn "sudo mode activated" + KIND_CMD="sudo kind" + else + KIND_CMD="kind" + fi +fi \ No newline at end of file