Updating user doc (#234)

* adding k8s deployment * updating doc for local/remote * adding terraform example * adding kubehound-ingestor compose file for KHaaS deployment * Update deployments/kubehound/README.md Co-authored-by: Edouard Schweisguth <edouard.schweisguth@datadoghq.com> * Update docs/user-guide/common-operations.md Co-authored-by: Edouard Schweisguth <edouard.schweisguth@datadoghq.com> * Update docs/user-guide/common-operations.md Co-authored-by: Edouard Schweisguth <edouard.schweisguth@datadoghq.com> * Update docs/user-guide/common-operations.md Co-authored-by: Edouard Schweisguth <edouard.schweisguth@datadoghq.com> * Update deployments/k8s/khaas/README.md Co-authored-by: Edouard Schweisguth <edouard.schweisguth@datadoghq.com> * Update docs/user-guide/common-operations.md Co-authored-by: Simon Maréchal <66471981+Minosity-VR@users.noreply.github.com> * Update deployments/kubehound/README.md Co-authored-by: Simon Maréchal <66471981+Minosity-VR@users.noreply.github.com> * PR comment * PR comment for wording * renamed image var * PR comment rewording * changing resources requirement --------- Co-authored-by: Edouard Schweisguth <edouard.schweisguth@datadoghq.com> Co-authored-by: Simon Maréchal <66471981+Minosity-VR@users.noreply.github.com>
DataDog · Jul 31, 2024 · 468926c · 468926c
1 parent 8ae59ac
commit 468926c
Show file tree

Hide file tree

Showing 28 changed files with 777 additions and 6 deletions.
diff --git a/deployments/k8s/khaas/Chart.yaml b/deployments/k8s/khaas/Chart.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+description: KubeHound as a Service Deployment
+name: khaas
+version: 0.0.1
diff --git a/deployments/k8s/khaas/README.md b/deployments/k8s/khaas/README.md
@@ -0,0 +1,9 @@
+# KubeHound as a Service - ingestor - k8s deployment
+
+To deploy KHaaS ingestor services in a Kubernetes environment please refer to [k8s-deployment](https://kubehound.io/user-guide/khaas-101/#k8s-deployment)
+
+All the Helm charts and templates are provided as example. You should tweak them to your own environment (resources limitation, endpoint configuration, ...). This will depend of the number/size of the clusters you want to ingest.
+
+* [Jupyter resources estimation](https://tljh.jupyter.org/en/latest/howto/admin/resource-estimation.html)
+* [MongoDB hardware considerations](https://www.mongodb.com/docs/manual/administration/production-notes/#hardware-considerations)
+* [Janusgraph InMemory Storage Backend](https://docs.janusgraph.org/storage-backend/inmemorybackend/)
diff --git a/deployments/k8s/khaas/conf/graph/readiness.sh b/deployments/k8s/khaas/conf/graph/readiness.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -u
+
+API_ENDPOINT="127.0.0.1:{{ $.Values.services.graph.port }}?gremlin="
+
+check_query() {
+    [[ $# -gt 0 ]] || return 1
+    status_code=$(curl -s -o /dev/null -w "%{http_code}" -XGET "$API_ENDPOINT$1")
+    [[ $status_code -ge 200 && $status_code -lt 400 ]] || return 1
+}
+
+# Request from https://github.com/JanusGraph/janusgraph/issues/2807
+check_query "graph.open" || exit 1
diff --git a/deployments/k8s/khaas/conf/graph/remote.yaml b/deployments/k8s/khaas/conf/graph/remote.yaml
@@ -0,0 +1,23 @@
+# Copyright 2019 JanusGraph Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this file can be used when using gremlin console via:
+#   :remote connect tinkerpop.server /etc/janusgraph/remote.yaml session
+hosts: [{{ $.Values.services.graph.host }}]
+port: 8182
+serializer: { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}
+connectionPool: {
+  # set to 8MB
+  maxContentLength: 8388608
+}
diff --git a/deployments/k8s/khaas/conf/ingestor/kubehound.yaml b/deployments/k8s/khaas/conf/ingestor/kubehound.yaml
@@ -0,0 +1,80 @@
+#
+# Default Datadog KubeHound configuration
+# NOTE: this is optimized for large clusters in the Datadog environment
+#
+
+# K8s collector configuration
+collector:
+  # Type of collector to use
+  type: live-k8s-api-collector
+
+  # Live collector configuration
+  live:
+    # Rate limit of requests/second to the Kubernetes API
+    rate_limit_per_second: 150
+
+    # Number of pages to buffer
+    page_buffer_size: 50
+
+# General storage configuration
+storage:
+  # Whether or not to wipe all data on startup 
+  wipe: false
+
+  # Number of connection retries before declaring an error
+  retry: 5
+
+  # Delay between connection retries
+  retry_delay: 10s
+
+# Store database configuration
+mongodb:
+  # Connection URL to the mongo DB instance
+  url: "mongodb://{{ $.Values.services.db.host }}:{{ $.Values.services.db.port }}"
+
+  # Timeout on requests to the mongo DB instance
+  connection_timeout: 30s
+
+# Graph database configuration
+janusgraph:
+  # Connection URL to the JanusGraph DB instance
+  url: "ws://{{ $.Values.services.graph.host }}:{{ $.Values.services.graph.port }}/gremlin"
+
+  # Timeout on requests to the JanusGraph DB instance
+  connection_timeout: 30s
+
+# Datadog telemetry configuration
+telemetry:
+  # Whether to enable Datadog telemetry (default false)
+  enabled: false
+
+# Graph builder configuration
+builder:
+  # Vertex builder configuration
+  vertex:
+    # Batch size for vertex inserts
+    batch_size: 500
+
+  # Edge builder configuration
+  edge:
+    worker_pool_size: 5
+
+    # Batch size for edge inserts
+    batch_size: 1000
+
+    # Cluster impact batch size for edge inserts
+    batch_size_cluster_impact: 10
+
+    # Enable for large clusters to prevent number of edges growing exponentially
+    large_cluster_optimizations: true
+
+ingestor:
+  blob:
+    bucket: "{{ $.Values.services.ingestor.bucket }}"
+    region: "{{ $.Values.services.ingestor.region }}"
+  temp_dir: "/tmp/kubehound"
+  archive_name: "archive.tar.gz"
+  max_archive_size: 2073741824 # 2GB
+  api: # GRPC endpoint for the ingestor
+    endpoint: "0.0.0.0:{{ $.Values.services.ingestor.port }}"
+    insecure: true
diff --git a/deployments/k8s/khaas/templates/configmap.yaml b/deployments/k8s/khaas/templates/configmap.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $.Chart.Name }}-ingestor
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: {{ $.Chart.Name | quote }}
+    chart_name: {{ $.Chart.Name | quote }}
+    chart_version: {{ $.Chart.Version }}
+    service: {{ $.Chart.Name | quote }}
+    team: {{ $.Values.team }}
+data:
+{{ tpl (.Files.Glob "conf/ingestor/*").AsConfig . | indent 2 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $.Chart.Name }}-graph
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: {{ $.Chart.Name | quote }}
+    chart_name: {{ $.Chart.Name | quote }} 
+    chart_version: {{ $.Chart.Version }}
+    service: {{ $.Chart.Name | quote }}
+    team: {{ $.Values.team }}
+data:
+{{ tpl (.Files.Glob "conf/graph/*").AsConfig . | indent 2 -}}
diff --git a/deployments/k8s/khaas/templates/deployment-db.yaml b/deployments/k8s/khaas/templates/deployment-db.yaml
@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ $.Chart.Name }}-db"
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: "{{ $.Chart.Name }}-db"
+    service: {{ $.Chart.Name }}
+    chart_version: {{ $.Chart.Version }}
+    chart_name: {{ $.Chart.Name }}
+    team: {{ $.Values.team }}
+
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: "{{ $.Chart.Name }}-db"
+  template:
+    metadata:
+      labels:
+        app: "{{ $.Chart.Name }}-db"
+        service: {{ $.Chart.Name }}
+        team: {{ $.Values.team }}
+        chart_name: {{ $.Chart.Name }}
+    spec:
+      containers:
+      - name: {{ $.Chart.Name }}-db
+        image: "{{ $.Values.services.db.image }}:{{ $.Values.services.db.version }}"
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: {{ $.Values.services.db.resources.requests.cpu }}
+            memory: {{ $.Values.services.db.resources.requests.memory }}
+          limits:
+            cpu: {{ $.Values.services.db.resources.limits.cpu }}
+            memory: {{ $.Values.services.db.resources.limits.memory }}
+        ports:
+          - name: db
+            containerPort: {{ $.Values.services.db.port }}
diff --git a/deployments/k8s/khaas/templates/deployment-graph.yaml b/deployments/k8s/khaas/templates/deployment-graph.yaml
@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ $.Chart.Name }}-graph"
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: "{{ $.Chart.Name }}-graph"
+    service: {{ $.Chart.Name }}
+    chart_version: {{ $.Chart.Version }}
+    chart_name: {{ $.Chart.Name }}
+    team: {{ $.Values.team }}
+
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: "{{ $.Chart.Name }}-graph"
+  template:
+    metadata:
+      labels:
+        app: "{{ $.Chart.Name }}-graph"
+        service: {{ $.Chart.Name }}
+        team: {{ $.Values.team }}
+        chart_name: {{ $.Chart.Name }}
+    spec:
+      containers:
+      - name: {{ $.Chart.Name }}-graph
+        image: "{{ $.Values.services.graph.image }}:{{ $.Values.services.graph.version}}"
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: {{ $.Values.services.graph.resources.requests.cpu }}
+            memory: {{ $.Values.services.graph.resources.requests.memory }}
+          limits:
+            cpu: {{ $.Values.services.graph.resources.limits.cpu }}
+            memory: {{ $.Values.services.graph.resources.limits.memory }}
+        livenessProbe:
+          tcpSocket:
+            port: graph
+          initialDelaySeconds: 60
+          periodSeconds: 5
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command: ["/bin/bash", "/etc/janusgraph/readiness.sh"]
+          initialDelaySeconds: 60
+          periodSeconds: 5
+          timeoutSeconds: 5
+        ports:
+          - name: graph
+            containerPort: {{ $.Values.services.graph.port }}
+        volumeMounts:
+        - name: conf
+          mountPath: /etc/janusgraph
+      volumes:
+        - name: conf
+          configMap:
+            name: {{ $.Chart.Name }}-graph
diff --git a/deployments/k8s/khaas/templates/deployment-ingestor.yaml b/deployments/k8s/khaas/templates/deployment-ingestor.yaml
@@ -0,0 +1,51 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ $.Chart.Name }}-ingestor"
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: "{{ $.Chart.Name }}-ingestor"
+    service: {{ $.Chart.Name }}
+    chart_version: {{ $.Chart.Version }}
+    chart_name: {{ $.Chart.Name }}
+    team: {{ $.Values.team }}
+
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: "{{ $.Chart.Name }}-ingestor"
+  template:
+    metadata:
+      labels:
+        app: "{{ $.Chart.Name }}-ingestor"
+        service: {{ $.Chart.Name }}
+        team: {{ $.Values.team }}
+        chart_name: {{ $.Chart.Name }}
+    spec:
+      serviceAccountName: "{{ $.Chart.Name }}-ingestor"
+      containers:
+      - name: {{ $.Chart.Name }}-ingestor
+        image: "{{ $.Values.services.ingestor.image }}:{{ $.Values.services.ingestor.version }}"
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: {{ $.Values.services.ingestor.resources.requests.cpu }}
+            memory: {{ $.Values.services.ingestor.resources.requests.memory }}
+          limits:
+            cpu: {{ $.Values.services.ingestor.resources.limits.cpu }}
+            memory: {{ $.Values.services.ingestor.resources.limits.memory }}
+        volumeMounts:
+          - name: config
+            mountPath: /etc/kubehound
+        command: ["/kubehound","serve","-c", "/etc/kubehound/kubehound.yaml"]
+        env:
+          - name: KH_LOG_FORMAT
+            value: json
+        ports:
+          - name: ingestor
+            containerPort: {{ $.Values.services.ingestor.port }}
+      volumes:
+        - name: config
+          configMap:
+            name: {{ $.Chart.Name }}-ingestor
diff --git a/deployments/k8s/khaas/templates/deployment-ui.yaml b/deployments/k8s/khaas/templates/deployment-ui.yaml
@@ -0,0 +1,41 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ $.Chart.Name }}-ui"
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: "{{ $.Chart.Name }}-ui"
+    service: {{ $.Chart.Name }}
+    chart_version: {{ $.Chart.Version }}
+    chart_name: {{ $.Chart.Name }}
+    team: {{ $.Values.team }}
+
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: "{{ $.Chart.Name }}-ui"
+  template:
+    metadata:
+      labels:
+        app: "{{ $.Chart.Name }}-ui"
+        service: {{ $.Chart.Name }}
+        team: {{ $.Values.team }}
+        chart_name: {{ $.Chart.Name }}
+    spec:
+      containers:
+      - name: {{ $.Chart.Name }}-ui
+        image: "{{ $.Values.services.ui.image }}:{{ $.Values.services.ui.version }}"
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: {{ $.Values.services.ui.resources.requests.cpu }}
+            memory: {{ $.Values.services.ui.resources.requests.memory }}
+          limits:
+            cpu: {{ $.Values.services.ui.resources.limits.cpu }}
+            memory: {{ $.Values.services.ui.resources.limits.memory }}
+        ports:
+          - name: ui-tree
+            containerPort: {{ $.Values.services.ui.ports.tree }}
+          - name: ui-lab
+            containerPort: {{ $.Values.services.ui.ports.lab }}