Merge pull request #77 from bostrt/crio

Adding support for CRI-O and flexibility for more.

go.mod

@@ -25,13 +25,11 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.6.2
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.6.1
 	golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect
 	golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sys v0.0.0-20200117145432-59e60aa80a0c // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
-	google.golang.org/appengine v1.6.5
 	gopkg.in/ini.v1 v1.51.1 // indirect
 	gopkg.in/yaml.v2 v2.2.7 // indirect
 	k8s.io/api v0.17.1

kube/kubernetes_api_service.go

@@ -3,6 +3,7 @@ package kube
 import (
 	"fmt"
 	"io"
+	"ksniff/pkg/service/sniffer/runtime"
 	"ksniff/utils"
 	"strings"
 	"time"
@@ -39,19 +40,21 @@ func NewKubernetesApiService(clientset *kubernetes.Clientset,
 		targetNamespace: targetNamespace}
 }
 
-func (k *KubernetesApiServiceImpl) IsDockerContainerRuntime(nodeName string) (bool, error) {
+func (k *KubernetesApiServiceImpl) IsSupportedContainerRuntime(nodeName string) (bool, error) {
 	node, err := k.clientset.CoreV1().Nodes().Get(nodeName, v1.GetOptions{})
 	if err != nil {
 		return false, err
 	}
 
 	nodeRuntimeVersion := node.Status.NodeInfo.ContainerRuntimeVersion
 
-	if strings.TrimPrefix(nodeRuntimeVersion, "docker") == nodeRuntimeVersion {
-		return false, nil
+	for _,runtime := range runtime.SupportedContainerRuntimes {
+		if strings.HasPrefix(nodeRuntimeVersion, runtime) {
+			return true, nil
+		}
 	}
 
-	return true, nil
+	return false, nil
 }
 
 func (k *KubernetesApiServiceImpl) ExecuteCommand(podName string, containerName string, command []string, stdOut io.Writer) (int, error) {
@@ -74,7 +77,7 @@ func (k *KubernetesApiServiceImpl) ExecuteCommand(podName string, containerName
 
 	exitCode, err := PodExecuteCommand(executeTcpdumpRequest)
 	if err != nil {
-		log.WithError(err).Error("failed executing command: '%s', exitCode: '%d', stdErr: '%s'",
+		log.WithError(err).Errorf("failed executing command: '%s', exitCode: '%d', stdErr: '%s'",
 			command, exitCode, stdErr.Output)
 
 		return exitCode, err
@@ -102,13 +105,13 @@ func (k *KubernetesApiServiceImpl) DeletePod(podName string) error {
 func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, image string) (*corev1.Pod, error) {
 	log.Debugf("creating privileged pod on remote node")
 
-	isDockerRuntime, err := k.IsDockerContainerRuntime(nodeName)
+	isSupported, err := k.IsSupportedContainerRuntime(nodeName)
 	if err != nil {
 		return nil, err
 	}
 
-	if !isDockerRuntime {
-		return nil, errors.Errorf("container runtime on node: '%s' isn't docker", nodeName)
+	if !isSupported {
+		return nil, errors.Errorf("Container runtime on node %s isn't supported. Supported container runtimes are: %v", nodeName, runtime.SupportedContainerRuntimes)
 	}
 
 	typeMetadata := v1.TypeMeta{
@@ -125,9 +128,9 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, image st
 	}
 
 	volumeMounts := []corev1.VolumeMount{{
-		Name:      "docker-sock",
+		Name:      "host",
 		ReadOnly:  true,
-		MountPath: "/var/run/docker.sock",
+		MountPath: "/host",
 	}}
 
 	privileged := true
@@ -143,20 +146,21 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, image st
 		VolumeMounts: volumeMounts,
 	}
 
-	hostPathType := corev1.HostPathFile
+	hostPathType := corev1.HostPathDirectory
 	volumeSources := corev1.VolumeSource{
 		HostPath: &corev1.HostPathVolumeSource{
-			Path: "/var/run/docker.sock",
+			Path: "/",
 			Type: &hostPathType,
 		},
 	}
 
 	podSpecs := corev1.PodSpec{
 		NodeName:      nodeName,
 		RestartPolicy: corev1.RestartPolicyNever,
+		HostPID: true,
 		Containers:    []corev1.Container{privilegedContainer},
 		Volumes: []corev1.Volume{{
-			Name:         "docker-sock",
+			Name:         "host",
 			VolumeSource: volumeSources,
 		},
 		},

pkg/cmd/sniff.go

@@ -6,6 +6,7 @@ import (
 	"ksniff/kube"
 	"ksniff/pkg/config"
 	"ksniff/pkg/service/sniffer"
+	"ksniff/pkg/service/sniffer/runtime"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -120,7 +121,7 @@ func NewCmdSniff(streams genericclioptions.IOStreams) *cobra.Command {
 	_ = viper.BindEnv("privileged", "KUBECTL_PLUGINS_LOCAL_FLAG_PRIVILEGED")
 	_ = viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
 
-	cmd.Flags().StringVarP(&ksniffSettings.Image, "image", "", "docker",
+	cmd.Flags().StringVarP(&ksniffSettings.Image, "image", "", "",
 		"the privileged container image (optional)")
 	_ = viper.BindEnv("image", "KUBECTL_PLUGINS_LOCAL_FLAG_IMAGE")
 	_ = viper.BindPFlag("image", cmd.Flags().Lookup("image"))
@@ -155,6 +156,7 @@ func (o *Ksniff) Complete(cmd *cobra.Command, args []string) error {
 	o.settings.UserSpecifiedVerboseMode = viper.GetBool("verbose")
 	o.settings.UserSpecifiedPrivilegedMode = viper.GetBool("privileged")
 	o.settings.UserSpecifiedKubeContext = viper.GetString("context")
+	o.settings.UseDefaultImage = !cmd.Flag("image").Changed
 
 	var err error
 
@@ -205,15 +207,6 @@ func (o *Ksniff) Complete(cmd *cobra.Command, args []string) error {
 
 	o.resultingContext = currentContext.DeepCopy()
 	o.resultingContext.Namespace = o.settings.UserSpecifiedNamespace
-	kubernetesApiService := kube.NewKubernetesApiService(o.clientset, o.restConfig, o.settings.UserSpecifiedNamespace)
-
-	if o.settings.UserSpecifiedPrivilegedMode {
-		log.Info("sniffing method: privileged pod")
-		o.snifferService = sniffer.NewPrivilegedPodRemoteSniffingService(o.settings, kubernetesApiService)
-	} else {
-		log.Info("sniffing method: upload static tcpdump")
-		o.snifferService = sniffer.NewUploadTcpdumpRemoteSniffingService(o.settings, kubernetesApiService)
-	}
 
 	return nil
 }
@@ -249,12 +242,15 @@ func (o *Ksniff) Validate() error {
 
 	var err error
 
-	o.settings.UserSpecifiedLocalTcpdumpPath, err = findLocalTcpdumpBinaryPath()
-	if err != nil {
-		return err
+	if ! o.settings.UserSpecifiedPrivilegedMode {
+		o.settings.UserSpecifiedLocalTcpdumpPath, err = findLocalTcpdumpBinaryPath()
+		if err != nil {
+			return err
+		}
+
+		log.Infof("using tcpdump path at: '%s'", o.settings.UserSpecifiedLocalTcpdumpPath)
 	}
 
-	log.Infof("using tcpdump path at: '%s'", o.settings.UserSpecifiedLocalTcpdumpPath)
 
 	pod, err := o.clientset.CoreV1().Pods(o.settings.UserSpecifiedNamespace).Get(o.settings.UserSpecifiedPodName, v1.GetOptions{})
 	if err != nil {
@@ -279,22 +275,40 @@ func (o *Ksniff) Validate() error {
 		log.Infof("selected container: '%s'", o.settings.UserSpecifiedContainer)
 	}
 
-	var containerFoundInPod = false
-	for _, containerStatus := range pod.Status.ContainerStatuses {
-		if o.settings.UserSpecifiedContainer == containerStatus.Name {
-			o.settings.DetectedContainerId = strings.TrimPrefix(containerStatus.ContainerID, "docker://")
-			containerFoundInPod = true
-			break
-		}
+	if err := o.findContainerId(pod); err != nil {
+		return err
 	}
 
-	if !containerFoundInPod {
-		return errors.Errorf("couldn't find container: '%s' in pod: '%s'", o.settings.UserSpecifiedContainer, o.settings.UserSpecifiedPodName)
+	kubernetesApiService := kube.NewKubernetesApiService(o.clientset, o.restConfig, o.settings.UserSpecifiedNamespace)
+
+	if o.settings.UserSpecifiedPrivilegedMode {
+		log.Info("sniffing method: privileged pod")
+		bridge := runtime.NewContainerRuntimeBridge(o.settings.DetectedContainerRuntime)
+		o.snifferService = sniffer.NewPrivilegedPodRemoteSniffingService(o.settings, kubernetesApiService, bridge)
+	} else {
+		log.Info("sniffing method: upload static tcpdump")
+		o.snifferService = sniffer.NewUploadTcpdumpRemoteSniffingService(o.settings, kubernetesApiService)
 	}
 
 	return nil
 }
 
+func (o *Ksniff) findContainerId(pod *corev1.Pod) error {
+	for _, containerStatus := range pod.Status.ContainerStatuses {
+		if o.settings.UserSpecifiedContainer == containerStatus.Name {
+			result := strings.Split(containerStatus.ContainerID, "://")
+			if len(result) != 2 {
+				break
+			}
+			o.settings.DetectedContainerRuntime = result[0]
+			o.settings.DetectedContainerId = result[1]
+			return nil
+		}
+	}
+
+	return errors.Errorf("couldn't find container: '%s' in pod: '%s'", o.settings.UserSpecifiedContainer, o.settings.UserSpecifiedPodName)
+}
+
 func findLocalTcpdumpBinaryPath() (string, error) {
 	log.Debugf("searching for tcpdump binary using lookup list: '%v'", tcpdumpLocalBinaryPathLookupList)
 

pkg/cmd/sniff_test.go

@@ -45,7 +45,7 @@ func TestComplete_PodNameSpecified(t *testing.T) {
 	// given
 	settings := config.NewKsniffSettings(genericclioptions.IOStreams{})
 	sniff := NewKsniff(settings)
-	cmd := &cobra.Command{}
+	cmd := NewCmdSniff(genericclioptions.IOStreams{})
 	var commands []string
 
 	// when

pkg/config/settings.go

@@ -15,9 +15,12 @@ type KsniffSettings struct {
 	UserSpecifiedRemoteTcpdumpPath string
 	UserSpecifiedVerboseMode       bool
 	UserSpecifiedPrivilegedMode    bool
+	UserSpecifiedImage             string
 	DetectedPodNodeName            string
 	DetectedContainerId            string
+	DetectedContainerRuntime       string
 	Image                          string
+	UseDefaultImage                bool
 	UserSpecifiedKubeContext       string
 }
 

pkg/service/sniffer/privileged_pod_sniffer_service.go

@@ -1,47 +1,67 @@
 package sniffer
 
 import (
-	"fmt"
+	"bytes"
+	log "github.com/sirupsen/logrus"
 	"io"
+	v1 "k8s.io/api/core/v1"
 	"ksniff/kube"
 	"ksniff/pkg/config"
-	"ksniff/utils"
-
-	log "github.com/sirupsen/logrus"
-	v1 "k8s.io/api/core/v1"
+	"ksniff/pkg/service/sniffer/runtime"
 )
 
 type PrivilegedPodSnifferService struct {
 	settings                *config.KsniffSettings
 	privilegedPod           *v1.Pod
 	privilegedContainerName string
+	targetProcessId         *string
 	kubernetesApiService    kube.KubernetesApiService
+	runtimeBridge  runtime.ContainerRuntimeBridge
 }
 
-func NewPrivilegedPodRemoteSniffingService(options *config.KsniffSettings, service kube.KubernetesApiService) SnifferService {
-	return &PrivilegedPodSnifferService{settings: options, kubernetesApiService: service}
+func NewPrivilegedPodRemoteSniffingService(options *config.KsniffSettings, service kube.KubernetesApiService, bridge runtime.ContainerRuntimeBridge) SnifferService {
+	return &PrivilegedPodSnifferService{settings: options, kubernetesApiService: service, runtimeBridge: bridge}
 }
 
 func (p *PrivilegedPodSnifferService) Setup() error {
 	var err error
 
 	log.Infof("creating privileged pod on node: '%s'", p.settings.DetectedPodNodeName)
 
-	p.privilegedPod, err = p.kubernetesApiService.CreatePrivilegedPod(p.settings.DetectedPodNodeName, p.settings.Image)
+	image := p.settings.Image
+
+	if p.settings.UseDefaultImage {
+		image = p.runtimeBridge.GetDefaultImage()
+	}
+
+	p.privilegedPod, err = p.kubernetesApiService.CreatePrivilegedPod(p.settings.DetectedPodNodeName, image)
 	if err != nil {
 		log.WithError(err).Errorf("failed to create privileged pod on node: '%s'", p.settings.DetectedPodNodeName)
 		return err
 	}
 
 	log.Infof("pod: '%s' created successfully on node: '%s'", p.privilegedPod.Name, p.settings.DetectedPodNodeName)
 
+	if p.runtimeBridge.NeedsPid() {
+		var buff bytes.Buffer
+		command := p.runtimeBridge.BuildInspectCommand(p.settings.DetectedContainerId)
+		exitCode, err := p.kubernetesApiService.ExecuteCommand(p.privilegedPod.Name, p.privilegedPod.Spec.Containers[0].Name, command, &buff)
+		if err != nil {
+			log.WithError(err).Errorf("failed to start sniffing using privileged pod, exit code: '%d'", exitCode)
+		}
+		p.targetProcessId, err = p.runtimeBridge.ExtractPid(buff.String())
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
 func (p *PrivilegedPodSnifferService) Cleanup() error {
 	log.Infof("removing privileged container: '%s'", p.privilegedContainerName)
 
-	command := []string{"docker", "rm", "-f", p.privilegedContainerName}
+	command := p.runtimeBridge.BuildCleanupCommand()
 
 	exitCode, err := p.kubernetesApiService.ExecuteCommand(p.privilegedPod.Name, p.privilegedPod.Spec.Containers[0].Name, command, &kube.NopWriter{})
 	if err != nil {
@@ -67,12 +87,7 @@ func (p *PrivilegedPodSnifferService) Cleanup() error {
 func (p *PrivilegedPodSnifferService) Start(stdOut io.Writer) error {
 	log.Info("starting remote sniffing using privileged pod")
 
-	p.privilegedContainerName = "ksniff-container-" + utils.GenerateRandomString(8)
-	containerNameFlag := fmt.Sprintf("--name=%s", p.privilegedContainerName)
-
-	command := []string{"docker", "run", "--rm", containerNameFlag,
-		fmt.Sprintf("--net=container:%s", p.settings.DetectedContainerId), "corfr/tcpdump", "-i",
-		p.settings.UserSpecifiedInterface, "-U", "-w", "-", p.settings.UserSpecifiedFilter}
+	command := p.runtimeBridge.BuildTcpdumpCommand(&p.settings.DetectedContainerId, p.settings.UserSpecifiedInterface, p.settings.UserSpecifiedFilter, p.targetProcessId)
 
 	exitCode, err := p.kubernetesApiService.ExecuteCommand(p.privilegedPod.Name, p.privilegedPod.Spec.Containers[0].Name, command, stdOut)
 	if err != nil {