Merge pull request #1419 from snyk/feat/migrate-to-sysdig-public-api

feat: use Sysdig EVE public API [CIT-704]
snyk · Dec 21, 2023 · 9615820 · 9615820
2 parents 5b82447 + 681f462
commit 9615820
Show file tree

Hide file tree

Showing 11 changed files with 591 additions and 120 deletions.
diff --git a/snyk-monitor/README.md b/snyk-monitor/README.md
@@ -193,13 +193,19 @@ If running with Operator Lifecycle Manager (OLM) then OLM will handle upgrades f
 
 We have partnered with Sysdig to enrich the issues detected by Snyk for workloads with runtime data provided by Sysdig.
 
-In order for the integration with Sysdig to work, the Snyk monitor requires an extra Secret in the `snyk-monitor` namespace. The Secret name is `sysdig-eve-secret`.
-
-Please refer to the [Sysdig Secret installation guide](https://docs.sysdig.com/en/docs/sysdig-secure/integrate-effective-vulnerability-exposure-with-snyk/#copy-the-sysdig-secret) to install the Secret. Once the Sysdig Secret is installed, you need to copy it over to the snyk-monitor namespace:
+For a successful integration with Sysdig, the Snyk Controller requires an extra Sysdig Secret in the snyk-monitor namespace. The Sysdig Secret name is snyk-sysdig-secret.
 
+Create the snyk-sysdig-secret in the snyk-monitor namespace:
 ```bash
-kubectl get secret sysdig-eve-secret -n sysdig-agent -o yaml | grep -v '^\s*namespace:\s' | kubectl apply -n snyk-monitor  -f -
+kubectl create secret generic snyk-sysdig-secret -n snyk-monitor \
+  --from-literal=token=$SYSDIG_RISK_SPOTLIGHT_TOKEN \
+  --from-literal=region=$SYSDIG_AGENT_REGION \
+  --from-literal=cluster=$SYSDIG_AGENT_CLUSTER
 ```
+SYSDIG_RISK_SPOTLIGHT_TOKEN is the "Risk Spotlight Integrations Token" and has to be generated via the Sysdig UI. To create this API token, see the
+[Sysdig Risk Spotlight guide](https://docs.sysdig.com/en/docs/sysdig-secure/integrations-for-sysdig-secure/risk-spotlight-integrations/#generate-a-token-for-the-integration).
+SYSDIG_AGENT_REGION and SYSDIG_AGENT_CLUSTER are the ones that you configured when installing the [On Prem Sysdig Agent](https://docs.sysdig.com/en/docs/installation/agent-install-for-on-prem/#options),
+global.sysdig.region and global.clusterConfig.name.
 
 To enable Snyk to integrate with Sysdig and collect information about packages executed at runtime, use `--set sysdig.enabled=true` when installing the snyk-monitor:
 
@@ -210,7 +216,7 @@ helm upgrade --install snyk-monitor snyk-charts/snyk-monitor \
   --set sysdig.enabled=true
 ```
 
-> NOTE: The above command should be executed right after installing Sysdig. This will upgrade or install the snyk monitor, to allow the detection of Sysdig in the cluster.
+> NOTE: The above command should be executed after installing Sysdig. This will upgrade or install the snyk monitor, to allow the detection of Sysdig in the cluster.
 
 The snyk-monitor will now collect data from Sysdig every 4 hours.
 

diff --git a/snyk-monitor/templates/deployment.yaml b/snyk-monitor/templates/deployment.yaml
@@ -188,6 +188,24 @@ spec:
           - name: SNYK_WORKERS_COUNT
             value: {{ quote .Values.workers.count }}
           {{- if .Values.sysdig.enabled }}
+          - name: SNYK_SYSDIG_RISK_SPOTLIGHT_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: snyk-sysdig-secret
+                key: token
+                optional: true
+          - name: SNYK_SYSDIG_REGION_URL
+            valueFrom:
+              secretKeyRef:
+                name: snyk-sysdig-secret
+                key: region
+                optional: true
+          - name: SNYK_SYSDIG_CLUSTER_NAME
+            valueFrom:
+              secretKeyRef:
+                name: snyk-sysdig-secret
+                key: cluster
+                optional: true
           - name: SNYK_SYSDIG_ENDPOINT
             valueFrom:
               secretKeyRef:

diff --git a/snyk-monitor/values.yaml b/snyk-monitor/values.yaml
@@ -153,9 +153,9 @@ skopeo:
 
 workers:
   count: 5
+
 sysdig:
   enabled: false
-  namespace: sysdig-agent
   secretName: sysdig-eve-secret
 
 strategy:

diff --git a/src/common/config.ts b/src/common/config.ts
@@ -47,7 +47,19 @@ config.EXCLUDED_NAMESPACES = loadExcludedNamespaces();
 config.WORKERS_COUNT = Number(config.WORKERS_COUNT) || 10;
 config.SKOPEO_COMPRESSION_LEVEL = Number(config.SKOPEO_COMPRESSION_LEVEL) || 6;
 
-// return Sysdig endpoint information
+// return Sysdig v2 endpoint information
+if (
+  config.SYSDIG_RISK_SPOTLIGHT_TOKEN &&
+  config.SYSDIG_REGION_URL &&
+  config.SYSDIG_CLUSTER_NAME
+) {
+  config.SYSDIG_RISK_SPOTLIGHT_TOKEN =
+    config.SYSDIG_RISK_SPOTLIGHT_TOKEN.trim();
+  config.SYSDIG_REGION_URL = config.SYSDIG_REGION_URL.trim();
+  config.SYSDIG_CLUSTER_NAME = config.SYSDIG_CLUSTER_NAME.trim();
+}
+
+// return Sysdig v1 endpoint information
 if (config.SYSDIG_ENDPOINT && config.SYSDIG_TOKEN) {
   config.SYSDIG_ENDPOINT = config.SYSDIG_ENDPOINT.trim();
   config.SYSDIG_TOKEN = config.SYSDIG_TOKEN.trim();

diff --git a/src/common/types.ts b/src/common/types.ts
@@ -31,8 +31,11 @@ export interface Config {
   POLICIES_STORAGE_ROOT: '/tmp/policies';
   EXCLUDED_NAMESPACES: string[] | null;
   SKOPEO_COMPRESSION_LEVEL: number;
-  SYSDIG_ENDPOINT: string;
-  SYSDIG_TOKEN: string;
+  SYSDIG_ENDPOINT?: string;
+  SYSDIG_TOKEN?: string;
+  SYSDIG_RISK_SPOTLIGHT_TOKEN?: string;
+  SYSDIG_REGION_URL?: string;
+  SYSDIG_CLUSTER_NAME?: string;
   HTTPS_PROXY: string | undefined;
   HTTP_PROXY: string | undefined;
   NO_PROXY: string | undefined;

diff --git a/src/data-scraper/index.ts b/src/data-scraper/index.ts
@@ -13,12 +13,23 @@ const httpsAgent = new HttpsAgent({
   rejectUnauthorized: false,
 });
 
-function getSysdigUrl(): string {
+// Soon to be deprecated
+function getSysdigV1Url(): string {
   return config.SYSDIG_ENDPOINT + '/v1/runtimeimages';
 }
+function getSysdigV1AuthHeader(): string {
+  return `Bearer ${config.SYSDIG_TOKEN}`;
+}
 
+function getSysdigUrl(): string {
+  return (
+    'https://' +
+    config.SYSDIG_REGION_URL +
+    '/api/scanning/eveintegration/v2/runtimeimages'
+  );
+}
 function getSysdigAuthHeader(): string {
-  return `Bearer ${config.SYSDIG_TOKEN}`;
+  return `Bearer ${config.SYSDIG_RISK_SPOTLIGHT_TOKEN}`;
 }
 
 function isSuccessStatusCode(statusCode: number | undefined): boolean {
@@ -29,19 +40,19 @@ function isSuccessStatusCode(statusCode: number | undefined): boolean {
 export async function validateConnectivity(): Promise<void> {
   const url = getSysdigUrl();
   const header = getSysdigAuthHeader();
+  const clusterName = config.SYSDIG_CLUSTER_NAME;
+
   const reqOptions: NeedleOptions = {
     agent: httpsAgent,
     headers: {
       authorization: header,
     },
     timeout: 10_000,
   };
-
   const limit: number = 1;
-  const cursor: string = '';
   const { response } = await retryRequest(
     'get',
-    `${url}?limit=${limit}&cursor=${cursor}`,
+    `${url}?clusterName=${clusterName}&limit=${limit}`,
     {},
     reqOptions,
   );
@@ -53,6 +64,99 @@ export async function validateConnectivity(): Promise<void> {
 export async function scrapeData(): Promise<void> {
   const url = getSysdigUrl();
   const header = getSysdigAuthHeader();
+  const clusterName = config.SYSDIG_CLUSTER_NAME;
+  const limit: number = 10;
+
+  const reqOptions: NeedleOptions = {
+    agent: httpsAgent,
+    headers: {
+      authorization: header,
+    },
+  };
+
+  let cursor: string = '';
+  while (true) {
+    try {
+      logger.info({ cursor }, 'attempting to get runtime images');
+
+      let requestUrl: string = `${url}?clusterName=${clusterName}&limit=${limit}`;
+      if (cursor) {
+        requestUrl = `${requestUrl}&cursor=${cursor}`;
+      }
+      const { response, attempt } = await retryRequest(
+        'get',
+        requestUrl,
+        {},
+        reqOptions,
+      );
+      if (!isSuccessStatusCode(response.statusCode)) {
+        throw new Error(
+          `${response.statusCode} ${response.statusMessage} for ${requestUrl}`,
+        );
+      }
+
+      logger.info(
+        {
+          attempt,
+          cursor,
+        },
+        'runtime images received successfully',
+      );
+
+      const responseBody: IRuntimeImagesResponse | undefined = response.body;
+      const runtimeDataPayload = constructRuntimeData(
+        responseBody?.data ?? [],
+        2,
+      );
+
+      logger.info({}, 'sending runtime data upstream');
+      await sendRuntimeData(runtimeDataPayload);
+
+      cursor = responseBody?.page.next || '';
+      if (!cursor) {
+        break;
+      }
+    } catch (error) {
+      logger.error(
+        {
+          error,
+          cursor,
+        },
+        'could not get runtime images',
+      );
+      break;
+    }
+  }
+}
+
+/** NOTE: This function can throw, so the caller should handle errors. */
+export async function validateConnectivityV1(): Promise<void> {
+  const url = getSysdigV1Url();
+  const header = getSysdigV1AuthHeader();
+  const reqOptions: NeedleOptions = {
+    agent: httpsAgent,
+    headers: {
+      authorization: header,
+    },
+    timeout: 10_000,
+  };
+
+  const limit: number = 1;
+  const cursor: string = '';
+  const { response } = await retryRequest(
+    'get',
+    `${url}?limit=${limit}&cursor=${cursor}`,
+    {},
+    reqOptions,
+  );
+  if (!isSuccessStatusCode(response.statusCode)) {
+    throw new Error(`${response.statusCode} ${response.statusMessage}`);
+  }
+}
+
+export async function scrapeDataV1(): Promise<void> {
+  const url = getSysdigV1Url();
+  const header = getSysdigV1AuthHeader();
 
   // limit: min 1, max 500, default 250
   const limit: number = 10;
@@ -87,7 +191,11 @@ export async function scrapeData(): Promise<void> {
       );
 
       const responseBody: IRuntimeImagesResponse | undefined = response.body;
-      const runtimeDataPayload = constructRuntimeData(responseBody?.data ?? []);
+      const runtimeDataPayload = constructRuntimeData(
+        responseBody?.data ?? [],
+        1,
+      );
+
       logger.info({}, 'sending runtime data upstream');
       await sendRuntimeData(runtimeDataPayload);
 

diff --git a/src/healthcheck.ts b/src/healthcheck.ts
@@ -4,6 +4,9 @@ import { state } from './state';
 
 import * as dataScraper from './data-scraper';
 
+export const sysdigV1 = 'V1';
+export const sysdigV2 = 'V2';
+
 export async function setupHealthCheck(): Promise<void> {
   const interval = 1 * 60 * 1000; // 1 minute in milliseconds
   setInterval(healthCheck, interval).unref();
@@ -20,13 +23,39 @@ async function healthCheck(): Promise<void> {
   await sysdigHealthCheck();
 }
 
+export function getSysdigVersion() {
+  if (
+    config.SYSDIG_REGION_URL &&
+    config.SYSDIG_RISK_SPOTLIGHT_TOKEN &&
+    config.SYSDIG_CLUSTER_NAME
+  ) {
+    return sysdigV2;
+  } else if (config.SYSDIG_ENDPOINT && config.SYSDIG_TOKEN) {
+    return sysdigV1;
+  } else {
+    return '';
+  }
+}
+
 async function sysdigHealthCheck(): Promise<void> {
-  if (!config.SYSDIG_ENDPOINT || !config.SYSDIG_TOKEN) {
+  if (
+    !(
+      config.SYSDIG_CLUSTER_NAME &&
+      config.SYSDIG_RISK_SPOTLIGHT_TOKEN &&
+      config.SYSDIG_REGION_URL
+    ) ||
+    !(config.SYSDIG_ENDPOINT && config.SYSDIG_TOKEN)
+  ) {
     return;
   }
 
   try {
-    await dataScraper.validateConnectivity();
+    let sysdigVersion = getSysdigVersion();
+    if (sysdigVersion == sysdigV1) {
+      await dataScraper.validateConnectivityV1();
+    } else {
+      await dataScraper.validateConnectivity();
+    }
   } catch (error) {
     logger.error({ error }, 'could not connect to the Sysdig integration');
   }

diff --git a/src/index.ts b/src/index.ts
@@ -10,8 +10,8 @@ import { beginWatchingWorkloads } from './supervisor/watchers';
 import { loadAndSendWorkloadEventsPolicy } from './common/policy';
 import { sendClusterMetadata } from './transmitter';
 import { setSnykMonitorAgentId } from './supervisor/agent';
-import { scrapeData } from './data-scraper';
-import { setupHealthCheck } from './healthcheck';
+import { scrapeData, scrapeDataV1 } from './data-scraper';
+import { getSysdigVersion, setupHealthCheck, sysdigV1 } from './healthcheck';
 
 process.on('uncaughtException', (error) => {
   if (state.shutdownInProgress) {
@@ -68,15 +68,29 @@ async function monitor(): Promise<void> {
 }
 
 async function setupSysdigIntegration(): Promise<void> {
-  if (!config.SYSDIG_ENDPOINT || !config.SYSDIG_TOKEN) {
-    logger.info({}, 'Sysdig integration not detected');
+  if (
+    !(
+      config.SYSDIG_REGION_URL &&
+      config.SYSDIG_RISK_SPOTLIGHT_TOKEN &&
+      config.SYSDIG_CLUSTER_NAME
+    ) ||
+    !(config.SYSDIG_ENDPOINT && config.SYSDIG_TOKEN)
+  ) {
+    console.log('Sysdig integration not enabled');
     return;
   }
 
-  const initialInterval: number = 20 * 60 * 1000; // 20 mins in milliseconds
+  let sysdigVersion = getSysdigVersion();
+  logger.info({}, `Sysdig ${sysdigVersion} data scraping starting`);
+
+  const initialInterval: number = 60 * 1000; // 20 mins in milliseconds
   setTimeout(async () => {
     try {
-      await scrapeData();
+      if (sysdigVersion == sysdigV1) {
+        await scrapeDataV1();
+      } else {
+        await scrapeData();
+      }
     } catch (error) {
       logger.error(
         { error },
@@ -88,7 +102,11 @@ async function setupSysdigIntegration(): Promise<void> {
   const interval: number = 4 * 60 * 60 * 1000; // 4 hours in milliseconds
   setInterval(async () => {
     try {
-      await scrapeData();
+      if (sysdigVersion == sysdigV1) {
+        await scrapeDataV1();
+      } else {
+        await scrapeData();
+      }
     } catch (error) {
       logger.error({ error }, 'an error occurred while scraping runtime data');
     }

diff --git a/src/transmitter/payload.ts b/src/transmitter/payload.ts
@@ -154,6 +154,7 @@ const workloadKindMap = {
 };
 export function constructRuntimeData(
   runtimeResults: IRuntimeImage[],
+  sysdigVersion: number,
 ): IRuntimeDataPayload {
   const filteredRuntimeResults = runtimeResults.reduce((acc, runtimeResult) => {
     if (!isExcludedNamespace(runtimeResult.namespace)) {
@@ -185,6 +186,7 @@ export function constructRuntimeData(
   return {
     identity: {
       type: 'sysdig',
+      sysdigVersion: sysdigVersion,
     },
     target: {
       agentId: config.AGENT_ID,

diff --git a/src/transmitter/types.ts b/src/transmitter/types.ts
@@ -128,6 +128,7 @@ export interface IRuntimeDataFact {
 export interface IRuntimeDataPayload extends Omit<ScanResult, 'target'> {
   identity: {
     type: RuntimeDataType;
+    sysdigVersion: number;
   };
   target: IRuntimeDataTarget;
   facts: [IRuntimeDataFact];