feat: api7 API-level resource type filter (#146)

api7 · Jul 16, 2024 · d678c65 · d678c65
1 parent 2804ab6
commit d678c65
Show file tree

Hide file tree

Showing 8 changed files with 256 additions and 6 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -36,7 +36,7 @@ jobs:
 
   api7:
     runs-on: ubuntu-latest
-    if: contains(github.event.pull_request.labels.*.name, 'test/api7')
+    if: contains(github.event.pull_request.labels.*.name, 'test/api7') || github.event_name == 'push'
     strategy:
       matrix:
         version: [3.2.13.0, 3.2.14.0]

diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml
@@ -21,5 +21,7 @@ jobs:
       # Run unit tests
       - name: Run CLI unit tests
         run: npx nx run cli:test
+      - name: Run API7 Backend unit tests
+        run: npx nx run backend-api7:test
       - name: Run OpenAPI Converter unit tests
-        run: NODE_OPTIONS=--experimental-vm-modules npx nx run converter-openapi:test
+        run: npx nx run converter-openapi:test
diff --git a/apps/cli/src/command/dump.command.ts b/apps/cli/src/command/dump.command.ts
@@ -1,3 +1,4 @@
+import { BackendAPI7 } from '@api7/adc-backend-api7';
 import * as ADCSDK from '@api7/adc-sdk';
 import { Listr, ListrTask } from 'listr2';
 import { writeFile } from 'node:fs/promises';
@@ -41,7 +42,9 @@ export const LoadRemoteConfigurationTask = ({
       {
         title: 'Filter configuration resource type',
         enabled: () =>
-          includeResourceType?.length > 0 || excludeResourceType?.length > 0,
+          //TODO implement API-level resource filtering on APISIX backend
+          !(backend instanceof BackendAPI7) &&
+          (includeResourceType?.length > 0 || excludeResourceType?.length > 0),
         task: () => {
           ctx.remote = filterResourceType(
             ctx.remote,

diff --git a/libs/backend-api7/src/fetcher.ts b/libs/backend-api7/src/fetcher.ts
@@ -1,6 +1,7 @@
 import * as ADCSDK from '@api7/adc-sdk';
 import { Axios } from 'axios';
 import { ListrTask } from 'listr2';
+import { isEmpty } from 'lodash';
 
 import { ToADC } from './transformer';
 import * as typing from './typing';
@@ -14,11 +15,15 @@ type FetchTask = ListrTask<{
 export class Fetcher {
   private readonly toADC = new ToADC();
 
-  constructor(private readonly client: Axios) {}
+  constructor(
+    private readonly client: Axios,
+    private readonly backendOpts: ADCSDK.BackendOptions,
+  ) {}
 
   public listServices(): FetchTask {
     return {
       title: 'Fetch services',
+      skip: this.isSkip([ADCSDK.ResourceType.SERVICE]),
       task: async (ctx, task) => {
         const resp = await this.client.get<{ list: Array<typing.Service> }>(
           `/api/gateway_groups/${ctx.gatewayGroupId}/services`,
@@ -62,6 +67,7 @@ export class Fetcher {
   public listConsumers(): FetchTask {
     return {
       title: 'Fetch consumers',
+      skip: this.isSkip([ADCSDK.ResourceType.CONSUMER]),
       task: async (ctx, task) => {
         const resp = await this.client.get<{ list: Array<typing.Consumer> }>(
           '/apisix/admin/consumers',
@@ -81,6 +87,7 @@ export class Fetcher {
   public listSSLs(): FetchTask {
     return {
       title: 'Fetch ssls',
+      skip: this.isSkip([ADCSDK.ResourceType.SSL]),
       task: async (ctx, task) => {
         const resp = await this.client.get<{ list: Array<typing.SSL> }>(
           '/apisix/admin/ssls',
@@ -100,6 +107,7 @@ export class Fetcher {
   public listGlobalRules(): FetchTask {
     return {
       title: 'Fetch global rules',
+      skip: this.isSkip([ADCSDK.ResourceType.GLOBAL_RULE]),
       task: async (ctx, task) => {
         const resp = await this.client.get<{ list: Array<typing.GlobalRule> }>(
           '/apisix/admin/global_rules',
@@ -119,6 +127,7 @@ export class Fetcher {
   public listMetadatas(): FetchTask {
     return {
       title: 'Fetch plugin metadata',
+      skip: this.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA]),
       task: async (ctx, task) => {
         const resp = await this.client.get<Array<string>>(
           '/apisix/admin/plugins/list',
@@ -160,6 +169,31 @@ export class Fetcher {
     };
   }
 
+  private isSkip(
+    requiredTypes: Array<ADCSDK.ResourceType>,
+  ): () => string | undefined {
+    return () => {
+      const msg = 'excluded by resource type filters';
+      if (
+        this.backendOpts?.includeResourceType &&
+        !isEmpty(this.backendOpts.includeResourceType) &&
+        !requiredTypes.some((item) =>
+          this.backendOpts.includeResourceType.includes(item),
+        )
+      ) {
+        return msg;
+      } else if (
+        this.backendOpts?.excludeResourceType &&
+        !isEmpty(this.backendOpts.excludeResourceType) &&
+        requiredTypes.every((item) =>
+          this.backendOpts.excludeResourceType.includes(item),
+        )
+      ) {
+        return msg;
+      }
+    };
+  }
+
   public allTask() {
     return [
       this.listServices(),

diff --git a/libs/backend-api7/src/index.ts b/libs/backend-api7/src/index.ts
@@ -178,7 +178,7 @@ export class BackendAPI7 implements ADCSDK.Backend {
   }
 
   public async dump(): Promise<Listr<{ remote: ADCSDK.Configuration }>> {
-    const fetcher = new Fetcher(this.client);
+    const fetcher = new Fetcher(this.client, this.opts);
 
     return new Listr<{ remote: ADCSDK.Configuration }>(
       [

diff --git a/libs/backend-api7/test/fetcher.spec.ts b/libs/backend-api7/test/fetcher.spec.ts
@@ -0,0 +1,206 @@
+import * as ADCSDK from '@api7/adc-sdk';
+import axios from 'axios';
+
+import { Fetcher } from '../src/fetcher';
+
+describe('Fetcher', () => {
+  describe('API-level resource type filters', () => {
+    type TrickFetcher = typeof Fetcher & {
+      isSkip: Fetcher['isSkip'];
+    };
+    const newFetcher = (opts?: Partial<ADCSDK.BackendOptions>) =>
+      new Fetcher(
+        axios.create(),
+        opts as ADCSDK.BackendOptions,
+      ) as unknown as TrickFetcher;
+    const checkSkip = (func: () => string | undefined) => !!func();
+
+    it('should include services', () => {
+      const fetcher = newFetcher({
+        includeResourceType: [ADCSDK.ResourceType.SERVICE],
+      });
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
+        true,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
+        true,
+      );
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
+      ).toEqual(true);
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
+      ).toEqual(true);
+    });
+
+    it('should include services, consumers, ssls', () => {
+      const fetcher = newFetcher({
+        includeResourceType: [
+          ADCSDK.ResourceType.SERVICE,
+          ADCSDK.ResourceType.CONSUMER,
+          ADCSDK.ResourceType.SSL,
+        ],
+      });
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
+        false,
+      );
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
+      ).toEqual(true);
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
+      ).toEqual(true);
+    });
+
+    it('should include all', () => {
+      const fetcher = newFetcher({
+        includeResourceType: [
+          ADCSDK.ResourceType.SERVICE,
+          ADCSDK.ResourceType.CONSUMER,
+          ADCSDK.ResourceType.SSL,
+          ADCSDK.ResourceType.GLOBAL_RULE,
+          ADCSDK.ResourceType.PLUGIN_METADATA,
+        ],
+      });
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
+        false,
+      );
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
+      ).toEqual(false);
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
+      ).toEqual(false);
+    });
+
+    it('should include all (include list defined but empty)', () => {
+      const fetcher = newFetcher({
+        includeResourceType: [],
+      });
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
+        false,
+      );
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
+      ).toEqual(false);
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
+      ).toEqual(false);
+    });
+
+    it('should exclude services', () => {
+      const fetcher = newFetcher({
+        excludeResourceType: [ADCSDK.ResourceType.SERVICE],
+      });
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
+        true,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
+        false,
+      );
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
+      ).toEqual(false);
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
+      ).toEqual(false);
+    });
+
+    it('should exclude services, consumers, ssls', () => {
+      const fetcher = newFetcher({
+        excludeResourceType: [
+          ADCSDK.ResourceType.SERVICE,
+          ADCSDK.ResourceType.CONSUMER,
+          ADCSDK.ResourceType.SSL,
+        ],
+      });
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
+        true,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
+        true,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
+        true,
+      );
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
+      ).toEqual(false);
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
+      ).toEqual(false);
+    });
+
+    it('should exclude all', () => {
+      const fetcher = newFetcher({
+        excludeResourceType: [
+          ADCSDK.ResourceType.SERVICE,
+          ADCSDK.ResourceType.CONSUMER,
+          ADCSDK.ResourceType.SSL,
+          ADCSDK.ResourceType.GLOBAL_RULE,
+          ADCSDK.ResourceType.PLUGIN_METADATA,
+        ],
+      });
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
+        true,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
+        true,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
+        true,
+      );
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
+      ).toEqual(true);
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
+      ).toEqual(true);
+    });
+
+    it('should include all (exclude list defined but empty)', () => {
+      const fetcher = newFetcher({
+        excludeResourceType: [],
+      });
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
+        false,
+      );
+      expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
+        false,
+      );
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
+      ).toEqual(false);
+      expect(
+        checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
+      ).toEqual(false);
+    });
+  });
+});
diff --git a/libs/backend-api7/tsconfig.spec.json b/libs/backend-api7/tsconfig.spec.json
@@ -13,6 +13,7 @@
     "src/**/*.test.ts",
     "src/**/*.spec.ts",
     "src/**/*.d.ts",
-    "e2e/**/*.e2e-spec.ts"
+    "e2e/**/*.e2e-spec.ts",
+    "test/**/*.spec.ts"
   ]
 }
diff --git a/libs/sdk/src/backend/index.ts b/libs/sdk/src/backend/index.ts
@@ -12,6 +12,10 @@ export interface BackendOptions {
   tlsClientKeyFile?: string;
   tlsSkipVerify?: boolean;
   verbose?: number;
+
+  labelSelector?: Record<string, string>;
+  includeResourceType?: Array<ADCSDK.ResourceType>;
+  excludeResourceType?: Array<ADCSDK.ResourceType>;
 }
 
 export interface Backend {