feat: improved sync through repositories

bcgov-nr · Jan 30, 2025 · a86e5b2 · a86e5b2
1 parent fe25e2b
commit a86e5b2
Show file tree

Hide file tree

Showing 28 changed files with 1,759 additions and 1,018 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -27,17 +27,17 @@
     "@mikro-orm/core": "^6.4.4",
     "@mikro-orm/mongodb": "^6.4.4",
     "@mikro-orm/nestjs": "^6.0.2",
-    "@nestjs/axios": "^3.1.3",
-    "@nestjs/common": "^10.4.15",
-    "@nestjs/config": "^3.3.0",
-    "@nestjs/core": "^10.4.15",
-    "@nestjs/jwt": "^10.2.0",
-    "@nestjs/passport": "^10.0.3",
-    "@nestjs/platform-express": "^10.4.15",
-    "@nestjs/schedule": "^4.1.2",
-    "@nestjs/serve-static": "^4.0.2",
-    "@nestjs/swagger": "^8.1.1",
-    "@nestjs/terminus": "^10.2.3",
+    "@nestjs/axios": "^4.0.0",
+    "@nestjs/common": "^11.0.6",
+    "@nestjs/config": "^4.0.0",
+    "@nestjs/core": "^11.0.6",
+    "@nestjs/jwt": "^11.0.0",
+    "@nestjs/passport": "^11.0.5",
+    "@nestjs/platform-express": "^11.0.6",
+    "@nestjs/schedule": "^5.0.1",
+    "@nestjs/serve-static": "^5.0.1",
+    "@nestjs/swagger": "^11.0.3",
+    "@nestjs/terminus": "^11.0.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "connect-mongo": "^5.1.0",
@@ -67,9 +67,9 @@
   "devDependencies": {
     "@eslint/js": "^9.19.0",
     "@golevelup/ts-jest": "^0.6.2",
-    "@nestjs/cli": "^10.4.9",
-    "@nestjs/schematics": "^10.2.3",
-    "@nestjs/testing": "^10.4.15",
+    "@nestjs/cli": "^11.0.2",
+    "@nestjs/schematics": "^11.0.0",
+    "@nestjs/testing": "^11.0.6",
     "@types/deep-equal": "^1.0.4",
     "@types/ejs": "^3.1.5",
     "@types/eslint__js": "^8.42.3",
@@ -80,17 +80,17 @@
     "@types/lodash.get": "^4.4.9",
     "@types/lodash.merge": "^4.6.9",
     "@types/lodash.set": "^4.3.9",
-    "@types/node": "^22.10.10",
+    "@types/node": "^22.12.0",
     "@types/passport": "^1.0.17",
     "@types/passport-http": "^0.3.11",
     "@types/passport-jwt": "^4.0.1",
     "@types/supertest": "^6.0.2",
     "@types/uuid": "^10.0.0",
-    "@typescript-eslint/eslint-plugin": "^8.19.1",
-    "@typescript-eslint/parser": "^8.19.1",
-    "eslint": "^9.18.0",
-    "eslint-config-prettier": "^9.1.0",
-    "eslint-plugin-prettier": "^5.2.1",
+    "@typescript-eslint/eslint-plugin": "^8.22.0",
+    "@typescript-eslint/parser": "^8.22.0",
+    "eslint": "^9.19.0",
+    "eslint-config-prettier": "^10.0.1",
+    "eslint-plugin-prettier": "^5.2.3",
     "jest": "^29.7.0",
     "prettier": "^3.4.2",
     "source-map-support": "^0.5.20",
@@ -100,7 +100,7 @@
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
     "typescript": "^5.7.3",
-    "typescript-eslint": "^8.19.1"
+    "typescript-eslint": "^8.22.0"
   },
   "jest": {
     "moduleFileExtensions": [
@@ -118,5 +118,11 @@
     ],
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
+  },
+  "overrides": {
+    "@mikro-orm/nestjs": {
+      "@nestjs/common": "^10.0.0 || ^11.0.0",
+      "@nestjs/core": "^10.0.0 || ^11.0.0"
+    }
   }
 }
diff --git a/src/audit/audit.service.ts b/src/audit/audit.service.ts
@@ -458,10 +458,13 @@ export class AuditService {
   }
 
   /**
-   * Records authorization events in the audit activity log
-   * @param req The initiating http request
-   * @param type Indicates if this is the start or end
-   * @param outcome The outcome of the authorization
+   * Records tool sync events in the audit activity log
+   * @param type
+   * @param outcome
+   * @param message
+   * @param project
+   * @param service
+   * @param failure
    */
   public recordToolsSync(
     type: 'start' | 'end' | 'info',

diff --git a/src/collection/account.service.ts b/src/collection/account.service.ts
@@ -4,7 +4,6 @@ import {
   BadRequestException,
   NotFoundException,
   ServiceUnavailableException,
-  HttpException,
 } from '@nestjs/common';
 import { Request } from 'express';
 import { Cron, CronExpression, SchedulerRegistry } from '@nestjs/schedule';
@@ -25,7 +24,6 @@ import {
   VAULT_KV_APPS_MOUNT,
   REDIS_PUBSUB,
   VAULT_KV_APPS_TOOLS_PATH_TPL,
-  REDIS_QUEUES,
 } from '../constants';
 import { ActionError } from '../intention/action.error';
 import { AuditService } from '../audit/audit.service';
@@ -148,7 +146,7 @@ export class AccountService {
     id: string,
     expirationInSeconds: number,
     patchVault: boolean,
-    sync: boolean,
+    syncSecrets: boolean,
     creatorGuid: string,
     autoRenew: boolean,
   ): Promise<TokenCreateDTO> {
@@ -204,7 +202,7 @@ export class AccountService {
     if (patchVault) {
       await this.addTokenToAccountServices(token, account);
     }
-    if (sync && this.githubSyncService.isEnabled()) {
+    if (syncSecrets && this.githubSyncService.isEnabled()) {
       try {
         await this.refresh(account.id.toString());
       } catch (error) {
@@ -370,116 +368,7 @@ export class AccountService {
     this.doSyncPreflight(id, account);
 
     // Queue the sync
-    this.redisService.queue(REDIS_QUEUES.VAULT_TOOLS_GITHUB_SYNC, id);
-  }
-
-  @Cron(CronExpression.EVERY_10_SECONDS, {
-    name: REDIS_QUEUES.VAULT_TOOLS_GITHUB_SYNC,
-  })
-  @CreateRequestContext()
-  async pollRefreshCron(): Promise<void> {
-    const job = this.schedulerRegistry.getCronJob(
-      REDIS_QUEUES.VAULT_TOOLS_GITHUB_SYNC,
-    );
-    job.stop(); // pausing the cron job
-    try {
-      const id = await this.redisService.dequeue(
-        REDIS_QUEUES.VAULT_TOOLS_GITHUB_SYNC,
-      );
-      if (id) {
-        await this.runRefresh(id);
-      }
-    } catch (error) {
-      // Continue - this.runRefresh() audits failures
-    } finally {
-      job.start(); // resuming the cron job
-    }
-  }
-
-  async runRefresh(id: string): Promise<void> {
-    const account = await this.getBrokerAccount(id);
-    this.doSyncPreflight(id, account);
-
-    this.auditService.recordToolsSync(
-      'info',
-      'unknown',
-      `Sync broker account (${account.clientId})`,
-    );
-
-    const downstreamServices =
-      await this.graphRepository.getDownstreamVertex<ServiceDto>(
-        account.vertex.toString(),
-        CollectionNameEnum.service,
-        4,
-      );
-    if (downstreamServices) {
-      const errors = [];
-      for (const serviceUpDown of downstreamServices) {
-        const service = serviceUpDown.collection;
-        const projectDtoArr =
-          await this.graphRepository.getUpstreamVertex<ProjectDto>(
-            service.vertex.toString(),
-            CollectionNameEnum.project,
-            ['component'],
-          );
-        if (projectDtoArr.length !== 1) {
-          this.auditService.recordToolsSync(
-            'info',
-            'unknown',
-            `Skip sync: ${service.name}`,
-            'Unknown',
-            service.name,
-          );
-          continue;
-        }
-        const project = projectDtoArr[0].collection;
-
-        this.auditService.recordToolsSync(
-          'start',
-          'unknown',
-          `Start sync: ${service.name}`,
-          project.name,
-          service.name,
-        );
-
-        try {
-          await this.githubSyncService.refresh(project, service);
-          this.auditService.recordToolsSync(
-            'end',
-            'success',
-            `End sync: ${service.name}`,
-            project.name,
-            service.name,
-          );
-        } catch (error) {
-          let httpErr = error;
-          if (!(httpErr instanceof HttpException)) {
-            httpErr = new BadRequestException({
-              message: error.message,
-            });
-          }
-          this.auditService.recordToolsSync(
-            'end',
-            'failure',
-            `End sync: ${service.name}`,
-            project.name,
-            service.name,
-            httpErr,
-          );
-          errors.push(httpErr);
-        }
-      }
-
-      if (errors.length > 0) {
-        throw errors[0];
-      }
-    } else {
-      this.auditService.recordToolsSync(
-        'info',
-        'unknown',
-        `No services associated with this broker account (${account.clientId})`,
-      );
-    }
+    this.githubSyncService.refreshByAccount(account);
   }
 
   @Cron(CronExpression.EVERY_MINUTE)

diff --git a/src/collection/collection.controller.ts b/src/collection/collection.controller.ts
@@ -44,11 +44,13 @@ import { CollectionNames } from '../persistence/dto/collection-dto-union.type';
 import { PersistenceCacheKey } from '../persistence/persistence-cache-key.decorator';
 import { PersistenceCacheInterceptor } from '../persistence/persistence-cache.interceptor';
 import { PERSISTENCE_CACHE_KEY_CONFIG } from '../persistence/persistence.constants';
-import { ExpiryQuery } from './dto/expiry-query.dto';
+import { BrokerAccountTokenGenerateQuery } from './dto/broker-account-token-generate-query.dto';
 import { RedisService } from '../redis/redis.service';
 import { JwtRegistryDto } from '../persistence/dto/jwt-registry.dto';
 import { UserBaseDto } from '../persistence/dto/user.dto';
 import { TeamCollectionService } from './team-collection.service';
+import { SyncRepositoryQuery } from './dto/sync-repository-query.dto';
+import { RepositoryCollectionService } from './repository-collection.service';
 
 @Controller({
   path: 'collection',
@@ -59,8 +61,9 @@ export class CollectionController {
     private readonly accountService: AccountService,
     private readonly service: CollectionService,
     private readonly redis: RedisService,
-    private readonly userCollectionService: UserCollectionService,
+    private readonly repositoryCollectionService: RepositoryCollectionService,
     private readonly teamCollectionService: TeamCollectionService,
+    private readonly userCollectionService: UserCollectionService,
   ) {}
 
   /**
@@ -152,13 +155,43 @@ export class CollectionController {
   @Roles('admin')
   @AllowOwner({
     graphObjectType: 'collection',
-    graphObjectCollection: 'brokerAccount',
+    graphObjectCollection: 'team',
+    graphIdFromParamKey: 'id',
+    permission: 'sudo',
+  })
+  @UseGuards(BrokerOidcAuthGuard)
+  @UsePipes(new ValidationPipe({ transform: true }))
+  async teamRefresh(
+    @Param('id') id: string,
+    @Query() syncQuery: SyncRepositoryQuery,
+  ): Promise<void> {
+    return await this.teamCollectionService.refresh(
+      id,
+      syncQuery.syncSecrets,
+      syncQuery.syncUsers,
+    );
+  }
+
+  @Post('repository/:id/refresh')
+  @Roles('admin')
+  @AllowOwner({
+    graphObjectType: 'collection',
+    graphObjectCollection: 'repository',
     graphIdFromParamKey: 'id',
     permission: 'sudo',
   })
   @UseGuards(BrokerOidcAuthGuard)
-  async refreshGitHub(@Param('id') id: string): Promise<void> {
-    return await this.teamCollectionService.refresh(id);
+  @UsePipes(new ValidationPipe({ transform: true }))
+  async repositoryRefresh(
+    @Param('id') id: string,
+    @Query() syncQuery: SyncRepositoryQuery,
+  ): Promise<void> {
+    console.log(syncQuery);
+    return await this.repositoryCollectionService.refresh(
+      id,
+      syncQuery.syncSecrets,
+      syncQuery.syncUsers,
+    );
   }
 
   @Post('broker-account/:id/token')
@@ -178,15 +211,15 @@ export class CollectionController {
   @UsePipes(new ValidationPipe({ transform: true }))
   async generateAccountToken(
     @Param('id') id: string,
-    @Query() expiryQuery: ExpiryQuery,
+    @Query() genQuery: BrokerAccountTokenGenerateQuery,
     @Request() req: ExpressRequest,
   ) {
     return this.accountService.generateAccountToken(
       req,
       id,
-      expiryQuery.expiration,
-      expiryQuery.patch ?? false,
-      expiryQuery.sync ?? false,
+      genQuery.expiration,
+      genQuery.patch ?? false,
+      genQuery.syncSecrets ?? false,
       get((req.user as any).userinfo, OAUTH2_CLIENT_MAP_GUID),
       false,
     );

diff --git a/src/collection/collection.module.ts b/src/collection/collection.module.ts
@@ -15,6 +15,7 @@ import { UtilModule } from '../util/util.module';
 import { RedisModule } from '../redis/redis.module';
 import { VaultModule } from '../vault/vault.module';
 import { TeamCollectionService } from './team-collection.service';
+import { RepositoryCollectionService } from './repository-collection.service';
 
 /**
  * The collection module enables the viewing and manipulation of the objects
@@ -38,8 +39,9 @@ import { TeamCollectionService } from './team-collection.service';
   providers: [
     AccountService,
     CollectionService,
-    UserCollectionService,
+    RepositoryCollectionService,
     TeamCollectionService,
+    UserCollectionService,
   ],
   exports: [CollectionService, UserCollectionService],
 })

diff --git a/src/collection/dto/expiry-query.dto.ts → ...roker-account-token-generate-query.dto.ts b/src/collection/dto/expiry-query.dto.ts → ...roker-account-token-generate-query.dto.ts
@@ -1,9 +1,10 @@
 import { Transform, Type } from 'class-transformer';
 import { IsBoolean, IsInt, IsOptional, Max } from 'class-validator';
+import { SyncRepositoryQuery } from './sync-repository-query.dto';
 
 export const DAYS_365_IN_SECONDS = 60 * 60 * 24 * 365;
 
-export class ExpiryQuery {
+export class BrokerAccountTokenGenerateQuery extends SyncRepositoryQuery {
   @IsInt()
   @Max(DAYS_365_IN_SECONDS)
   @Type(() => Number)
@@ -13,9 +14,4 @@ export class ExpiryQuery {
   @IsOptional()
   @Transform(({ value }) => value === 'true')
   patch?: boolean;
-
-  @IsBoolean()
-  @IsOptional()
-  @Transform(({ value }) => value === 'true')
-  sync?: boolean;
 }