improve batch loader of products and environments

src/batch/data-rules.js

@@ -376,6 +376,7 @@ const metadata = {
   Product: {
     query: 'allProducts',
     refKey: 'appId',
+    compositeRefKey: ['name', 'namespace'],
     sync: ['name', 'namespace'],
     transformations: {
       dataset: { name: 'connectOne', list: 'allDatasets', refKey: 'name' },
@@ -410,12 +411,14 @@ const metadata = {
         name: 'connectMany',
         list: 'allGatewayServices',
         refKey: 'name',
+        filterByNamespace: true,
       },
       legal: { name: 'connectOne', list: 'allLegals', refKey: 'reference' },
       credentialIssuer: {
         name: 'connectOne',
         list: 'allCredentialIssuers',
         refKey: 'name',
+        filterByNamespace: true,
       },
     },
     validations: {

src/batch/feed-worker.ts

@@ -55,6 +55,7 @@ const transformations = {
 export const putFeedWorker = async (context: any, req: any, res: any) => {
   const entity = req.params['entity'];
   assert.strictEqual(entity in metadata, true);
+  logger.info('putFeedWorker %s', entity);
 
   const md = metadata[entity];
   const refKey = md.refKey;
@@ -164,7 +165,8 @@ export const getFeedWorker = async (context: any, req: any, res: any) => {
 const syncListOfRecords = async function (
   keystone: any,
   transformInfo: any,
-  records: any
+  records: any,
+  parentRecord?: any
 ): Promise<BatchResult[]> {
   const result: BatchResult[] = [];
   if (records == null || typeof records == 'undefined') {
@@ -179,7 +181,8 @@ const syncListOfRecords = async function (
         transformInfo.list,
         record[recordKey],
         record,
-        true
+        true,
+        parentRecord
       )
     );
   }
@@ -245,7 +248,7 @@ function buildQueryResponse(md: any, children: string[] = undefined): string[] {
     });
   }
   if ('ownedBy' in md) {
-    response.push(`${md.ownedBy} { id }`);
+    response.push(`${md.ownedBy} { id, namespace }`);
   }
 
   logger.debug('[buildQueryResponse] FINAL (%s) %j', md.query, response);
@@ -307,7 +310,8 @@ export const syncRecords = async function (
   feedEntity: string,
   eid: string,
   json: any,
-  children = false
+  children = false,
+  parentRecord: any = undefined
 ): Promise<BatchResult> {
   const md = (metadata as any)[feedEntity];
   const entity = 'entity' in md ? md['entity'] : feedEntity;
@@ -318,11 +322,20 @@ export const syncRecords = async function (
     'This entity is only part of a child.'
   );
 
-  assert.strictEqual(
-    typeof eid === 'string' && eid.length > 0,
-    true,
-    `Invalid ID for ${feedEntity} ${eid}`
-  );
+  const compositeKeyValues: any = {};
+  if (md.compositeRefKey) {
+    md.compositeRefKey.forEach((key: string) => {
+      compositeKeyValues[key] = json[key];
+    });
+  } else {
+    assert.strictEqual(
+      typeof eid === 'string' && eid.length > 0,
+      true,
+      `Invalid ID for ${feedEntity} ${md.refKey} = ${eid || 'blank'}`
+    );
+
+    compositeKeyValues[md.refKey] = eid;
+  }
 
   const batchService = new BatchService(context);
 
@@ -342,10 +355,9 @@ export const syncRecords = async function (
 
   let childResults: BatchResult[] = [];
 
-  const localRecord = await batchService.lookup(
+  const localRecord = await batchService.lookupUsingCompositeKey(
     md.query,
-    md.refKey,
-    eid,
+    compositeKeyValues,
     buildQueryResponse(md)
   );
   if (localRecord == null) {
@@ -365,9 +377,12 @@ export const syncRecords = async function (
             const allIds = await syncListOfRecords(
               context,
               transformInfo,
-              json[transformKey]
+              json[transformKey],
+              json
             );
             logger.debug('CHILDREN [%s] %j', transformKey, allIds);
+            childResults.push(...allIds);
+
             assert.strictEqual(
               allIds.filter((record) => record.status != 200).length,
               0,
@@ -380,8 +395,9 @@ export const syncRecords = async function (
               'There are some child records that have exclusive ownership already!'
             );
             json[transformKey + '_ids'] = allIds.map((status) => status.id);
-
-            childResults.push(...allIds);
+          }
+          if (transformInfo.filterByNamespace) {
+            json['_namespace'] = parentRecord['namespace'];
           }
           const transformMutation = await transformations[transformInfo.name](
             context,
@@ -403,7 +419,9 @@ export const syncRecords = async function (
           }
         }
       }
-      data[md.refKey] = eid;
+      if (eid) {
+        data[md.refKey] = eid;
+      }
       const nr = await batchService.create(entity, data);
       if (nr == null) {
         logger.error('CREATE FAILED (%s) %j', nr, data);
@@ -452,9 +470,13 @@ export const syncRecords = async function (
             const allIds = await syncListOfRecords(
               context,
               transformInfo,
-              json[transformKey]
+              json[transformKey],
+              json
             );
+
             logger.debug('CHILDREN [%s] %j', transformKey, allIds);
+            childResults.push(...allIds);
+
             assert.strictEqual(
               allIds.filter((record) => record.status != 200).length,
               0,
@@ -468,13 +490,14 @@ export const syncRecords = async function (
                   record.ownedBy != localRecord.id
               ).length,
               0,
-              'There are some child records that had ownership already (w/ local record)!'
+              'There are some child records that have ownership already (update not allowed)!'
             );
 
             json[transformKey + '_ids'] = allIds.map((status) => status.id);
-            childResults.push(...allIds);
           }
-
+          if (transformInfo.filterByNamespace) {
+            json['_namespace'] = parentRecord['namespace'];
+          }
           const transformMutation = await transformations[transformInfo.name](
             context,
             transformInfo,

src/batch/transformations/connectMany.ts

@@ -17,12 +17,21 @@ export async function connectMany(
   const batchService = new BatchService(keystone);
   if (idList != null) {
     for (const uniqueKey of idList) {
-      const lkup = await batchService.lookup(
-        transformInfo['list'],
-        transformInfo['refKey'],
-        uniqueKey,
-        []
-      );
+      const nsFilter = { namespace: inputData['_namespace'] } as any;
+      nsFilter[transformInfo['refKey']] = uniqueKey;
+      logger.error('T = %s -- %j %j', uniqueKey, inputData, currentData);
+      const lkup = transformInfo['filterByNamespace']
+        ? await batchService.lookupUsingCompositeKey(
+            transformInfo['list'],
+            nsFilter,
+            []
+          )
+        : await batchService.lookup(
+            transformInfo['list'],
+            transformInfo['refKey'],
+            uniqueKey,
+            []
+          );
       if (lkup == null) {
         logger.error(
           `Lookup failed for ${transformInfo['list']} ${transformInfo['refKey']}!`

src/batch/transformations/connectOne.ts

@@ -29,12 +29,21 @@ export async function connectOne(
     }
   }
 
-  const lkup = await batchService.lookup(
-    transformInfo['list'],
-    transformInfo['refKey'],
-    value,
-    []
-  );
+  const nsFilter = { namespace: inputData['_namespace'] } as any;
+  nsFilter[transformInfo['refKey']] = value;
+  const lkup = transformInfo['filterByNamespace']
+    ? await batchService.lookupUsingCompositeKey(
+        transformInfo['list'],
+        nsFilter,
+        []
+      )
+    : await batchService.lookup(
+        transformInfo['list'],
+        transformInfo['refKey'],
+        value,
+        []
+      );
+
   if (lkup == null) {
     logger.error(
       `Lookup failed for ${transformInfo['list']} ${transformInfo['refKey']}!`

src/lists/Product.js

@@ -98,12 +98,12 @@ module.exports = {
       );
     },
 
-    // beforeDelete: async function ({ existingItem, context }) {
-    //   await DeleteProductEnvironments(
-    //     context,
-    //     context.authedItem['namespace'],
-    //     existingItem.id
-    //   );
-    // },
+    beforeDelete: async function ({ existingItem, context }) {
+      await DeleteProductEnvironments(
+        context.createContext({ skipAccessControl: true }),
+        context.authedItem['namespace'],
+        existingItem.id
+      );
+    },
   },
 };

src/services/keystone/batch-service.ts

@@ -96,6 +96,52 @@ export class BatchService {
     return result['data'][query].length == 0 ? [] : result['data'][query];
   }
 
+  public async lookupUsingCompositeKey(
+    query: string,
+    compositeKeyValues: any,
+    fields: string[]
+  ) {
+    const where: string[] = [];
+    const params: string[] = [];
+    for (const [key, val] of Object.entries(compositeKeyValues)) {
+      assert.strictEqual(
+        typeof val != 'undefined' && val != null,
+        true,
+        `Invalid key ${key}`
+      );
+      where.push(`${key} : $${key}`);
+      params.push(`$${key}: String`);
+    }
+
+    logger.debug(
+      '[lookupUsingCompositeKey] : %s :: %j',
+      query,
+      compositeKeyValues
+    );
+
+    const queryString = `query(${params.join(', ')}) {
+      ${query}(where: { ${where.join(', ')} }) {
+        id, ${Object.keys(compositeKeyValues).join(', ')}, ${fields.join(',')}
+      }
+    }`;
+    logger.debug('[lookupUsingCompositeKey] %s', queryString);
+    const result = await this.context.executeGraphQL({
+      query: queryString,
+      variables: compositeKeyValues,
+    });
+    logger.debug(
+      '[lookupUsingCompositeKey] RESULT %j with vars %j',
+      result,
+      compositeKeyValues
+    );
+    if (result['data'][query] == null || result['data'][query].length > 1) {
+      throw Error(
+        'Expecting zero or one rows ' + query + ' ' + compositeKeyValues
+      );
+    }
+    return result['data'][query].length == 0 ? null : result['data'][query][0];
+  }
+
   public async lookup(
     query: string,
     refKey: string,

src/services/workflow/delete-environment.ts

@@ -71,14 +71,21 @@ export const DeleteEnvironment = async (
     prodEnvId
   );
 
+  // no longer doing a cascade delete of service access / consumer data
+  assert.strictEqual(
+    force,
+    false,
+    'Force delete environment no longer supported'
+  );
+
   const envDetail = await lookupEnvironmentAndIssuerById(context, prodEnvId);
 
   const accessList = await lookupServiceAccessesByEnvironment(context, ns, [
     prodEnvId,
   ]);
 
   assert.strictEqual(
-    force == true || accessList.length == 0,
+    accessList.length == 0,
     true,
     `${accessList.length} ${
       accessList.length == 1 ? 'consumer has' : 'consumers have'
@@ -102,21 +109,21 @@ export const CascadeDeleteEnvironment = async (
   ns: string,
   prodEnvId: string
 ): Promise<void> => {
-  await deleteRecords(
-    context,
-    'ServiceAccess',
-    { productEnvironment: { id: prodEnvId } },
-    true,
-    ['id']
-  );
-
-  await deleteRecords(
-    context,
-    'AccessRequest',
-    { productEnvironment: { id: prodEnvId } },
-    true,
-    ['id']
-  );
+  // await deleteRecords(
+  //   context,
+  //   'ServiceAccess',
+  //   { productEnvironment: { id: prodEnvId } },
+  //   true,
+  //   ['id']
+  // );
+
+  // await deleteRecords(
+  //   context,
+  //   'AccessRequest',
+  //   { productEnvironment: { id: prodEnvId } },
+  //   true,
+  //   ['id']
+  // );
 
   await deleteRecords(context, 'Environment', { id: prodEnvId }, false, ['id']);
 };