From cb1646e99257eff785983ee899c2d385059c5af7 Mon Sep 17 00:00:00 2001 From: ikethecoder Date: Mon, 5 Feb 2024 23:23:08 -0800 Subject: [PATCH] improve batch loader of products and environments --- src/batch/data-rules.js | 3 + src/batch/feed-worker.ts | 63 ++++++++++++++------- src/batch/transformations/connectMany.ts | 21 +++++-- src/batch/transformations/connectOne.ts | 21 +++++-- src/lists/Product.js | 14 ++--- src/services/keystone/batch-service.ts | 46 +++++++++++++++ src/services/workflow/delete-environment.ts | 39 +++++++------ src/services/workflow/delete-namespace.ts | 20 ++++--- src/services/workflow/delete-product.ts | 26 ++++----- src/test/integrated/batchworker/product.ts | 59 +++++++++++++++++++ 10 files changed, 236 insertions(+), 76 deletions(-) create mode 100644 src/test/integrated/batchworker/product.ts diff --git a/src/batch/data-rules.js b/src/batch/data-rules.js index 241e38969..fab034420 100644 --- a/src/batch/data-rules.js +++ b/src/batch/data-rules.js @@ -376,6 +376,7 @@ const metadata = { Product: { query: 'allProducts', refKey: 'appId', + compositeRefKey: ['name', 'namespace'], sync: ['name', 'namespace'], transformations: { dataset: { name: 'connectOne', list: 'allDatasets', refKey: 'name' }, @@ -410,12 +411,14 @@ const metadata = { name: 'connectMany', list: 'allGatewayServices', refKey: 'name', + filterByNamespace: true, }, legal: { name: 'connectOne', list: 'allLegals', refKey: 'reference' }, credentialIssuer: { name: 'connectOne', list: 'allCredentialIssuers', refKey: 'name', + filterByNamespace: true, }, }, validations: { diff --git a/src/batch/feed-worker.ts b/src/batch/feed-worker.ts index b2e368a3c..d3309fc79 100644 --- a/src/batch/feed-worker.ts +++ b/src/batch/feed-worker.ts @@ -55,6 +55,7 @@ const transformations = { export const putFeedWorker = async (context: any, req: any, res: any) => { const entity = req.params['entity']; assert.strictEqual(entity in metadata, true); + logger.info('putFeedWorker %s', entity); const md = metadata[entity]; const refKey = md.refKey; @@ -164,7 +165,8 @@ export const getFeedWorker = async (context: any, req: any, res: any) => { const syncListOfRecords = async function ( keystone: any, transformInfo: any, - records: any + records: any, + parentRecord?: any ): Promise { const result: BatchResult[] = []; if (records == null || typeof records == 'undefined') { @@ -179,7 +181,8 @@ const syncListOfRecords = async function ( transformInfo.list, record[recordKey], record, - true + true, + parentRecord ) ); } @@ -245,7 +248,7 @@ function buildQueryResponse(md: any, children: string[] = undefined): string[] { }); } if ('ownedBy' in md) { - response.push(`${md.ownedBy} { id }`); + response.push(`${md.ownedBy} { id, namespace }`); } logger.debug('[buildQueryResponse] FINAL (%s) %j', md.query, response); @@ -307,7 +310,8 @@ export const syncRecords = async function ( feedEntity: string, eid: string, json: any, - children = false + children = false, + parentRecord: any = undefined ): Promise { const md = (metadata as any)[feedEntity]; const entity = 'entity' in md ? md['entity'] : feedEntity; @@ -318,11 +322,20 @@ export const syncRecords = async function ( 'This entity is only part of a child.' ); - assert.strictEqual( - typeof eid === 'string' && eid.length > 0, - true, - `Invalid ID for ${feedEntity} ${eid}` - ); + const compositeKeyValues: any = {}; + if (md.compositeRefKey) { + md.compositeRefKey.forEach((key: string) => { + compositeKeyValues[key] = json[key]; + }); + } else { + assert.strictEqual( + typeof eid === 'string' && eid.length > 0, + true, + `Invalid ID for ${feedEntity} ${md.refKey} = ${eid || 'blank'}` + ); + + compositeKeyValues[md.refKey] = eid; + } const batchService = new BatchService(context); @@ -342,10 +355,9 @@ export const syncRecords = async function ( let childResults: BatchResult[] = []; - const localRecord = await batchService.lookup( + const localRecord = await batchService.lookupUsingCompositeKey( md.query, - md.refKey, - eid, + compositeKeyValues, buildQueryResponse(md) ); if (localRecord == null) { @@ -365,9 +377,12 @@ export const syncRecords = async function ( const allIds = await syncListOfRecords( context, transformInfo, - json[transformKey] + json[transformKey], + json ); logger.debug('CHILDREN [%s] %j', transformKey, allIds); + childResults.push(...allIds); + assert.strictEqual( allIds.filter((record) => record.status != 200).length, 0, @@ -380,8 +395,9 @@ export const syncRecords = async function ( 'There are some child records that have exclusive ownership already!' ); json[transformKey + '_ids'] = allIds.map((status) => status.id); - - childResults.push(...allIds); + } + if (transformInfo.filterByNamespace) { + json['_namespace'] = parentRecord['namespace']; } const transformMutation = await transformations[transformInfo.name]( context, @@ -403,7 +419,9 @@ export const syncRecords = async function ( } } } - data[md.refKey] = eid; + if (eid) { + data[md.refKey] = eid; + } const nr = await batchService.create(entity, data); if (nr == null) { logger.error('CREATE FAILED (%s) %j', nr, data); @@ -452,9 +470,13 @@ export const syncRecords = async function ( const allIds = await syncListOfRecords( context, transformInfo, - json[transformKey] + json[transformKey], + json ); + logger.debug('CHILDREN [%s] %j', transformKey, allIds); + childResults.push(...allIds); + assert.strictEqual( allIds.filter((record) => record.status != 200).length, 0, @@ -468,13 +490,14 @@ export const syncRecords = async function ( record.ownedBy != localRecord.id ).length, 0, - 'There are some child records that had ownership already (w/ local record)!' + 'There are some child records that have ownership already (update not allowed)!' ); json[transformKey + '_ids'] = allIds.map((status) => status.id); - childResults.push(...allIds); } - + if (transformInfo.filterByNamespace) { + json['_namespace'] = parentRecord['namespace']; + } const transformMutation = await transformations[transformInfo.name]( context, transformInfo, diff --git a/src/batch/transformations/connectMany.ts b/src/batch/transformations/connectMany.ts index f26dde80a..c0022e952 100644 --- a/src/batch/transformations/connectMany.ts +++ b/src/batch/transformations/connectMany.ts @@ -17,12 +17,21 @@ export async function connectMany( const batchService = new BatchService(keystone); if (idList != null) { for (const uniqueKey of idList) { - const lkup = await batchService.lookup( - transformInfo['list'], - transformInfo['refKey'], - uniqueKey, - [] - ); + const nsFilter = { namespace: inputData['_namespace'] } as any; + nsFilter[transformInfo['refKey']] = uniqueKey; + logger.error('T = %s -- %j %j', uniqueKey, inputData, currentData); + const lkup = transformInfo['filterByNamespace'] + ? await batchService.lookupUsingCompositeKey( + transformInfo['list'], + nsFilter, + [] + ) + : await batchService.lookup( + transformInfo['list'], + transformInfo['refKey'], + uniqueKey, + [] + ); if (lkup == null) { logger.error( `Lookup failed for ${transformInfo['list']} ${transformInfo['refKey']}!` diff --git a/src/batch/transformations/connectOne.ts b/src/batch/transformations/connectOne.ts index ca95335e0..bb9aff0c9 100644 --- a/src/batch/transformations/connectOne.ts +++ b/src/batch/transformations/connectOne.ts @@ -29,12 +29,21 @@ export async function connectOne( } } - const lkup = await batchService.lookup( - transformInfo['list'], - transformInfo['refKey'], - value, - [] - ); + const nsFilter = { namespace: inputData['_namespace'] } as any; + nsFilter[transformInfo['refKey']] = value; + const lkup = transformInfo['filterByNamespace'] + ? await batchService.lookupUsingCompositeKey( + transformInfo['list'], + nsFilter, + [] + ) + : await batchService.lookup( + transformInfo['list'], + transformInfo['refKey'], + value, + [] + ); + if (lkup == null) { logger.error( `Lookup failed for ${transformInfo['list']} ${transformInfo['refKey']}!` diff --git a/src/lists/Product.js b/src/lists/Product.js index 06d74d686..d19b745b1 100644 --- a/src/lists/Product.js +++ b/src/lists/Product.js @@ -98,12 +98,12 @@ module.exports = { ); }, - // beforeDelete: async function ({ existingItem, context }) { - // await DeleteProductEnvironments( - // context, - // context.authedItem['namespace'], - // existingItem.id - // ); - // }, + beforeDelete: async function ({ existingItem, context }) { + await DeleteProductEnvironments( + context.createContext({ skipAccessControl: true }), + context.authedItem['namespace'], + existingItem.id + ); + }, }, }; diff --git a/src/services/keystone/batch-service.ts b/src/services/keystone/batch-service.ts index 3ed6c076e..8ee36f31f 100644 --- a/src/services/keystone/batch-service.ts +++ b/src/services/keystone/batch-service.ts @@ -96,6 +96,52 @@ export class BatchService { return result['data'][query].length == 0 ? [] : result['data'][query]; } + public async lookupUsingCompositeKey( + query: string, + compositeKeyValues: any, + fields: string[] + ) { + const where: string[] = []; + const params: string[] = []; + for (const [key, val] of Object.entries(compositeKeyValues)) { + assert.strictEqual( + typeof val != 'undefined' && val != null, + true, + `Invalid key ${key}` + ); + where.push(`${key} : $${key}`); + params.push(`$${key}: String`); + } + + logger.debug( + '[lookupUsingCompositeKey] : %s :: %j', + query, + compositeKeyValues + ); + + const queryString = `query(${params.join(', ')}) { + ${query}(where: { ${where.join(', ')} }) { + id, ${Object.keys(compositeKeyValues).join(', ')}, ${fields.join(',')} + } + }`; + logger.debug('[lookupUsingCompositeKey] %s', queryString); + const result = await this.context.executeGraphQL({ + query: queryString, + variables: compositeKeyValues, + }); + logger.debug( + '[lookupUsingCompositeKey] RESULT %j with vars %j', + result, + compositeKeyValues + ); + if (result['data'][query] == null || result['data'][query].length > 1) { + throw Error( + 'Expecting zero or one rows ' + query + ' ' + compositeKeyValues + ); + } + return result['data'][query].length == 0 ? null : result['data'][query][0]; + } + public async lookup( query: string, refKey: string, diff --git a/src/services/workflow/delete-environment.ts b/src/services/workflow/delete-environment.ts index cb1f87ad2..1292700c9 100644 --- a/src/services/workflow/delete-environment.ts +++ b/src/services/workflow/delete-environment.ts @@ -71,6 +71,13 @@ export const DeleteEnvironment = async ( prodEnvId ); + // no longer doing a cascade delete of service access / consumer data + assert.strictEqual( + force, + false, + 'Force delete environment no longer supported' + ); + const envDetail = await lookupEnvironmentAndIssuerById(context, prodEnvId); const accessList = await lookupServiceAccessesByEnvironment(context, ns, [ @@ -78,7 +85,7 @@ export const DeleteEnvironment = async ( ]); assert.strictEqual( - force == true || accessList.length == 0, + accessList.length == 0, true, `${accessList.length} ${ accessList.length == 1 ? 'consumer has' : 'consumers have' @@ -102,21 +109,21 @@ export const CascadeDeleteEnvironment = async ( ns: string, prodEnvId: string ): Promise => { - await deleteRecords( - context, - 'ServiceAccess', - { productEnvironment: { id: prodEnvId } }, - true, - ['id'] - ); - - await deleteRecords( - context, - 'AccessRequest', - { productEnvironment: { id: prodEnvId } }, - true, - ['id'] - ); + // await deleteRecords( + // context, + // 'ServiceAccess', + // { productEnvironment: { id: prodEnvId } }, + // true, + // ['id'] + // ); + + // await deleteRecords( + // context, + // 'AccessRequest', + // { productEnvironment: { id: prodEnvId } }, + // true, + // ['id'] + // ); await deleteRecords(context, 'Environment', { id: prodEnvId }, false, ['id']); }; diff --git a/src/services/workflow/delete-namespace.ts b/src/services/workflow/delete-namespace.ts index 4abd6063a..2fa4151d5 100644 --- a/src/services/workflow/delete-namespace.ts +++ b/src/services/workflow/delete-namespace.ts @@ -25,7 +25,7 @@ import { Environment } from '../keystone/types'; import { lookupEnvironmentsByNS } from '../keystone/product-environment'; import { FieldErrors } from 'tsoa'; import { updateActivity } from '../keystone/activity'; -import { CascadeDeleteEnvironment } from './delete-environment'; +//import { CascadeDeleteEnvironment } from './delete-environment'; import { GWAService } from '../gwaapi'; import getSubjectToken from '../../auth/auth-token'; @@ -128,13 +128,17 @@ export const DeleteNamespace = async ( const envs = await lookupEnvironmentsByNS(context, ns); const ids = envs.map((e: Environment) => e.id); - for (const envId of ids) { - await CascadeDeleteEnvironment(context, ns, envId); - } - - await deleteRecords(context, 'ServiceAccess', { namespace: ns }, true, [ - 'id', - ]); + // "DeleteNamespaceValidate" is called prior to this one, so + // it won't reach here if there are Service Access records + // but to be extra safe, lets keep this code + // + // for (const envId of ids) { + // await CascadeDeleteEnvironment(context, ns, envId); + // } + + // await deleteRecords(context, 'ServiceAccess', { namespace: ns }, true, [ + // 'id', + // ]); await deleteRecords(context, 'Product', { namespace: ns }, true, ['id']); diff --git a/src/services/workflow/delete-product.ts b/src/services/workflow/delete-product.ts index d39525e34..ab1a179f7 100644 --- a/src/services/workflow/delete-product.ts +++ b/src/services/workflow/delete-product.ts @@ -46,19 +46,19 @@ export const DeleteProductValidate = async ( ); }; -// export const DeleteProductEnvironments = async ( -// context: any, -// ns: string, -// id: string -// ) => { -// logger.debug('Deleting Product ns=%s, id=%s', ns, id); +export const DeleteProductEnvironments = async ( + context: any, + ns: string, + id: string +) => { + logger.debug('Deleting environments for ns=%s, product=%s', ns, id); -// const product = await lookupProduct(context, ns, id); -// logger.error('Product %j', product); + const product = await lookupProduct(context, ns, id); + logger.info('Deleting product environments %j', product); -// const ids = product.environments.map((e: Environment) => e.id); + const ids = product.environments.map((e: Environment) => e.id); -// for (const envId of ids) { -// await deleteRecords(context, 'Environment', { id: envId }, false, ['id']); -// } -// }; + for (const envId of ids) { + await deleteRecords(context, 'Environment', { id: envId }, false, ['id']); + } +}; diff --git a/src/test/integrated/batchworker/product.ts b/src/test/integrated/batchworker/product.ts new file mode 100644 index 000000000..eac8c069b --- /dev/null +++ b/src/test/integrated/batchworker/product.ts @@ -0,0 +1,59 @@ +/* +Wire up directly with Keycloak and use the Services +To run: +npm run ts-build +npm run ts-watch +node dist/test/integrated/batchworker/product.js +*/ + +import InitKeystone from '../keystonejs/init'; +import { + getRecords, + parseJsonString, + transformAllRefID, + removeEmpty, + removeKeys, + syncRecords, +} from '../../../batch/feed-worker'; +import { o } from '../util'; +import { BatchService } from '../../../services/keystone/batch-service'; + +(async () => { + const keystone = await InitKeystone(); + console.log('K = ' + keystone); + + const ns = 'refactortime'; + const skipAccessControl = false; + + const identity = { + id: null, + username: 'sample_username', + namespace: ns, + roles: JSON.stringify(['api-owner']), + scopes: [], + userId: null, + } as any; + + const ctx = keystone.createContext({ + skipAccessControl, + authentication: { item: identity }, + }); + + const json = { + name: 'Refactor Time Test', + namespace: ns, + environments: [ + { + name: 'stage', + appId: '0A021EB0', + //services: [] as any, + services: ['a-service-for-refactortime'], + // services: ['a-service-for-refactortime', 'a-service-for-aps-moh-proto'], + }, + ] as any, + }; + const res = await syncRecords(ctx, 'Product', null, json); + o(res); + + await keystone.disconnect(); +})();