From e00d6c7736009b8e100df988baefcb434fdded5a Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 7 Nov 2024 18:50:27 -0500 Subject: [PATCH 1/6] Add support for task outputs save and passing Signed-off-by: Peter Zhu --- src/call/add-issue-to-github-project-v2.ts | 6 ++++++ src/config/operation-config.ts | 4 +++- src/service/service.ts | 17 +++++++++++------ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/call/add-issue-to-github-project-v2.ts b/src/call/add-issue-to-github-project-v2.ts index 42d6c4d..447544e 100644 --- a/src/call/add-issue-to-github-project-v2.ts +++ b/src/call/add-issue-to-github-project-v2.ts @@ -49,6 +49,12 @@ export default async function addIssueToGitHubProjectV2( if (!(await validateResourceConfig(app, context, resource))) return null; if (!(await validateProjects(app, resource, projects))) return null; + // Verify triggered event + if (!context.payload.label) { + app.log.error('Only \'issues.labeled\' event is supported on this call.') + return null; + } + // Verify triggered label const label = context.payload.label.name.trim(); if (!labels.includes(label)) { diff --git a/src/config/operation-config.ts b/src/config/operation-config.ts index bc67d63..b98c3ea 100644 --- a/src/config/operation-config.ts +++ b/src/config/operation-config.ts @@ -67,9 +67,11 @@ export class OperationConfig extends Config { } private static async _initTasks(taskDataArray: TaskData[]): Promise { + let taskCounter: number = 0; const taskObjArray = taskDataArray.map((taskData) => { + taskCounter++; const taskObj = new Task(taskData.call, taskData.args, taskData.name); - console.log(`Setup Task: ${taskObj.name}`); + console.log(`Setup Task ${taskCounter}: ${taskObj.name}`); return taskObj; }); return taskObjArray; diff --git a/src/service/service.ts b/src/service/service.ts index 237787c..0f6a2bc 100644 --- a/src/service/service.ts +++ b/src/service/service.ts @@ -26,6 +26,9 @@ export class Service { private _app: Probot; + // Map> + private _outputs: Map>; + constructor(name: string) { this._name = name; } @@ -55,6 +58,7 @@ export class Service { this._resource = await resConfigObj.initResource(); const opConfigObj = new OperationConfig(operationConfigPath); this._operation = await opConfigObj.initOperation(); + this._outputs = new Map(); this._registerEvents(); } @@ -63,8 +67,7 @@ export class Service { await promise; // Make sure tasks are completed in sequential orders const callPath = await realpath(`./bin/call/${task.callName}.js`); - const { callFunc } = task; - const { callArgs } = task; + const { name, callFunc, callArgs } = task; console.log(`[${event}]: Verify call lib: ${callPath}`); try { @@ -76,7 +79,9 @@ export class Service { const callStack = await import(callPath); if (callFunc === 'default') { console.log(`[${event}]: Call default function: [${callStack.default.name}]`); - await callStack.default(this.app, context, this.resource, { ...callArgs }); + const resultDefault = await callStack.default(this.app, context, this.resource, { ...callArgs }); + this._outputs.get(event)?.set(name, resultDefault); + console.log(this._outputs.get(event)); } else { console.log(callStack); const callFuncCustom = callStack[callFunc]; @@ -84,14 +89,13 @@ export class Service { if (!(typeof callFuncCustom === 'function')) { throw new Error(`[${event}]: ${callFuncCustom} is not a function, please verify in ${callPath}`); } - await callFuncCustom(this.app, context, this.resource, { ...callArgs }); + this._outputs.set(name, await callFuncCustom(this.app, context, this.resource, { ...callArgs })); } }, Promise.resolve()); } private async _registerEvents(): Promise { - const { events } = this.operation; - const { tasks } = this.operation; + const { events, tasks } = this.operation; console.log(`Evaluate events: [${events}]`); if (!events) { throw new Error('No events defined in the operation!'); @@ -102,6 +106,7 @@ export class Service { events.forEach((event) => { console.log(`Register event: "${event}"`); + this._outputs.set(event, new Map()); if (event === 'all') { console.warn('WARNING! All events will be listened based on the config!'); this._app.onAny(async (context) => { From d2c93f0ac3e28877a9d2bb445514ca18549e2f5a Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 19 Nov 2024 16:20:21 -0500 Subject: [PATCH 2/6] Switch name postfix to task order Signed-off-by: Peter Zhu --- src/config/operation-config.ts | 1 + src/service/operation/task.ts | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/config/operation-config.ts b/src/config/operation-config.ts index b98c3ea..bb37ff3 100644 --- a/src/config/operation-config.ts +++ b/src/config/operation-config.ts @@ -71,6 +71,7 @@ export class OperationConfig extends Config { const taskObjArray = taskDataArray.map((taskData) => { taskCounter++; const taskObj = new Task(taskData.call, taskData.args, taskData.name); + taskObj.name += '#' + taskCounter; console.log(`Setup Task ${taskCounter}: ${taskObj.name}`); return taskObj; }); diff --git a/src/service/operation/task.ts b/src/service/operation/task.ts index 45f8b29..40166aa 100644 --- a/src/service/operation/task.ts +++ b/src/service/operation/task.ts @@ -7,11 +7,10 @@ * compatible open source license. */ -import randomstring from 'randomstring'; import { TaskArgData } from '../../config/types'; export class Task { - private readonly _name: string; // uid + private _name: string; // uid private readonly _callName: string; @@ -23,8 +22,14 @@ export class Task { const callArray = call.trim().split('@'); [this._callName, this._callFunc] = callArray; this._callArgs = callArgs; - const namePostfix = randomstring.generate(8); - this._name = name ? `${name}#${namePostfix}` : `${this.callName}#${namePostfix}`; + this._name = name || this.callName; + } + + public set name(taskName: string) { + if (!taskName || taskName.trim() === '') { + throw new Error('Task Name input cannot be empty'); + } + this._name = taskName; } public get name(): string { From c133455ca500d38d455e93bc34f7732c0985f9ef Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 22 Nov 2024 17:58:02 -0500 Subject: [PATCH 3/6] Add initial outputs passing from one to another task Signed-off-by: Peter Zhu --- src/service/service.ts | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/service/service.ts b/src/service/service.ts index 0f6a2bc..3792a0c 100644 --- a/src/service/service.ts +++ b/src/service/service.ts @@ -29,6 +29,8 @@ export class Service { // Map> private _outputs: Map>; + private readonly subPattern = /\$\{\{\s*(.*?)\s*\}\}/; + constructor(name: string) { this._name = name; } @@ -79,6 +81,27 @@ export class Service { const callStack = await import(callPath); if (callFunc === 'default') { console.log(`[${event}]: Call default function: [${callStack.default.name}]`); + if (callArgs) { + console.log(`[${event}]: Call with args:`); + for (const name2 in callArgs) { + console.log(`[${event}]: args: ${name2}: ${callArgs[name2]}`); + if (Array.isArray(callArgs[name2])) { + for (let i = 0; i < callArgs[name2].length; i++) { + if (this.subPattern.test(callArgs[name2][i])) { + console.log(`Array: ${callArgs[name2][i]}`); + } + } + } else { + const match = (callArgs[name2] as string).match(this.subPattern); + if (match) { + console.log(`StrSub: ${callArgs[name2]}, Match: ${match[1]}, ${match[1].replace('outputs.', '')}`); + console.log(this._outputs.get(event)?.get(match[1].replace('outputs.', ''))) + callArgs[name2] = JSON.stringify(Object.fromEntries(this._outputs.get(event)?.get(match[1].replace('outputs.', '')))) + } + } + } + } + const resultDefault = await callStack.default(this.app, context, this.resource, { ...callArgs }); this._outputs.get(event)?.set(name, resultDefault); console.log(this._outputs.get(event)); @@ -89,7 +112,7 @@ export class Service { if (!(typeof callFuncCustom === 'function')) { throw new Error(`[${event}]: ${callFuncCustom} is not a function, please verify in ${callPath}`); } - this._outputs.set(name, await callFuncCustom(this.app, context, this.resource, { ...callArgs })); + this._outputs.get(event)?.set(name, await callFuncCustom(this.app, context, this.resource, { ...callArgs })); } }, Promise.resolve()); } From 10cc59e08b7b4b125c2e2ece43185e943bdaa7bf Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 25 Nov 2024 16:57:56 -0500 Subject: [PATCH 4/6] Allow outputs to substitute without changing initial values Signed-off-by: Peter Zhu --- package-lock.json | 4 +- package.json | 2 +- src/call/add-issue-to-github-project-v2.ts | 9 ++-- src/service/service.ts | 54 +++++++++++++--------- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/package-lock.json b/package-lock.json index c6b44a9..0b761d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "opensearch-automation-app", - "version": "0.1.18", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "opensearch-automation-app", - "version": "0.1.18", + "version": "0.2.0", "dependencies": { "@aws-sdk/client-cloudwatch": "^3.664.0", "@aws-sdk/client-opensearch": "^3.658.1", diff --git a/package.json b/package.json index bb5a82f..d32a765 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "opensearch-automation-app", - "version": "0.1.18", + "version": "0.2.0", "description": "An Automation App that handles all your GitHub Repository Activities", "author": "Peter Zhu", "homepage": "https://github.com/opensearch-project/automation-app", diff --git a/src/call/add-issue-to-github-project-v2.ts b/src/call/add-issue-to-github-project-v2.ts index 447544e..84b62b1 100644 --- a/src/call/add-issue-to-github-project-v2.ts +++ b/src/call/add-issue-to-github-project-v2.ts @@ -45,7 +45,7 @@ export default async function addIssueToGitHubProjectV2( context: any, resource: Resource, { labels, projects }: AddIssueToGitHubProjectV2Params, -): Promise | null> { +): Promise { if (!(await validateResourceConfig(app, context, resource))) return null; if (!(await validateProjects(app, resource, projects))) return null; @@ -66,7 +66,7 @@ export default async function addIssueToGitHubProjectV2( const repoName = context.payload.repository.name; const issueNumber = context.payload.issue.number; const issueNodeId = context.payload.issue.node_id; - const itemIdMap = new Map(); + let itemId = null // Add to project try { @@ -91,8 +91,7 @@ export default async function addIssueToGitHubProjectV2( `; const responseAddToProject = await context.octokit.graphql(addToProjectMutation); app.log.info(responseAddToProject); - const itemId = responseAddToProject.addProjectV2ItemById.item.id; - itemIdMap.set(project, [itemId, label]); + itemId = responseAddToProject.addProjectV2ItemById.item.id; }), ); } catch (e) { @@ -100,5 +99,5 @@ export default async function addIssueToGitHubProjectV2( return null; } - return itemIdMap; + return itemId; } diff --git a/src/service/service.ts b/src/service/service.ts index 3792a0c..8358d65 100644 --- a/src/service/service.ts +++ b/src/service/service.ts @@ -15,6 +15,7 @@ import { Operation } from './operation/operation'; import { Task } from './operation/task'; import { ResourceConfig } from '../config/resource-config'; import { OperationConfig } from '../config/operation-config'; +import { TaskArgData } from '../config/types'; import { octokitAuth } from '../utility/probot/octokit'; export class Service { @@ -78,31 +79,13 @@ export class Service { console.error(`ERROR: ${e}`); } + const callStack = await import(callPath); + const callArgsSub = await this._outputsSubstitution({ ...callArgs }, event); + if (callFunc === 'default') { console.log(`[${event}]: Call default function: [${callStack.default.name}]`); - if (callArgs) { - console.log(`[${event}]: Call with args:`); - for (const name2 in callArgs) { - console.log(`[${event}]: args: ${name2}: ${callArgs[name2]}`); - if (Array.isArray(callArgs[name2])) { - for (let i = 0; i < callArgs[name2].length; i++) { - if (this.subPattern.test(callArgs[name2][i])) { - console.log(`Array: ${callArgs[name2][i]}`); - } - } - } else { - const match = (callArgs[name2] as string).match(this.subPattern); - if (match) { - console.log(`StrSub: ${callArgs[name2]}, Match: ${match[1]}, ${match[1].replace('outputs.', '')}`); - console.log(this._outputs.get(event)?.get(match[1].replace('outputs.', ''))) - callArgs[name2] = JSON.stringify(Object.fromEntries(this._outputs.get(event)?.get(match[1].replace('outputs.', '')))) - } - } - } - } - - const resultDefault = await callStack.default(this.app, context, this.resource, { ...callArgs }); + const resultDefault = await callStack.default(this.app, context, this.resource, { ...callArgsSub }); this._outputs.get(event)?.set(name, resultDefault); console.log(this._outputs.get(event)); } else { @@ -112,11 +95,36 @@ export class Service { if (!(typeof callFuncCustom === 'function')) { throw new Error(`[${event}]: ${callFuncCustom} is not a function, please verify in ${callPath}`); } - this._outputs.get(event)?.set(name, await callFuncCustom(this.app, context, this.resource, { ...callArgs })); + this._outputs.get(event)?.set(name, await callFuncCustom(this.app, context, this.resource, { ...callArgsSub })); } }, Promise.resolve()); } + private async _outputsSubstitution(callArgsData: TaskArgData, event: string): Promise { + console.log(`[${event}]: Call with args:`); + for (const argName in callArgsData) { + console.log(`[${event}]: args: ${argName}: ${callArgsData[argName]}`); + if (Array.isArray(callArgsData[argName])) { + for (let i = 0; i < callArgsData[argName].length; i++) { + (callArgsData[argName] as string[])[i] = await this._matchSubPattern((callArgsData[argName][i] as string), event); + } + } else { + (callArgsData[argName] as string) = await this._matchSubPattern((callArgsData[argName] as string), event); + } + } + return callArgsData; + } + + private async _matchSubPattern(callArgsValue: string, event: string): Promise { + const match = callArgsValue.match(this.subPattern); + if (match) { + const outputMatch = this._outputs.get(event)?.get(match[1].replace('outputs.', '')); + console.log(`StrSub: ${callArgsValue}, Match: ${match[1]}, Output: ${outputMatch}`); + return outputMatch; + } + return callArgsValue; + } + private async _registerEvents(): Promise { const { events, tasks } = this.operation; console.log(`Evaluate events: [${events}]`); From a73009fcdfadd7e4cf00ab011f713e02e31cadc7 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 2 Dec 2024 13:33:32 -0500 Subject: [PATCH 5/6] Tweak formats Signed-off-by: Peter Zhu --- src/call/add-issue-to-github-project-v2.ts | 4 ++-- src/service/service.ts | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/call/add-issue-to-github-project-v2.ts b/src/call/add-issue-to-github-project-v2.ts index 84b62b1..ab903e5 100644 --- a/src/call/add-issue-to-github-project-v2.ts +++ b/src/call/add-issue-to-github-project-v2.ts @@ -51,7 +51,7 @@ export default async function addIssueToGitHubProjectV2( // Verify triggered event if (!context.payload.label) { - app.log.error('Only \'issues.labeled\' event is supported on this call.') + app.log.error("Only 'issues.labeled' event is supported on this call."); return null; } @@ -66,7 +66,7 @@ export default async function addIssueToGitHubProjectV2( const repoName = context.payload.repository.name; const issueNumber = context.payload.issue.number; const issueNodeId = context.payload.issue.node_id; - let itemId = null + let itemId = null; // Add to project try { diff --git a/src/service/service.ts b/src/service/service.ts index 8358d65..69f0120 100644 --- a/src/service/service.ts +++ b/src/service/service.ts @@ -79,7 +79,6 @@ export class Service { console.error(`ERROR: ${e}`); } - const callStack = await import(callPath); const callArgsSub = await this._outputsSubstitution({ ...callArgs }, event); @@ -106,10 +105,10 @@ export class Service { console.log(`[${event}]: args: ${argName}: ${callArgsData[argName]}`); if (Array.isArray(callArgsData[argName])) { for (let i = 0; i < callArgsData[argName].length; i++) { - (callArgsData[argName] as string[])[i] = await this._matchSubPattern((callArgsData[argName][i] as string), event); + (callArgsData[argName] as string[])[i] = await this._matchSubPattern(callArgsData[argName][i] as string, event); } } else { - (callArgsData[argName] as string) = await this._matchSubPattern((callArgsData[argName] as string), event); + (callArgsData[argName] as string) = await this._matchSubPattern(callArgsData[argName] as string, event); } } return callArgsData; From be183d646f569a4d6a48945e2d4563bd34842cbc Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 2 Dec 2024 15:49:15 -0500 Subject: [PATCH 6/6] Add test cases Signed-off-by: Peter Zhu --- src/config/operation-config.ts | 4 +-- src/service/service.ts | 35 ++++++++++++------- .../add-issue-to-github-project-v2.test.ts | 11 +++++- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/config/operation-config.ts b/src/config/operation-config.ts index bb37ff3..d8addbf 100644 --- a/src/config/operation-config.ts +++ b/src/config/operation-config.ts @@ -69,9 +69,9 @@ export class OperationConfig extends Config { private static async _initTasks(taskDataArray: TaskData[]): Promise { let taskCounter: number = 0; const taskObjArray = taskDataArray.map((taskData) => { - taskCounter++; + taskCounter += 1; const taskObj = new Task(taskData.call, taskData.args, taskData.name); - taskObj.name += '#' + taskCounter; + taskObj.name += `#${taskCounter}`; console.log(`Setup Task ${taskCounter}: ${taskObj.name}`); return taskObj; }); diff --git a/src/service/service.ts b/src/service/service.ts index 69f0120..b456c92 100644 --- a/src/service/service.ts +++ b/src/service/service.ts @@ -72,7 +72,8 @@ export class Service { const callPath = await realpath(`./bin/call/${task.callName}.js`); const { name, callFunc, callArgs } = task; - console.log(`[${event}]: Verify call lib: ${callPath}`); + console.log(`[${event}]: Start call now: ${name}`); + console.log(`[${event}]: Check call lib: ${callPath}`); try { await access(callPath); } catch (e) { @@ -99,24 +100,34 @@ export class Service { }, Promise.resolve()); } - private async _outputsSubstitution(callArgsData: TaskArgData, event: string): Promise { + private async _outputsSubstitution(callArgs: TaskArgData, event: string): Promise { console.log(`[${event}]: Call with args:`); - for (const argName in callArgsData) { - console.log(`[${event}]: args: ${argName}: ${callArgsData[argName]}`); - if (Array.isArray(callArgsData[argName])) { - for (let i = 0; i < callArgsData[argName].length; i++) { - (callArgsData[argName] as string[])[i] = await this._matchSubPattern(callArgsData[argName][i] as string, event); + + const callArgsTemp = callArgs; + const argEntries = Object.entries(callArgsTemp); + + await Promise.all( + argEntries.map(async ([argName, argValue]) => { + console.log(`[${event}]: args: ${argName}: ${argValue}`); + + // Overwrite callArgsTemp if user choose to substitute value with outputs from previous task ${{ outputs.# }} + if (Array.isArray(argValue)) { + // string[] + callArgsTemp[argName] = await Promise.all(argValue.map(async (argValueItem) => this._matchSubPattern(argValueItem as string, event))); + } else { + // string + callArgsTemp[argName] = await this._matchSubPattern(argValue as string, event); } - } else { - (callArgsData[argName] as string) = await this._matchSubPattern(callArgsData[argName] as string, event); - } - } - return callArgsData; + }), + ); + return callArgsTemp; } private async _matchSubPattern(callArgsValue: string, event: string): Promise { const match = callArgsValue.match(this.subPattern); if (match) { + // If user substitution pattern ${{ outputs.# }} found in operation config + // Return substituion value based on return value saved in outputs const outputMatch = this._outputs.get(event)?.get(match[1].replace('outputs.', '')); console.log(`StrSub: ${callArgsValue}, Match: ${match[1]}, Output: ${outputMatch}`); return outputMatch; diff --git a/test/call/add-issue-to-github-project-v2.test.ts b/test/call/add-issue-to-github-project-v2.test.ts index 7620867..6034c0b 100644 --- a/test/call/add-issue-to-github-project-v2.test.ts +++ b/test/call/add-issue-to-github-project-v2.test.ts @@ -80,6 +80,15 @@ describe('addIssueToGitHubProjectV2Functions', () => { }); describe('addIssueToGitHubProjectV2', () => { + it('should print error and return null if it is not a issues.labeled event', async () => { + context.payload.label = undefined; + + const result = await addIssueToGitHubProjectV2(app, context, resource, params); + + expect(app.log.error).toHaveBeenCalledWith("Only 'issues.labeled' event is supported on this call."); + expect(result).toBe(null); + }); + it('should print error if context label does not match the ones in resource config', async () => { context.payload.label.name = 'enhancement'; @@ -116,7 +125,7 @@ describe('addIssueToGitHubProjectV2Functions', () => { /* prettier-ignore-end */ expect(context.octokit.graphql).toHaveBeenCalledWith(graphQLCallStack); - expect(JSON.stringify((result as Map).get('test-org/222'))).toBe('["new-item-id","Meta"]'); + expect(result).toBe('new-item-id'); expect(app.log.info).toHaveBeenCalledWith(graphQLResponse); });