From 6eebc5e7b148c193323bb936bb4d5b9df339d267 Mon Sep 17 00:00:00 2001 From: Brandon Shien Date: Wed, 5 Jun 2024 17:02:46 -0700 Subject: [PATCH] Added canary url and stepfunction monitoring with sns email and slack integration Signed-off-by: Brandon Shien --- .gitignore | 1 + build.gradle | 1 + .../canary/nodejs/node_modules/urlMonitor.js | 102 ++++++++ infrastructure/lib/constructs/canarySns.ts | 40 +++ infrastructure/lib/constructs/snsMonitor.ts | 64 +++++ .../lib/constructs/stepFunctionSns.ts | 45 ++++ infrastructure/lib/infrastructure-stack.ts | 17 +- infrastructure/lib/stacks/metricsWorkflow.ts | 9 + .../lib/stacks/monitoringDashboard.ts | 111 +++++++++ infrastructure/lib/stacks/secrets.ts | 14 ++ infrastructure/package-lock.json | 21 +- infrastructure/package.json | 2 +- infrastructure/test/monitoring-stack.test.ts | 234 ++++++++++++++++++ infrastructure/test/secrets-stack.test.ts | 14 ++ .../dagger/CommonModule.java | 14 ++ .../dagger/ServiceComponent.java | 3 + .../datasource/DataSourceType.java | 7 + .../opensearchmetrics/lambda/SlackLambda.java | 112 +++++++++ .../util/SecretsManagerUtil.java | 62 +++++ .../lambda/SlackLambdaTest.java | 146 +++++++++++ .../util/SecretsManagerUtilTest.java | 66 +++++ 21 files changed, 1072 insertions(+), 13 deletions(-) create mode 100644 infrastructure/canary/nodejs/node_modules/urlMonitor.js create mode 100644 infrastructure/lib/constructs/canarySns.ts create mode 100644 infrastructure/lib/constructs/snsMonitor.ts create mode 100644 infrastructure/lib/constructs/stepFunctionSns.ts create mode 100644 infrastructure/lib/stacks/monitoringDashboard.ts create mode 100644 infrastructure/lib/stacks/secrets.ts create mode 100644 infrastructure/test/monitoring-stack.test.ts create mode 100644 infrastructure/test/secrets-stack.test.ts create mode 100644 src/main/java/org/opensearchmetrics/datasource/DataSourceType.java create mode 100644 src/main/java/org/opensearchmetrics/lambda/SlackLambda.java create mode 100644 src/main/java/org/opensearchmetrics/util/SecretsManagerUtil.java create mode 100644 src/test/java/org/opensearchmetrics/lambda/SlackLambdaTest.java create mode 100644 src/test/java/org/opensearchmetrics/util/SecretsManagerUtilTest.java diff --git a/.gitignore b/.gitignore index b2692ee..11a2a20 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ infrastructure/**/*.js infrastructure/!jest.config.js infrastructure/**/*.d.ts infrastructure/node_modules +!infrastructure/canary/nodejs/node_modules/urlMonitor.js # CDK asset staging directory infrastructure/.cdk.staging diff --git a/build.gradle b/build.gradle index ca24666..f7faf76 100644 --- a/build.gradle +++ b/build.gradle @@ -28,6 +28,7 @@ dependencies { implementation 'io.github.acm19:aws-request-signing-apache-interceptor:2.3.1' implementation 'com.amazonaws:aws-lambda-java-core:1.2.3' + implementation 'com.amazonaws:aws-lambda-java-events:3.7.0' implementation 'com.google.code.gson:gson:2.10.1' diff --git a/infrastructure/canary/nodejs/node_modules/urlMonitor.js b/infrastructure/canary/nodejs/node_modules/urlMonitor.js new file mode 100644 index 0000000..607a6fa --- /dev/null +++ b/infrastructure/canary/nodejs/node_modules/urlMonitor.js @@ -0,0 +1,102 @@ +const { URL } = require('url'); +const synthetics = require('Synthetics'); +const log = require('SyntheticsLogger'); +const syntheticsConfiguration = synthetics.getConfiguration(); +const syntheticsLogHelper = require('SyntheticsLogHelper'); + +const loadBlueprint = async function () { + + const urls = [process.env.SITE_URL]; + + // Set screenshot option + const takeScreenshot = true; + + /* Disabling default step screen shots taken during Synthetics.executeStep() calls + * Step will be used to publish metrics on time taken to load dom content but + * Screenshots will be taken outside the executeStep to allow for page to completely load with domcontentloaded + * You can change it to load, networkidle0, networkidle2 depending on what works best for you. + */ + syntheticsConfiguration.disableStepScreenshots(); + syntheticsConfiguration.setConfig({ + continueOnStepFailure: true, + includeRequestHeaders: true, // Enable if headers should be displayed in HAR + includeResponseHeaders: true, // Enable if headers should be displayed in HAR + restrictedHeaders: [], // Value of these headers will be redacted from logs and reports + restrictedUrlParameters: [] // Values of these url parameters will be redacted from logs and reports + + }); + + let page = await synthetics.getPage(); + + for (const url of urls) { + await loadUrl(page, url, takeScreenshot); + } +}; + +// Reset the page in-between +const resetPage = async function(page) { + try { + await page.goto('about:blank',{waitUntil: ['load', 'networkidle0'], timeout: 30000} ); + } catch (e) { + synthetics.addExecutionError('Unable to open a blank page. ', e); + } +} + +const loadUrl = async function (page, url, takeScreenshot) { + let stepName = null; + let domcontentloaded = false; + + try { + stepName = new URL(url).hostname; + } catch (e) { + const errorString = `Error parsing url: ${url}. ${e}`; + log.error(errorString); + /* If we fail to parse the URL, don't emit a metric with a stepName based on it. + It may not be a legal CloudWatch metric dimension name and we may not have an alarms + setup on the malformed URL stepName. Instead, fail this step which will + show up in the logs and will fail the overall canary and alarm on the overall canary + success rate. + */ + throw e; + } + + await synthetics.executeStep(stepName, async function () { + const sanitizedUrl = syntheticsLogHelper.getSanitizedUrl(url); + + /* You can customize the wait condition here. For instance, using 'networkidle2' or 'networkidle0' to load page completely. + networkidle0: Navigation is successful when the page has had no network requests for half a second. This might never happen if page is constantly loading multiple resources. + networkidle2: Navigation is successful when the page has no more then 2 network requests for half a second. + domcontentloaded: It's fired as soon as the page DOM has been loaded, without waiting for resources to finish loading. If needed add explicit wait with await new Promise(r => setTimeout(r, milliseconds)) + */ + const response = await page.goto(url, { waitUntil: ['domcontentloaded'], timeout: 30000}); + if (response) { + domcontentloaded = true; + const status = response.status(); + const statusText = response.statusText(); + + logResponseString = `Response from url: ${sanitizedUrl} Status: ${status} Status Text: ${statusText}`; + + //If the response status code is not a 2xx success code + if (response.status() < 200 || response.status() > 299) { + throw new Error(`Failed to load url: ${sanitizedUrl} ${response.status()} ${response.statusText()}`); + } + } else { + const logNoResponseString = `No response returned for url: ${sanitizedUrl}`; + log.error(logNoResponseString); + throw new Error(logNoResponseString); + } + }); + + // Wait for 15 seconds to let page load fully before taking screenshot. + if (domcontentloaded && takeScreenshot) { + await new Promise(r => setTimeout(r, 15000)); + await synthetics.takeScreenshot(stepName, 'loaded'); + } + + // Reset page + await resetPage(page); +}; + +exports.handler = async () => { + return await loadBlueprint(); +}; \ No newline at end of file diff --git a/infrastructure/lib/constructs/canarySns.ts b/infrastructure/lib/constructs/canarySns.ts new file mode 100644 index 0000000..a9fa1bd --- /dev/null +++ b/infrastructure/lib/constructs/canarySns.ts @@ -0,0 +1,40 @@ +import {SnsMonitors} from "./snsMonitor"; +import {SnsMonitorsProps} from "./snsMonitor"; +import {Construct} from "constructs"; +import {Alarm} from "aws-cdk-lib/aws-cloudwatch"; +import { Canary } from 'aws-cdk-lib/aws-synthetics'; +import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch"; + +interface canarySnsProps extends SnsMonitorsProps { + readonly canaryAlarms: Array<{ alertName: string, canary: Canary }>; +} + +export class canarySns extends SnsMonitors { + private readonly canaryAlarms: Array<{ alertName: string, canary: Canary }>; + constructor(scope: Construct, id: string, props: canarySnsProps) { + super(scope, id, props); + this.canaryAlarms = props.canaryAlarms; + this.canaryAlarms.forEach(({ alertName, canary }) => + { + const alarm = this.canaryFailed(alertName, canary); + this.map[alarm[1]] = alarm[0]; + }); + this.createTopic(); + } + + private canaryFailed(alertName: string, canary: Canary): [Alarm, string] { + const alarmObject = new cloudwatch.Alarm(this, `error_alarm_${alertName}`, { + metric: canary.metricSuccessPercent(), + threshold: 100, + evaluationPeriods: 1, + comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD, + datapointsToAlarm: 1, + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + alarmDescription: "Detect Canary failure", + alarmName: alertName, + }); + return [alarmObject, alertName]; + } +} + + diff --git a/infrastructure/lib/constructs/snsMonitor.ts b/infrastructure/lib/constructs/snsMonitor.ts new file mode 100644 index 0000000..8c7e5de --- /dev/null +++ b/infrastructure/lib/constructs/snsMonitor.ts @@ -0,0 +1,64 @@ +import { Construct } from 'constructs'; +import * as sns from "aws-cdk-lib/aws-sns"; +import * as subscriptions from "aws-cdk-lib/aws-sns-subscriptions"; +import * as actions from "aws-cdk-lib/aws-cloudwatch-actions"; +import { Canary } from 'aws-cdk-lib/aws-synthetics'; +import {OpenSearchLambda} from "./lambda"; + +export interface SnsMonitorsProps { + readonly region: string; + readonly accountId: string; + readonly alarmNameSpace: string; + readonly snsTopicName: string; + readonly slackLambda: OpenSearchLambda; +} + +export class SnsMonitors extends Construct { + protected readonly region: string; + protected readonly accountId: string; + protected readonly alarmNameSpace: string; + protected readonly map: { [id: string]: any }; + private readonly snsTopicName: string; + private readonly slackLambda: OpenSearchLambda; + private readonly emailList: Array; + + + constructor(scope: Construct, id: string, props: SnsMonitorsProps) { + super(scope, id); + this.region = props.region; + this.accountId = props.accountId; + this.alarmNameSpace = props.alarmNameSpace; + this.snsTopicName = props.snsTopicName; + this.slackLambda = props.slackLambda; + + // The email list for receiving alerts + this.emailList = [ + 'insert@mail.here' + ]; + + // Create alarms + this.map = {}; + + } + + protected createTopic(){ + // Create SNS topic for alarms to be sent to + const snsTopic = new sns.Topic(this, `OpenSearchMetrics-Alarm-${this.snsTopicName}`, { + displayName: `OpenSearchMetrics-Alarm-${this.snsTopicName}` + }); + + // Iterate map to create SNS topic and add alarms on it + Object.keys(this.map).map(key => { + // Connect the alarm to the SNS + this.map[key].addAlarmAction(new actions.SnsAction(snsTopic)); + }) + + // Send email notification to the recipients + for (const email of this.emailList) { + snsTopic.addSubscription(new subscriptions.EmailSubscription(email)); + } + + // Send slack notification + snsTopic.addSubscription(new subscriptions.LambdaSubscription(this.slackLambda.lambda)); + } +} diff --git a/infrastructure/lib/constructs/stepFunctionSns.ts b/infrastructure/lib/constructs/stepFunctionSns.ts new file mode 100644 index 0000000..5b9e517 --- /dev/null +++ b/infrastructure/lib/constructs/stepFunctionSns.ts @@ -0,0 +1,45 @@ +import {SnsMonitors} from "./snsMonitor"; +import {SnsMonitorsProps} from "./snsMonitor"; +import {Construct} from "constructs"; +import {Alarm} from "aws-cdk-lib/aws-cloudwatch"; +import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch"; + +interface stepFunctionSnsProps extends SnsMonitorsProps { + readonly stepFunctionSnsAlarms: Array<{ alertName: string, stateMachineName: string }>; +} + +export class StepFunctionSns extends SnsMonitors { + private readonly stepFunctionSnsAlarms: Array<{ alertName: string, stateMachineName: string }>; + constructor(scope: Construct, id: string, props: stepFunctionSnsProps) { + super(scope, id, props); + this.stepFunctionSnsAlarms = props.stepFunctionSnsAlarms; + this.stepFunctionSnsAlarms.forEach(({ alertName, stateMachineName }) => + { + const alarm = this.stepFunctionExecutionsFailed(alertName, stateMachineName); + this.map[alarm[1]] = alarm[0]; + }); + this.createTopic(); + } + + private stepFunctionExecutionsFailed(alertName: string, stateMachineName: string): [Alarm, string] { + const alarmObject = new cloudwatch.Alarm(this, `error_alarm_${alertName}`, { + metric: new cloudwatch.Metric({ + namespace: this.alarmNameSpace, + metricName: "ExecutionsFailed", + statistic: "Sum", + dimensionsMap: { + StateMachineArn: `arn:aws:states:${this.region}:${this.accountId}:stateMachine:${stateMachineName}` + } + }), + threshold: 1, + evaluationPeriods: 1, + comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + datapointsToAlarm: 1, + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + alarmDescription: "Detect SF execution failure", + alarmName: alertName, + }); + return [alarmObject, alertName]; + } +} + diff --git a/infrastructure/lib/infrastructure-stack.ts b/infrastructure/lib/infrastructure-stack.ts index e4d966e..adf43ac 100644 --- a/infrastructure/lib/infrastructure-stack.ts +++ b/infrastructure/lib/infrastructure-stack.ts @@ -9,6 +9,8 @@ import {OpenSearchMetricsNginxReadonly} from "./stacks/opensearchNginxProxyReado import {ArnPrincipal} from "aws-cdk-lib/aws-iam"; import {OpenSearchWAF} from "./stacks/waf"; import {OpenSearchMetricsNginxCognito} from "./constructs/opensearchNginxProxyCognito"; +import {OpenSearchMetricsMonitoringStack} from "./stacks/monitoringDashboard"; +import {OpenSearchMetricsSecrets} from "./stacks/secrets"; // import * as sqs from 'aws-cdk-lib/aws-sqs'; export class InfrastructureStack extends Stack { @@ -34,12 +36,25 @@ export class InfrastructureStack extends Stack { } }); - // Create OpenSearch Metrics Lambda setup const openSearchMetricsWorkflowStack = new OpenSearchMetricsWorkflowStack(app, 'OpenSearchMetrics-Workflow', { opensearchDomainStack: openSearchDomainStack, vpcStack: vpcStack, lambdaPackage: Project.LAMBDA_PACKAGE}) openSearchMetricsWorkflowStack.node.addDependency(vpcStack, openSearchDomainStack); + // Create Secrets Manager + + const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets"); + + // Create Monitoring Dashboard + + const openSearchMetricsMonitoringStack = new OpenSearchMetricsMonitoringStack(app, "OpenSearchMetrics-Monitoring", { + region: Project.REGION, + account: Project.AWS_ACCOUNT, + workflowComponent: openSearchMetricsWorkflowStack.workflowComponent, + lambdaPackage: Project.LAMBDA_PACKAGE, + secrets: openSearchMetricsSecretsStack.secretsObject, + vpcStack: vpcStack + }); // Create OpenSearch Metrics Frontend DNS const metricsHostedZone = new OpenSearchHealthRoute53(app, "OpenSearchMetrics-HostedZone", { diff --git a/infrastructure/lib/stacks/metricsWorkflow.ts b/infrastructure/lib/stacks/metricsWorkflow.ts index b2426e2..39521ac 100644 --- a/infrastructure/lib/stacks/metricsWorkflow.ts +++ b/infrastructure/lib/stacks/metricsWorkflow.ts @@ -13,7 +13,12 @@ export interface OpenSearchMetricsStackProps extends StackProps { readonly vpcStack: VpcStack; readonly lambdaPackage: string } + +export interface WorkflowComponent { + opensearchMetricsWorkflowStateMachineName: string +} export class OpenSearchMetricsWorkflowStack extends Stack { + public readonly workflowComponent: WorkflowComponent; constructor(scope: Construct, id: string, props: OpenSearchMetricsStackProps) { super(scope, id, props); @@ -39,6 +44,10 @@ export class OpenSearchMetricsWorkflowStack extends Stack { schedule: Schedule.expression('cron(0 7 * * ? *)'), targets: [new SfnStateMachine(opensearchMetricsWorkflow)], }); + + this.workflowComponent = { + opensearchMetricsWorkflowStateMachineName: opensearchMetricsWorkflow.stateMachineName + } } private createMetricsTask(scope: Construct, opensearchDomainStack: OpenSearchDomainStack, diff --git a/infrastructure/lib/stacks/monitoringDashboard.ts b/infrastructure/lib/stacks/monitoringDashboard.ts new file mode 100644 index 0000000..b98a349 --- /dev/null +++ b/infrastructure/lib/stacks/monitoringDashboard.ts @@ -0,0 +1,111 @@ +import {Duration, Stack, StackProps} from "aws-cdk-lib"; +import { Construct } from 'constructs'; +import { WorkflowComponent } from "./metricsWorkflow"; +import {OpenSearchLambda} from "../constructs/lambda"; +import {Secret} from "aws-cdk-lib/aws-secretsmanager"; +import { VpcStack } from "./vpc"; +import { Runtime, Canary, Test, Code, Schedule } from "aws-cdk-lib/aws-synthetics"; +import * as path from "path"; +import Project from "../enums/project"; +import {Effect, PolicyDocument, PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam"; +import {StepFunctionSns} from "../constructs/stepFunctionSns"; +import {canarySns} from "../constructs/canarySns"; + + +interface OpenSearchMetricsMonitoringStackProps extends StackProps { + readonly region: string; + readonly account: string; + readonly workflowComponent: WorkflowComponent; + readonly lambdaPackage: string; + readonly secrets: Secret; + readonly vpcStack: VpcStack; +} + +export class OpenSearchMetricsMonitoringStack extends Stack { + + private readonly slackLambda: OpenSearchLambda; + + constructor(scope: Construct, id: string, readonly props: OpenSearchMetricsMonitoringStackProps) { + super(scope, id, props); + + const slackLambdaRole = new Role(this, 'OpenSearchSlackLambdaRole', { + assumedBy: new ServicePrincipal('lambda.amazonaws.com'), + description: "OpenSearch Metrics Slack Lambda Execution Role", + roleName: "OpenSearchSlackLambdaRole" + }); + + slackLambdaRole.addToPolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["secretsmanager:GetSecretValue"], + resources: [`${props.secrets.secretFullArn}`], + }), + ); + + this.slackLambda = new OpenSearchLambda(this, "OpenSearchMetricsSlackLambdaFunction", { + lambdaNameBase: "OpenSearchMetricsDashboardsSlackLambda", + handler: "org.opensearchmetrics.lambda.SlackLambda", + lambdaZipPath: `../../../build/distributions/${props.lambdaPackage}`, + role: slackLambdaRole, + environment: { + SLACK_CREDENTIALS_SECRETS: props.secrets.secretName, + SECRETS_MANAGER_REGION: props.secrets.env.region + } + }); + this.snsMonitorStepFunctionExecutionsFailed(); + this.snsMonitorCanaryFailed('metrics_heartbeat', `https://${Project.METRICS_HOSTED_ZONE}`, props.vpcStack); + } + + /** + * Create SNS alarms for failure StepFunction jobs. + */ + private snsMonitorStepFunctionExecutionsFailed(): void { + const stepFunctionSnsAlarms = [ + { alertName: 'StepFunction_execution_errors_MetricsWorkflow', stateMachineName: this.props.workflowComponent.opensearchMetricsWorkflowStateMachineName }, + ]; + + new StepFunctionSns(this, "SnsMonitors-StepFunctionExecutionsFailed", { + region: this.props.region, + accountId: this.props.account, + stepFunctionSnsAlarms: stepFunctionSnsAlarms, + alarmNameSpace: "AWS/States", + snsTopicName: "StepFunctionExecutionsFailed", + slackLambda: this.slackLambda + }); + } + + /** + * Create SNS alarms for failure Canaries. + */ + private snsMonitorCanaryFailed(canaryName: string, canaryUrl: string, vpcStack: VpcStack): void { + const canary = new Canary(this, 'CanaryHeartbeatMonitor', { + canaryName: canaryName, + schedule: Schedule.rate(Duration.minutes(1)), + test: Test.custom({ + code: Code.fromAsset(path.join(__dirname, '../../canary')), + handler: 'urlMonitor.handler', + }), + runtime: Runtime.SYNTHETICS_NODEJS_PUPPETEER_7_0, + environmentVariables: { + SITE_URL: canaryUrl + }, + vpc: vpcStack.vpc, + vpcSubnets: vpcStack.subnets, + securityGroups: [vpcStack.securityGroup], + }); + + const canaryAlarms = [ + { alertName: 'Canary_failed_MetricsWorkflow', canary: canary }, + ]; + + new canarySns(this, "SnsMonitors-CanaryFailed", { + region: this.props.region, + accountId: this.props.account, + canaryAlarms: canaryAlarms, + alarmNameSpace: "CloudWatchSynthetics", + snsTopicName: "CanaryFailed", + slackLambda: this.slackLambda + }); + } +} + diff --git a/infrastructure/lib/stacks/secrets.ts b/infrastructure/lib/stacks/secrets.ts new file mode 100644 index 0000000..250b03e --- /dev/null +++ b/infrastructure/lib/stacks/secrets.ts @@ -0,0 +1,14 @@ +import {Stack} from "aws-cdk-lib"; +import { Construct } from 'constructs'; +import {Secret} from "aws-cdk-lib/aws-secretsmanager"; + +export class OpenSearchMetricsSecrets extends Stack { + readonly secretsObject: Secret; + + constructor(scope: Construct, id: string) { + super(scope, id); + this.secretsObject = new Secret(this, 'MetricsCreds', { + secretName: 'metrics-creds', + }); + } +} diff --git a/infrastructure/package-lock.json b/infrastructure/package-lock.json index f5ae636..e4c9c66 100644 --- a/infrastructure/package-lock.json +++ b/infrastructure/package-lock.json @@ -8,7 +8,7 @@ "name": "infrastructure", "version": "0.1.0", "dependencies": { - "aws-cdk-lib": "2.131.0", + "aws-cdk-lib": "2.144.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" }, @@ -1250,9 +1250,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.131.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.131.0.tgz", - "integrity": "sha512-9XLgiTgY+q0S3K93VPeJO0chIN8BZwZ3aSrILvF868Dz+0NTNrD2m5M0xGK5Rw0uoJS+N+DvGaz/2hLAiVqcBw==", + "version": "2.144.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.144.0.tgz", + "integrity": "sha512-DpyIyTs8NHX6WgAyYM2mGorirIk+eTjWzXGQRfzAe40qkwcqsb5Ax4JEl5gz1OEo9QIJIgWDtmImgWN0tUbILA==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -1269,7 +1269,7 @@ "dependencies": { "@aws-cdk/asset-awscli-v1": "^2.2.202", "@aws-cdk/asset-kubectl-v20": "^2.1.2", - "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.3", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", "fs-extra": "^11.2.0", @@ -1279,7 +1279,7 @@ "minimatch": "^3.1.2", "punycode": "^2.3.1", "semver": "^7.6.0", - "table": "^6.8.1", + "table": "^6.8.2", "yaml": "1.10.2" }, "engines": { @@ -1295,14 +1295,14 @@ "license": "Apache-2.0" }, "node_modules/aws-cdk-lib/node_modules/ajv": { - "version": "8.12.0", + "version": "8.13.0", "inBundle": true, "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", + "fast-deep-equal": "^3.1.3", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" + "uri-js": "^4.4.1" }, "funding": { "type": "github", @@ -1567,7 +1567,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/table": { - "version": "6.8.1", + "version": "6.8.2", "inBundle": true, "license": "BSD-3-Clause", "dependencies": { @@ -3866,7 +3866,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "engines": { "node": ">=6" } diff --git a/infrastructure/package.json b/infrastructure/package.json index cffa0ad..d68646a 100644 --- a/infrastructure/package.json +++ b/infrastructure/package.json @@ -20,7 +20,7 @@ "typescript": "~3.9.7" }, "dependencies": { - "aws-cdk-lib": "2.131.0", + "aws-cdk-lib": "2.144.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" } diff --git a/infrastructure/test/monitoring-stack.test.ts b/infrastructure/test/monitoring-stack.test.ts new file mode 100644 index 0000000..5d4fb75 --- /dev/null +++ b/infrastructure/test/monitoring-stack.test.ts @@ -0,0 +1,234 @@ +import {App} from "aws-cdk-lib"; +import {Template} from "aws-cdk-lib/assertions"; +import {OpenSearchMetricsWorkflowStack} from "../lib/stacks/metricsWorkflow"; +import Project from "../lib/enums/project"; +import {OpenSearchDomainStack} from "../lib/stacks/opensearch"; +import {VpcStack} from "../lib/stacks/vpc"; +import {ArnPrincipal} from "aws-cdk-lib/aws-iam"; +import {OpenSearchMetricsMonitoringStack} from "../lib/stacks/monitoringDashboard"; +import {OpenSearchMetricsSecrets} from "../lib/stacks/secrets"; + +test('Monitoring Stack Test', () => { + const app = new App(); + const vpcStack = new VpcStack(app, 'OpenSearchHealth-VPC', {}); + const openSearchMetricsWorkflowStack = new OpenSearchMetricsWorkflowStack(app, 'OpenSearchMetrics-Workflow', { + opensearchDomainStack: new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', { + region: "us-east-1", + account: "test-account", + vpcStack: vpcStack, + enableNginxCognito: true, + jenkinsAccess: { + jenkinsAccountRoles: [ + new ArnPrincipal(Project.JENKINS_MASTER_ROLE), + new ArnPrincipal(Project.JENKINS_AGENT_ROLE) + ] + } + }), + vpcStack: vpcStack, + lambdaPackage: Project.LAMBDA_PACKAGE + }); + const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets"); + const openSearchMetricsMonitoringStack = new OpenSearchMetricsMonitoringStack(app, "OpenSearchMetrics-Monitoring", { + region: Project.REGION, + account: Project.AWS_ACCOUNT, + workflowComponent: openSearchMetricsWorkflowStack.workflowComponent, + lambdaPackage: Project.LAMBDA_PACKAGE, + secrets: openSearchMetricsSecretsStack.secretsObject, + vpcStack: vpcStack + }); + const template = Template.fromStack(openSearchMetricsMonitoringStack); + template.resourceCountIs('AWS::IAM::Role', 2); + template.resourceCountIs('AWS::IAM::Policy', 1); + template.resourceCountIs('AWS::CloudWatch::Alarm', 2); + template.resourceCountIs('AWS::SNS::Topic', 2); + template.resourceCountIs('AWS::Synthetics::Canary', 1); + template.hasResourceProperties('AWS::IAM::Role', { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "RoleName": "OpenSearchSlackLambdaRole" + }); + + template.hasResourceProperties('AWS::IAM::Policy', { + "PolicyDocument": { + "Statement": [ + { + "Action": "secretsmanager:GetSecretValue", + "Effect": "Allow", + "Resource": { + "Fn::ImportValue": "OpenSearchMetrics-Secrets:ExportsOutputRefMetricsCreds2260E61E4655F9C2" + } + }, + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17", + }, + "PolicyName": "OpenSearchSlackLambdaRoleDefaultPolicy849E8281", + "Roles": [ + { + "Ref": "OpenSearchSlackLambdaRole441FAD2D" + } + ] + }); + template.hasResourceProperties('AWS::Lambda::Function', { + "FunctionName": "OpenSearchMetricsDashboardsSlackLambdaLambda", + "Handler": "org.opensearchmetrics.lambda.SlackLambda", + "MemorySize": 1024, + "Role": { + "Fn::GetAtt": [ + "OpenSearchSlackLambdaRole441FAD2D", + "Arn" + ] + }, + "Runtime": "java17", + "Timeout": 900, + "TracingConfig": { + "Mode": "Active" + } + }); + template.hasResourceProperties('AWS::Lambda::Permission', { + "Action": "lambda:InvokeFunction", + "FunctionName": { + "Fn::GetAtt": [ + "OpenSearchMetricsDashboardsSlackLambdaLambda28DA56CA", + "Arn" + ] + }, + "Principal": "sns.amazonaws.com", + "SourceArn": { + "Ref": "SnsMonitorsStepFunctionExecutionsFailedOpenSearchMetricsAlarmStepFunctionExecutionsFailed0B259DBC" + } + }); + template.hasResourceProperties('AWS::CloudWatch::Alarm', { + "AlarmActions": [ + { + "Ref": "SnsMonitorsStepFunctionExecutionsFailedOpenSearchMetricsAlarmStepFunctionExecutionsFailed0B259DBC" + } + ], + "AlarmDescription": "Detect SF execution failure", + "AlarmName": "StepFunction_execution_errors_MetricsWorkflow", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "DatapointsToAlarm": 1, + "Dimensions": [ + { + "Name": "StateMachineArn", + "Value": { + "Fn::Join": [ + "", + [ + "arn:aws:states:::stateMachine:", + { + "Fn::ImportValue": "OpenSearchMetrics-Workflow:ExportsOutputFnGetAttOpenSearchMetricsWorkflowDB4D4CB1NameE4E75A02" + } + ] + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ExecutionsFailed", + "Namespace": "AWS/States", + "Period": 300, + "Statistic": "Sum", + "Threshold": 1, + "TreatMissingData": "notBreaching" + }); + template.hasResourceProperties('AWS::Synthetics::Canary', { + "ArtifactS3Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "CanaryHeartbeatMonitorArtifactsBucketA8125411" + } + ] + ] + }, + "Code": { + "Handler": "urlMonitor.handler", + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "3add60a2b13650e2ad0c97ef8b24082c52ea91e59b8b8bac89874c602a6b908d.zip" + }, + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "CanaryHeartbeatMonitorServiceRole424026C0", + "Arn" + ] + }, + "Name": "metrics_heartbeat", + "RunConfig": { + "EnvironmentVariables": { + "SITE_URL": "https://metrics.opensearch.org" + } + }, + "RuntimeVersion": "syn-nodejs-puppeteer-7.0", + "Schedule": { + "DurationInSeconds": "0", + "Expression": "rate(1 minute)" + }, + "StartCanaryAfterCreation": true, + "VPCConfig": { + "SecurityGroupIds": [ + { + "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputFnGetAttVpcSecurityGroup092B7291GroupIdA3F0A2EB" + } + ], + "SubnetIds": [ + { + "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet1Subnet529349B600974078" + }, + { + "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet2SubnetBA599EDB2BEEEA30" + } + ], + "VpcId": { + "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcB885AABED860B3EB" + } + } + }); + template.hasResourceProperties('AWS::CloudWatch::Alarm', { + "AlarmActions": [ + { + "Ref": "SnsMonitorsCanaryFailedOpenSearchMetricsAlarmCanaryFailed4CF8A950" + } + ], + "AlarmDescription": "Detect Canary failure", + "AlarmName": "Canary_failed_MetricsWorkflow", + "ComparisonOperator": "LessThanThreshold", + "DatapointsToAlarm": 1, + "Dimensions": [ + { + "Name": "CanaryName", + "Value": { + "Ref": "CanaryHeartbeatMonitorFE4C06BE" + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "SuccessPercent", + "Namespace": "CloudWatchSynthetics", + "Period": 300, + "Statistic": "Average", + "Threshold": 100, + "TreatMissingData": "notBreaching" + }); +}); \ No newline at end of file diff --git a/infrastructure/test/secrets-stack.test.ts b/infrastructure/test/secrets-stack.test.ts new file mode 100644 index 0000000..ed7ee96 --- /dev/null +++ b/infrastructure/test/secrets-stack.test.ts @@ -0,0 +1,14 @@ +import {App} from "aws-cdk-lib"; +import {Template} from "aws-cdk-lib/assertions"; +import {OpenSearchMetricsSecrets} from "../lib/stacks/secrets"; + +test('Secrets Stack Test', () => { + const app = new App(); + const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets"); + const template = Template.fromStack(openSearchMetricsSecretsStack); + template.resourceCountIs('AWS::SecretsManager::Secret', 1); + template.hasResourceProperties('AWS::SecretsManager::Secret', { + "GenerateSecretString": {}, + "Name": "metrics-creds" + }); +}); \ No newline at end of file diff --git a/src/main/java/org/opensearchmetrics/dagger/CommonModule.java b/src/main/java/org/opensearchmetrics/dagger/CommonModule.java index 72b72a3..2328685 100644 --- a/src/main/java/org/opensearchmetrics/dagger/CommonModule.java +++ b/src/main/java/org/opensearchmetrics/dagger/CommonModule.java @@ -1,5 +1,6 @@ package org.opensearchmetrics.dagger; +import org.opensearchmetrics.util.SecretsManagerUtil; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; @@ -16,6 +17,8 @@ import org.opensearchmetrics.metrics.label.LabelMetrics; import org.opensearchmetrics.metrics.release.ReleaseMetrics; import org.opensearchmetrics.util.OpenSearchUtil; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.signer.Aws4Signer; import software.amazon.awssdk.services.sts.StsClient; @@ -30,6 +33,7 @@ public class CommonModule { private static final String OPENSEARCH_DOMAIN_REGION = "OPENSEARCH_DOMAIN_REGION"; private static final String OPENSEARCH_DOMAIN_ROLE = "OPENSEARCH_DOMAIN_ROLE"; private static final String ROLE_SESSION_NAME = "OpenSearchHealth"; + private static final String SECRETS_MANAGER_REGION = "SECRETS_MANAGER_REGION"; @Singleton @Provides @@ -92,4 +96,14 @@ public MetricsCalculation getMetricsCalculation(OpenSearchUtil openSearchUtil, O issuePositiveReactions, issueNegativeReactions, labelMetrics, releaseMetrics); } + + @Provides + @Singleton + public SecretsManagerUtil getSecretsManagerUtil(ObjectMapper mapper) { + final String region = System.getenv(SECRETS_MANAGER_REGION); + final AWSSecretsManager secretsManager = AWSSecretsManagerClientBuilder.standard() + .withRegion(region) + .build(); + return new SecretsManagerUtil(secretsManager, mapper); + } } diff --git a/src/main/java/org/opensearchmetrics/dagger/ServiceComponent.java b/src/main/java/org/opensearchmetrics/dagger/ServiceComponent.java index 8c1254c..e214b88 100644 --- a/src/main/java/org/opensearchmetrics/dagger/ServiceComponent.java +++ b/src/main/java/org/opensearchmetrics/dagger/ServiceComponent.java @@ -6,6 +6,7 @@ import org.opensearchmetrics.metrics.general.Metrics; import org.opensearchmetrics.metrics.label.LabelMetrics; import org.opensearchmetrics.util.OpenSearchUtil; +import org.opensearchmetrics.util.SecretsManagerUtil; import javax.inject.Named; import javax.inject.Singleton; @@ -20,6 +21,8 @@ public interface ServiceComponent { MetricsCalculation getMetricsCalculation(); + SecretsManagerUtil getSecretsManagerUtil(); + @Named(MetricsModule.UNTRIAGED_ISSUES) Metrics getUntriagedIssues(); diff --git a/src/main/java/org/opensearchmetrics/datasource/DataSourceType.java b/src/main/java/org/opensearchmetrics/datasource/DataSourceType.java new file mode 100644 index 0000000..b64ab73 --- /dev/null +++ b/src/main/java/org/opensearchmetrics/datasource/DataSourceType.java @@ -0,0 +1,7 @@ +package org.opensearchmetrics.datasource; + +public enum DataSourceType { + SLACK_WEBHOOK_URL, + SLACK_CHANNEL, + SLACK_USERNAME +} diff --git a/src/main/java/org/opensearchmetrics/lambda/SlackLambda.java b/src/main/java/org/opensearchmetrics/lambda/SlackLambda.java new file mode 100644 index 0000000..59bc761 --- /dev/null +++ b/src/main/java/org/opensearchmetrics/lambda/SlackLambda.java @@ -0,0 +1,112 @@ +package org.opensearchmetrics.lambda; + +import org.opensearchmetrics.util.SecretsManagerUtil; +import org.opensearchmetrics.datasource.DataSourceType; +import com.google.common.annotations.VisibleForTesting; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import lombok.Data; +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SNSEvent; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.opensearchmetrics.dagger.ServiceComponent; +import org.opensearchmetrics.dagger.DaggerServiceComponent; + +import java.io.IOException; + +@Slf4j +public class SlackLambda implements RequestHandler { + private static final ServiceComponent COMPONENT = DaggerServiceComponent.create(); + private final SecretsManagerUtil secretsManagerUtil; + private final ObjectMapper mapper; + + public SlackLambda() { + this(COMPONENT.getSecretsManagerUtil()); + } + + @VisibleForTesting + SlackLambda(@NonNull SecretsManagerUtil secretsManagerUtil) { + this.secretsManagerUtil = secretsManagerUtil; + this.mapper = COMPONENT.getObjectMapper(); + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown = true) + @Data + private static class AlarmObject { + @JsonProperty("AlarmName") + private String alarmName; + @JsonProperty("AlarmDescription") + private String alarmDescription; + @JsonProperty("StateChangeTime") + private String stateChangeTime; + @JsonProperty("Region") + private String region; + @JsonProperty("AlarmArn") + private String alarmArn; + } + + @Override + public Void handleRequest(SNSEvent event, Context context) { + String slackWebhookURL; + String slackChannel; + String slackUsername; + try { + slackWebhookURL = secretsManagerUtil.getSlackCredentials(DataSourceType.SLACK_WEBHOOK_URL).get(); + slackChannel = secretsManagerUtil.getSlackCredentials(DataSourceType.SLACK_CHANNEL).get(); + slackUsername = secretsManagerUtil.getSlackCredentials(DataSourceType.SLACK_USERNAME).get(); + } catch (Exception ex) { + log.error("Unable to get Slack credentials", ex); + throw new RuntimeException(ex); + } + String message = event.getRecords().get(0).getSNS().getMessage(); + try { + sendMessageToSlack(message, slackWebhookURL, slackChannel, slackUsername); + } catch (Exception ex) { + log.error("Unable to send message to Slack", ex); + throw new RuntimeException(ex); + } + return null; + } + + private void sendMessageToSlack(String message, String slackWebhookURL, String slackChannel, String slackUsername) throws IOException { + AlarmObject alarmObject = + mapper.readValue(message, AlarmObject.class); + String alarmMessage = ":alert: OpenSearch Metrics Dashboard Monitoring alarm activated. Please investigate the issue. \n" + + "- Name: " + alarmObject.getAlarmName() + "\n" + + "- Description: " + alarmObject.getAlarmDescription() + "\n" + + "- StateChangeTime: " + alarmObject.getStateChangeTime() + "\n" + + "- Region: " + alarmObject.getRegion() + "\n" + + "- AlarmArn: " + alarmObject.getAlarmArn(); + ObjectNode payload = mapper.createObjectNode(); + payload.put("channel", slackChannel); + payload.put("username", slackUsername); + payload.put("Content", alarmMessage); + payload.put("icon_emoji", ""); + + try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { + HttpPost httpPost = new HttpPost(slackWebhookURL); + httpPost.setEntity(new StringEntity(mapper.writeValueAsString(payload), "UTF-8")); + HttpResponse response = httpClient.execute(httpPost); + + System.out.println("{" + + "\"message\": \"" + alarmMessage + "\"," + + "\"status_code\": " + response.getStatusLine().getStatusCode() + "," + + "\"response\": \"" + response.getEntity().getContent().toString() + "\"" + + "}"); + } catch (IOException ex) { + ex.printStackTrace(); + throw new RuntimeException(ex); + } + } +} diff --git a/src/main/java/org/opensearchmetrics/util/SecretsManagerUtil.java b/src/main/java/org/opensearchmetrics/util/SecretsManagerUtil.java new file mode 100644 index 0000000..b3a1f27 --- /dev/null +++ b/src/main/java/org/opensearchmetrics/util/SecretsManagerUtil.java @@ -0,0 +1,62 @@ +package org.opensearchmetrics.util; + +import org.opensearchmetrics.datasource.DataSourceType; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +@Slf4j +public class SecretsManagerUtil { + private static final String SLACK_CREDENTIALS_SECRETS = "SLACK_CREDENTIALS_SECRETS"; + private final AWSSecretsManager secretsManager; + private final ObjectMapper mapper; + + public SecretsManagerUtil(AWSSecretsManager secretsManager, ObjectMapper mapper) { + this.secretsManager = secretsManager; + this.mapper = mapper; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonIgnoreProperties(ignoreUnknown = true) + @Data + public static class SlackCredentials { + @JsonProperty("slackWebhookURL") + private String slackWebhookURL; + @JsonProperty("slackChannel") + private String slackChannel; + @JsonProperty("slackUsername") + private String slackUsername; + } + + public Optional getSlackCredentials(DataSourceType datasourceType) throws IOException { + String secretName = System.getenv(SLACK_CREDENTIALS_SECRETS); + log.info("Retrieving secrets value from secrets = {} ", secretName); + GetSecretValueResult getSecretValueResult = + secretsManager.getSecretValue(new GetSecretValueRequest().withSecretId(secretName)); + log.info("Successfully retrieved secrets for data source credentials"); + SlackCredentials credentials = + mapper.readValue(getSecretValueResult.getSecretString(), SlackCredentials.class); + switch (datasourceType) { + case SLACK_WEBHOOK_URL: + return Optional.of(credentials.getSlackWebhookURL()); + case SLACK_CHANNEL: + return Optional.of(credentials.getSlackChannel()); + case SLACK_USERNAME: + return Optional.of(credentials.getSlackUsername()); + default: + return Optional.empty(); + } + } +} diff --git a/src/test/java/org/opensearchmetrics/lambda/SlackLambdaTest.java b/src/test/java/org/opensearchmetrics/lambda/SlackLambdaTest.java new file mode 100644 index 0000000..7c749b1 --- /dev/null +++ b/src/test/java/org/opensearchmetrics/lambda/SlackLambdaTest.java @@ -0,0 +1,146 @@ +package org.opensearchmetrics.lambda; + +import com.amazonaws.services.lambda.runtime.Context; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.terms.ParsedStringTerms; +import org.opensearchmetrics.metrics.MetricsCalculation; +import org.opensearchmetrics.util.OpenSearchUtil; +import org.opensearchmetrics.datasource.DataSourceType; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.HttpResponse; +import org.apache.http.StatusLine; +import org.apache.http.HttpEntity; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.opensearchmetrics.util.SecretsManagerUtil; +import com.amazonaws.services.lambda.runtime.events.SNSEvent; +import com.amazonaws.services.lambda.runtime.events.SNSEvent.SNS; + +import java.util.Collections; +import java.util.Optional; +import java.util.List; +import java.io.IOException; +import java.io.InputStream; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + + +public class SlackLambdaTest { + + @Mock + private SecretsManagerUtil secretsManagerUtil; + + @BeforeEach + public void setUp() { + MockitoAnnotations.openMocks(this); + } + + @Test + public void testHandleRequest() throws IOException{ + SlackLambda slackLambda = new SlackLambda(secretsManagerUtil); + + Optional optional = mock(Optional.class); + SNSEvent snsEvent = getSNSEventFromMessage("{\"test\":\"test\"}"); + Context context = mock(Context.class); + CloseableHttpClient httpClient = mock(CloseableHttpClient.class); + HttpClientBuilder httpClientBuilder = mock(HttpClientBuilder.class); + StatusLine statusLine = mock(StatusLine.class); + CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + HttpEntity httpEntity = mock(HttpEntity.class); + InputStream inputStream = mock(InputStream.class); + + when(secretsManagerUtil.getSlackCredentials(any(DataSourceType.class))).thenReturn(optional); + when(optional.get()).thenReturn(""); + + try (var mockedHttpClientBuilder = mockStatic(HttpClientBuilder.class)) { + mockedHttpClientBuilder.when(HttpClientBuilder::create).thenReturn(httpClientBuilder); + when(httpClientBuilder.build()).thenReturn(httpClient); + when(httpClient.execute(any(HttpPost.class))).thenReturn(httpResponse); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + when(statusLine.getStatusCode()).thenReturn(HttpStatus.SC_OK); + when(httpResponse.getEntity()).thenReturn(httpEntity); + when(httpEntity.getContent()).thenReturn(inputStream); + + slackLambda.handleRequest(snsEvent, context); + } + + + // Assert + verify(httpClient, times(1)).execute(any(HttpPost.class)); + verify(secretsManagerUtil, times(3)).getSlackCredentials(any(DataSourceType.class)); + } + + @Test + public void testHandleRequestWithSecretManagerUtilException() throws IOException{ + SlackLambda slackLambda = new SlackLambda(secretsManagerUtil); + + Optional optional = mock(Optional.class); + SNSEvent snsEvent = getSNSEventFromMessage("{\"test\":\"test\"}"); + Context context = mock(Context.class); + + when(optional.get()).thenReturn(""); + + doThrow(new IOException("Error running getSlackCredentials")).when(secretsManagerUtil).getSlackCredentials(any(DataSourceType.class)); + + try { + slackLambda.handleRequest(snsEvent, context); + fail("Expected a RuntimeException to be thrown"); + } catch (RuntimeException e) { + // Exception caught as expected + System.out.println("Caught exception message: " + e.getMessage()); + assertTrue(e.getMessage().contains("Error running getSlackCredentials")); + } + } + + @Test + public void testHandleRequestWithHttpClientException() throws IOException{ + SlackLambda slackLambda = new SlackLambda(secretsManagerUtil); + + Optional optional = mock(Optional.class); + SNSEvent snsEvent = getSNSEventFromMessage("{\"test\":\"test\"}"); + Context context = mock(Context.class); + CloseableHttpClient httpClient = mock(CloseableHttpClient.class); + HttpClientBuilder httpClientBuilder = mock(HttpClientBuilder.class); + StatusLine statusLine = mock(StatusLine.class); + CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + HttpEntity httpEntity = mock(HttpEntity.class); + InputStream inputStream = mock(InputStream.class); + + when(secretsManagerUtil.getSlackCredentials(any(DataSourceType.class))).thenReturn(optional); + when(optional.get()).thenReturn(""); + + try (var mockedHttpClientBuilder = mockStatic(HttpClientBuilder.class)) { + mockedHttpClientBuilder.when(HttpClientBuilder::create).thenReturn(httpClientBuilder); + when(httpClientBuilder.build()).thenReturn(httpClient); + doThrow(new IOException("Error running httpClient execute")).when(httpClient).execute(any(HttpPost.class)); + + try { + slackLambda.handleRequest(snsEvent, context); + fail("Expected a RuntimeException to be thrown"); + } catch (RuntimeException e) { + // Exception caught as expected + System.out.println("Caught exception message: " + e.getMessage()); + assertTrue(e.getMessage().contains("Error running httpClient execute")); + } + } + } + + private SNSEvent getSNSEventFromMessage(String message) throws IOException { + SNSEvent event = new SNSEvent(); + SNSEvent.SNSRecord record = new SNSEvent.SNSRecord(); + record.setSns(new SNSEvent.SNS().withMessage(message)); + event.setRecords(List.of(record)); + return event; + } +} diff --git a/src/test/java/org/opensearchmetrics/util/SecretsManagerUtilTest.java b/src/test/java/org/opensearchmetrics/util/SecretsManagerUtilTest.java new file mode 100644 index 0000000..ea58682 --- /dev/null +++ b/src/test/java/org/opensearchmetrics/util/SecretsManagerUtilTest.java @@ -0,0 +1,66 @@ +package org.opensearchmetrics.util; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import org.opensearchmetrics.datasource.DataSourceType; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.IOException; +import java.util.Optional; + +public class SecretsManagerUtilTest { + + @Mock + private AWSSecretsManager secretsManager; + + @Mock + private ObjectMapper mapper; + + private SecretsManagerUtil secretsManagerUtil; + + @BeforeEach + void setup() { + MockitoAnnotations.openMocks(this); + secretsManagerUtil = new SecretsManagerUtil(secretsManager, mapper); + } + + @Test + void testGetRedshiftCredentials() throws IOException { + + String slackWebhookURL = "slack-webhook-url"; + String slackChannel = "slack-channel"; + String slackUsername = "slack-username"; + SecretsManagerUtil.SlackCredentials slackCredentials = new SecretsManagerUtil.SlackCredentials(); + slackCredentials.setSlackWebhookURL(slackWebhookURL); + slackCredentials.setSlackChannel(slackChannel); + slackCredentials.setSlackUsername(slackUsername); + + String secretString = "secret-string-with-slack-credentials"; + GetSecretValueResult getSecretValueResult = new GetSecretValueResult(); + getSecretValueResult.setSecretString(secretString); + + when(secretsManager.getSecretValue(any(GetSecretValueRequest.class))) + .thenReturn(getSecretValueResult); + when(mapper.readValue(eq(secretString), eq(SecretsManagerUtil.SlackCredentials.class))) + .thenReturn(slackCredentials); + + Optional webhookURLResult = secretsManagerUtil.getSlackCredentials(DataSourceType.SLACK_WEBHOOK_URL); + Optional channelResult = secretsManagerUtil.getSlackCredentials(DataSourceType.SLACK_CHANNEL); + Optional usernameResult = secretsManagerUtil.getSlackCredentials(DataSourceType.SLACK_USERNAME); + + assertTrue(webhookURLResult.isPresent()); + assertTrue(channelResult.isPresent()); + assertTrue(usernameResult.isPresent()); + assertEquals(slackWebhookURL, webhookURLResult.get()); + assertEquals(slackChannel, channelResult.get()); + assertEquals(slackUsername, usernameResult.get()); + } +}