Added canary url and stepfunction monitoring with sns email and slack…

… integration

Signed-off-by: Brandon Shien <bshien@amazon.com>

.gitignore

@@ -40,6 +40,7 @@ infrastructure/**/*.js
 infrastructure/!jest.config.js
 infrastructure/**/*.d.ts
 infrastructure/node_modules
+!infrastructure/canary/nodejs/node_modules/urlMonitor.js
 
 # CDK asset staging directory
 infrastructure/.cdk.staging

build.gradle

@@ -28,6 +28,7 @@ dependencies {
     implementation 'io.github.acm19:aws-request-signing-apache-interceptor:2.3.1'
 
     implementation 'com.amazonaws:aws-lambda-java-core:1.2.3'
+    implementation 'com.amazonaws:aws-lambda-java-events:3.7.0'
 
     implementation 'com.google.code.gson:gson:2.10.1'
 

infrastructure/lib/constructs/canarySns.ts

@@ -0,0 +1,40 @@
+import {SnsMonitors} from "./snsMonitor";
+import {SnsMonitorsProps} from "./snsMonitor";
+import {Construct} from "constructs";
+import {Alarm} from "aws-cdk-lib/aws-cloudwatch";
+import { Canary } from 'aws-cdk-lib/aws-synthetics';
+import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
+
+interface canarySnsProps extends SnsMonitorsProps {
+    readonly canaryAlarms: Array<{ alertName: string, canary: Canary }>;
+}
+
+export class canarySns extends SnsMonitors {
+    private readonly canaryAlarms: Array<{ alertName: string, canary: Canary }>;
+    constructor(scope: Construct, id: string, props: canarySnsProps) {
+        super(scope, id, props);
+        this.canaryAlarms = props.canaryAlarms;
+        this.canaryAlarms.forEach(({ alertName, canary }) =>
+        {
+            const alarm = this.canaryFailed(alertName, canary);
+            this.map[alarm[1]] = alarm[0];
+        });
+        this.createTopic();
+    }
+
+    private canaryFailed(alertName: string, canary: Canary): [Alarm, string] {
+        const alarmObject = new cloudwatch.Alarm(this, `error_alarm_${alertName}`, {
+            metric: canary.metricSuccessPercent(),
+            threshold: 100,
+            evaluationPeriods: 1,
+            comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
+            datapointsToAlarm: 1,
+            treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
+            alarmDescription: "Detect Canary failure",
+            alarmName: alertName,
+        });
+        return [alarmObject, alertName];
+    }
+}
+
+

infrastructure/lib/constructs/snsMonitor.ts

@@ -0,0 +1,64 @@
+import { Construct } from 'constructs';
+import * as sns from "aws-cdk-lib/aws-sns";
+import * as subscriptions from "aws-cdk-lib/aws-sns-subscriptions";
+import * as actions from "aws-cdk-lib/aws-cloudwatch-actions";
+import { Canary } from 'aws-cdk-lib/aws-synthetics';
+import {OpenSearchLambda} from "./lambda";
+
+export interface SnsMonitorsProps {
+    readonly region: string;
+    readonly accountId: string;
+    readonly alarmNameSpace: string;
+    readonly snsTopicName: string;
+    readonly slackLambda: OpenSearchLambda;
+}
+
+export class SnsMonitors extends Construct {
+    protected readonly region: string;
+    protected readonly accountId: string;
+    protected readonly alarmNameSpace: string;
+    protected readonly map: { [id: string]: any };
+    private readonly snsTopicName: string;
+    private readonly slackLambda: OpenSearchLambda;
+    private readonly emailList: Array<string>;
+
+
+    constructor(scope: Construct, id: string, props: SnsMonitorsProps) {
+        super(scope, id);
+        this.region = props.region;
+        this.accountId = props.accountId;
+        this.alarmNameSpace = props.alarmNameSpace;
+        this.snsTopicName = props.snsTopicName;
+        this.slackLambda = props.slackLambda;
+
+        // The email list for receiving alerts
+        this.emailList = [
+            'insert@mail.here'
+        ];
+
+        // Create alarms
+        this.map = {};
+
+    }
+
+    protected createTopic(){
+        // Create SNS topic for alarms to be sent to
+        const snsTopic = new sns.Topic(this, `OpenSearchMetrics-Alarm-${this.snsTopicName}`, {
+            displayName: `OpenSearchMetrics-Alarm-${this.snsTopicName}`
+        });
+
+        // Iterate map to create SNS topic and add alarms on it
+        Object.keys(this.map).map(key => {
+            // Connect the alarm to the SNS
+            this.map[key].addAlarmAction(new actions.SnsAction(snsTopic));
+        })
+
+        // Send email notification to the recipients
+        for (const email of this.emailList) {
+            snsTopic.addSubscription(new subscriptions.EmailSubscription(email));
+        }
+
+        // Send slack notification
+        snsTopic.addSubscription(new subscriptions.LambdaSubscription(this.slackLambda.lambda));
+    }
+}

infrastructure/lib/constructs/stepFunctionSns.ts

@@ -0,0 +1,45 @@
+import {SnsMonitors} from "./snsMonitor";
+import {SnsMonitorsProps} from "./snsMonitor";
+import {Construct} from "constructs";
+import {Alarm} from "aws-cdk-lib/aws-cloudwatch";
+import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
+
+interface stepFunctionSnsProps extends SnsMonitorsProps {
+    readonly stepFunctionSnsAlarms: Array<{ alertName: string, stateMachineName: string }>;
+}
+
+export class StepFunctionSns extends SnsMonitors {
+    private readonly stepFunctionSnsAlarms: Array<{ alertName: string, stateMachineName: string }>;
+    constructor(scope: Construct, id: string, props: stepFunctionSnsProps) {
+        super(scope, id, props);
+        this.stepFunctionSnsAlarms = props.stepFunctionSnsAlarms;
+        this.stepFunctionSnsAlarms.forEach(({ alertName, stateMachineName }) =>
+        {
+            const alarm = this.stepFunctionExecutionsFailed(alertName, stateMachineName);
+            this.map[alarm[1]] = alarm[0];
+        });
+        this.createTopic();
+    }
+
+    private stepFunctionExecutionsFailed(alertName: string, stateMachineName: string): [Alarm, string] {
+        const alarmObject = new cloudwatch.Alarm(this, `error_alarm_${alertName}`, {
+            metric: new cloudwatch.Metric({
+                namespace:  this.alarmNameSpace,
+                metricName: "ExecutionsFailed",
+                statistic: "Sum",
+                dimensionsMap: {
+                    StateMachineArn: `arn:aws:states:${this.region}:${this.accountId}:stateMachine:${stateMachineName}`
+                }
+            }),
+            threshold: 1,
+            evaluationPeriods: 1,
+            comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+            datapointsToAlarm: 1,
+            treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
+            alarmDescription: "Detect SF execution failure",
+            alarmName: alertName,
+        });
+        return [alarmObject, alertName];
+    }
+}
+

infrastructure/lib/infrastructure-stack.ts

@@ -9,6 +9,8 @@ import {OpenSearchMetricsNginxReadonly} from "./stacks/opensearchNginxProxyReado
 import {ArnPrincipal} from "aws-cdk-lib/aws-iam";
 import {OpenSearchWAF} from "./stacks/waf";
 import {OpenSearchMetricsNginxCognito} from "./constructs/opensearchNginxProxyCognito";
+import {OpenSearchMetricsMonitoringStack} from "./stacks/monitoringDashboard";
+import {OpenSearchMetricsSecrets} from "./stacks/secrets";
 
 // import * as sqs from 'aws-cdk-lib/aws-sqs';
 export class InfrastructureStack extends Stack {
@@ -34,12 +36,25 @@ export class InfrastructureStack extends Stack {
       }
     });
 
-
     // Create OpenSearch Metrics Lambda setup
     const openSearchMetricsWorkflowStack = new OpenSearchMetricsWorkflowStack(app, 'OpenSearchMetrics-Workflow', {
       opensearchDomainStack: openSearchDomainStack, vpcStack: vpcStack, lambdaPackage: Project.LAMBDA_PACKAGE})
     openSearchMetricsWorkflowStack.node.addDependency(vpcStack, openSearchDomainStack);
 
+    // Create Secrets Manager
+
+    const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets");
+
+    // Create Monitoring Dashboard
+
+    const openSearchMetricsMonitoringStack = new OpenSearchMetricsMonitoringStack(app, "OpenSearchMetrics-Monitoring", {
+      region: Project.REGION,
+      account: Project.AWS_ACCOUNT,
+      workflowComponent: openSearchMetricsWorkflowStack.workflowComponent,
+      lambdaPackage: Project.LAMBDA_PACKAGE,
+      secrets: openSearchMetricsSecretsStack.secretsObject,
+      vpcStack: vpcStack
+    });
 
     // Create OpenSearch Metrics Frontend DNS
     const metricsHostedZone = new OpenSearchHealthRoute53(app, "OpenSearchMetrics-HostedZone", {

infrastructure/lib/stacks/metricsWorkflow.ts

@@ -13,7 +13,12 @@ export interface OpenSearchMetricsStackProps extends StackProps {
     readonly vpcStack: VpcStack;
     readonly lambdaPackage: string
 }
+
+export interface WorkflowComponent {
+    opensearchMetricsWorkflowStateMachineName: string
+}
 export class OpenSearchMetricsWorkflowStack extends Stack {
+    public readonly workflowComponent: WorkflowComponent;
     constructor(scope: Construct, id: string, props: OpenSearchMetricsStackProps) {
         super(scope, id, props);
 
@@ -39,6 +44,10 @@ export class OpenSearchMetricsWorkflowStack extends Stack {
             schedule: Schedule.expression('cron(0 7 * * ? *)'),
             targets: [new SfnStateMachine(opensearchMetricsWorkflow)],
         });
+
+        this.workflowComponent = {
+            opensearchMetricsWorkflowStateMachineName: opensearchMetricsWorkflow.stateMachineName
+        }
     }
 
     private createMetricsTask(scope: Construct, opensearchDomainStack: OpenSearchDomainStack,