chore: Xmas24 updates (#29)

* Create README_WORKFLOW.md

* drop fix_deprecation_warning

Not needed, and causes errors

* cdk version 173

* split sentinel file into prefix/suffix

* explicit omics status

* Update poetry.lock

* boostrap reminder

---------

Co-authored-by: Dr. Ernie Prabhakar <19791+drernie@users.noreply.github.com>

.projenrc.ts

@@ -1,6 +1,6 @@
 import { awscdk } from 'projen';
 
-const cdkVersion = '2.114.1';
+const cdkVersion = '2.173.2';
 const solutionName = 'omics-quilt-demo';
 const project = new awscdk.AwsCdkTypeScriptApp({
   cdkVersion: cdkVersion,
@@ -35,7 +35,6 @@ const project = new awscdk.AwsCdkTypeScriptApp({
   ],
 });
 override_file_key('.github/workflows/build.yml', 'jobs.build.env');
-fix_deprecation_warning();
 /*
 const appTestTask = project.addTask('pytest', {
   cwd: 'src/packager',
@@ -58,13 +57,3 @@ function override_file_key(file: string, key: string) {
 
   project.tryFindObjectFile(file)?.addOverride(key, opts);
 }
-
-// Fix Jest 29 warning about deprecated config in `globals`
-function fix_deprecation_warning() {
-  project.jest!.config.transform ??= {};
-  project.jest!.config.transform['\\.ts$'] = [
-    'ts-jest',
-    project.jest?.config.globals['ts-jest'],
-  ];
-  delete project.jest!.config.globals['ts-jest'];
-}

README.md

@@ -7,14 +7,17 @@ Use CDK to create Quilt packages from AWS HealthOmics
 ### Installation
 
 Use CDK to create and deploy the stack.
+Note that it requires a large Cloud9 instance to run Docker.
 
 ```bash
 cp example.env .env # and edit
+source .env
 aws configure list-profiles # verify AWS credentials exist
 # npx npm install # if npm not present (but npx is)
 npm install yarn -g # if yarn not present (but npm is)
 yarn install
-npx cdk bootstrap # if not yet done for this account/region
+# set bootstrap region (if new or changed)
+npx cdk bootstrap aws://$CDK_DEFAULT_ACCOUNT/$CDK_DEFAULT_REGION
 # start Docker if not already running
 sudo systemctl start docker # e.g. on Linux (requires large Cloud9 instance!)
 npm run deploy

README_WORKFLOW.md

@@ -0,0 +1,16 @@
+# Workflow Sequest
+
+What happens when you run the workflow?
+
+1. User uploads (or updates) `input_metadata.json` file in the `fastq/region` folder in the input bucket
+2. An `S3EventSource` watching for that files triggers the `fastqLambda` to `run_workflow` and `save_metadata` to the output bucket.
+3. When run, FASTQ generates the output file `bqsr_report/*.hg38.recal_data.csv`
+4. This acts as a sentinel file to trigger the `packager` lambda, which creates
+   a package in the output bucket from everything in the parent of
+   `bqsr_report`.
+
+## Open Issues
+
+- Why can't we just use the `omics` events to trigger the packager?
+- S3 notifications for the output bucket are not working.
+  The package is created, but we don't see it the Packages view.

src/constants.ts

@@ -20,7 +20,8 @@ export class Constants {
     MANIFEST_SUFFIX: '.json',
     INPUT_METADATA: 'input_metadata.json',
     QUILT_METADATA: 'quilt_metadata.json',
-    FASTQ_SENTINEL: 'out/bqsr_report/NA12878.hg38.recal_data.csv',
+    FASTQ_PREFIX: 'out/bqsr_report/',
+    FASTQ_SUFFIX: '.hg38.recal_data.csv',
     MEMORY_SIZE: 8192,
     TIMEOUT: '900',
   };

src/omics-quilt.ts

@@ -52,7 +52,8 @@ export class OmicsQuiltStack extends Stack {
 
   public readonly manifest_prefix: string;
   public readonly manifest_suffix: string;
-  public readonly packager_sentinel: string;
+  public readonly packager_prefix: string;
+  public readonly packager_suffix: string;
 
   readonly cc: Constants;
   readonly lambdaRole: Role;
@@ -66,7 +67,8 @@ export class OmicsQuiltStack extends Stack {
     const manifest_root = this.cc.get('MANIFEST_ROOT');
     this.manifest_prefix = `${manifest_root}/${this.cc.region}`;
     this.manifest_suffix = this.cc.get('MANIFEST_SUFFIX');
-    this.packager_sentinel = this.cc.get('FASTQ_SENTINEL');
+    this.packager_prefix = this.cc.get('FASTQ_PREFIX');
+    this.packager_suffix = this.cc.get('FASTQ_SUFFIX');
 
     // Create Input/Output S3 buckets
     this.inputBucket = this.makeBucket('input');
@@ -100,7 +102,7 @@ export class OmicsQuiltStack extends Stack {
     const packagerTrigger = new S3EventSource(this.outputBucket, {
       events: [EventType.OBJECT_CREATED],
       filters: [
-        { suffix: this.packager_sentinel },
+        { prefix: this.packager_prefix, suffix: this.packager_suffix },
       ],
     });
     packagerLambda.addEventSource(packagerTrigger);
@@ -148,17 +150,15 @@ export class OmicsQuiltStack extends Stack {
   }
 
   private makeOmicsEventRule(ruleName: string) {
-    const ruleOmics = new Rule(this, ruleName,
-      {
-        eventPattern: {
-          source: ['aws.omics'],
-          detailType: ['Run Status Change'],
-          detail: {
-            status: ['*'],
-          },
+    const ruleOmics = new Rule(this, ruleName, {
+      eventPattern: {
+        source: ['aws.omics'],
+        detailType: ['Run Status Change'],
+        detail: {
+          status: ['PENDING', 'STARTING', 'RUNNING', 'COMPLETED', 'FAILED', 'CANCELLED', 'STOPPING', 'DELETED'],
         },
       },
-    );
+    });
     return ruleOmics;
   }
 
@@ -230,12 +230,14 @@ export class OmicsQuiltStack extends Stack {
     const output = ['s3:/', this.outputBucket.bucketName, this.cc.app];
     const input = ['s3:/', this.inputBucket.bucketName, this.manifest_prefix];
     const final_env = {
+      AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
       ECR_REGISTRY: this.cc.getEcrRegistry(),
       INPUT_S3_LOCATION: input.join('/'),
       LOG_LEVEL: 'ALL',
       OMICS_ROLE: this.omicsRole.roleArn,
       OUTPUT_S3_LOCATION: output.join('/'),
-      SENTINEL_FILE: this.packager_sentinel,
+      SENTINEL_PREFIX: this.packager_prefix,
+      SENTINEL_SUFFIX: this.packager_suffix,
       INPUT_METADATA: this.cc.get('INPUT_METADATA'),
       QUILT_METADATA: this.cc.get('QUILT_METADATA'),
       WORKFLOW_ID: this.cc.get('READY2RUN_WORKFLOW_ID'),

src/packager/packager/constants.py

@@ -21,14 +21,13 @@ class Constants:
         "APP_NAME": "packager",
         "CDK_DEFAULT_EMAIL": "test@example.com",
         "CDK_DEFAULT_REGION": "us-east-1",
-        "SENTINEL_BUCKET": "data-yaml-spec-tests",
-        "SENTINEL_FILE": "quilt_metadata.json",
+        # "SENTINEL_BUCKET": "data-yaml-spec-tests",
+        # "SENTINEL_FILE": "quilt_metadata.json",
         "TIMESTAMP_FILE": "quilt_timestamp.json",
         "SOURCE_APP": "omics-quilt",
         "QUILT_METADATA": "quilt_metadata.json",
         "INPUT_METADATA": "input_metadata.json",
         "QUILT_SUMMARIZE": "quilt_summarize.json",
-        "FASTQ_SENTINEL": "out/bqsr_report/NA12878.hg38.recal_data.csv",
     }
 
     @classmethod