Merge pull request #15 from Bayer-Group/dev2023

Dev2023

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog for BayerCLAW
 
+## [v1.2.0] 2023-05-24 Feature release
+
+**IMPORTANT:** Because of changes to the workflow specification language, as well as new features
+surrounding Batch job scheduling and operation, this version of BayerCLAW is not backward compatible
+with previous versions. It is recommended that users install this version alongside any previous
+installation so existing workflows can continue running until they can be converted.
+
+### Added
+- Compile-time workflow parameters.
+- Versioned workflows and Blue/Green workflow deployment.
+- Fair share batch job scheduling.
+- Improvements to Subpipes make them operate more like regular workflows.
+- Increased ability to update Batch Compute Environment parameters.
+- In Batch jobs, CloudWatch log messages originating from user commands
+are labeled with "USER_CMD".
+- Streamlined the Lambda that handles the Gather end of Scatter/Gather.
+- Optionally add log subscription filters to CloudWatch logs.
+
+### Removed
+- The global `params` has been removed to prevent confusion with thw new `Parameters` block. The
+`params.repository` field has been made into a top-level key in the workflow spec.
+- Stepwise `params` blocks (which have been deprecated since before v1.0.0) removed.
+- The `s3_request_id` field of the Step Function execution document has been removed.
+
 ## [v1.1.4] 2022-09-26 Feature release
 
 ### Added

bclaw_runner/Dockerfile.alpine

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/python:3.10.7-alpine3.16 AS base
+FROM public.ecr.aws/docker/library/python:3.10-alpine AS base
 
 LABEL maintainer="jack.tabaska@bayer.com"
 
@@ -26,5 +26,8 @@ RUN pytest -s -vvv tests/
 
 FROM base AS build
 
+ARG BC_VERSION_ARG
+ENV BC_VERSION=$BC_VERSION_ARG
+
 ENV PYTHONBUFFERED=1
-ENV PATH=/bclaw:$PATH
+ENV PATH=/bclaw_runner:$PATH

bclaw_runner/src/runner/custom_logs.py

@@ -2,7 +2,7 @@
 import logging
 import os
 
-from .version import VERSION
+# from .version import VERSION
 
 
 class JSONFormatter(logging.Formatter):
@@ -20,10 +20,8 @@ def format(self, record: logging.LogRecord):
             },
             "sfn_execution_id": os.environ.get("BC_EXECUTION_ID"),
             "branch": os.environ.get("BC_BRANCH_IDX"),
-            "batch": {
-                "runner": f"bclaw_runner v{VERSION}",
-                "job_id": os.environ.get("AWS_BATCH_JOB_ID"),
-            },
+            "batch_job_id": os.environ.get("AWS_BATCH_JOB_ID"),
+            "bclaw_version": os.environ.get("BC_VERSION"),
         }
         if record.exc_info is not None:
             obj["exception"] = self.formatException(record.exc_info)

bclaw_runner/src/runner/dind.py

@@ -71,6 +71,12 @@ def get_mounts(metadata: dict, parent_workspace: str, child_workspace: str) -> G
 
         elif "DockerName" in volume_spec:
             # this handles per-job EFS mounts
+            # template:
+            #   efs_id: fs-b5a4dd01
+            #   host_path: /efs
+            # volume_spec:
+            #   {'DockerName': 'ecs-jax-efs-test-EfsTestJobDef--1-1-fs-b5a4dd01-volume-bcd3a18fd8e28a9fe901',
+            #    'Destination': '/efs'}
             yield Mount(volume_spec["Destination"], volume_spec["DockerName"], type="volume", read_only=True,
                         driver_config=DriverConfig("amazon-ecs-volume-plugin"))
 
@@ -133,7 +139,8 @@ def run_child_container(image_tag: str, command: str, parent_workspace: str, par
             try:
                 with closing(container.logs(stream=True)) as fp:
                     for line in fp:
-                        logger.info(line.decode("utf-8"))
+                        logger.user_cmd(line.decode("utf-8"))
+                        # logger.info(line.decode("utf-8"))
 
             except Exception:
                 logger.exception("----- error during subprocess logging: ")

bclaw_runner/src/runner/runner_main.py

@@ -17,8 +17,10 @@
     --version            show version
 """
 
+from functools import partial, partialmethod
 import json
 import logging.config
+import os
 from typing import Dict, List
 
 from docopt import docopt
@@ -29,7 +31,7 @@
 from .repo import Repository
 from .tagging import tag_this_instance
 from .termination import spot_termination_checker
-from .version import VERSION
+# from .version import VERSION
 from .workspace import workspace, write_job_data_file, run_commands
 
 
@@ -109,8 +111,16 @@ def main(commands: List[str],
 
 def cli() -> int:
     tag_this_instance()
+
+    # create custom log level for user commands
+    # https://stackoverflow.com/a/55276759
+    logging.USER_CMD = logging.INFO + 5  # between INFO and WARNING
+    logging.addLevelName(logging.USER_CMD, "USER_CMD")
+    logging.Logger.user_cmd = partialmethod(logging.Logger.log, logging.USER_CMD)
+    logging.user_cmd = partial(logging.log, logging.USER_CMD)
+
     with spot_termination_checker():
-        args = docopt(__doc__, version=VERSION)
+        args = docopt(__doc__, version=os.environ["BC_VERSION"])
 
         logger.info(f"{args = }")
 

bclaw_runner/src/runner/string_subs.py

@@ -28,8 +28,9 @@ def substitute(target: Any, spec: dict) -> Any:
 
 
 def substitute_image_tag(image_tag: str, spec: dict) -> str:
-    name, *version_tag = image_tag.rsplit(":", 1)
+    parts = image_tag.split("/")
+    name_ver = parts.pop(-1)
     _lookup = partial(lookup, spec=spec)
-    subbed = [SUB_FINDER.sub(_lookup, v) for v in version_tag]
-    ret = ":".join([name] + subbed)
+    subbed = SUB_FINDER.sub(_lookup, name_ver)
+    ret = "/".join(parts + [subbed])
     return ret

bclaw_runner/tests/test_custom_logs.py

@@ -2,7 +2,7 @@
 import logging
 
 from ..src.runner.custom_logs import JSONFormatter
-from ..src.runner.version import VERSION
+# from ..src.runner.version import VERSION
 
 
 def test_JSONFormatter(monkeypatch):
@@ -12,6 +12,7 @@ def test_JSONFormatter(monkeypatch):
     monkeypatch.setenv("BC_LAUNCH_VERSION", "testVersion")
     monkeypatch.setenv("BC_LAUNCH_BUCKET", "testLaunchBucket")
     monkeypatch.setenv("BC_STEP_NAME", "testStepName")
+    monkeypatch.setenv("BC_VERSION", "v1.2.3")
     monkeypatch.setenv("BC_WORKFLOW_NAME", "testWorkflowName")
     monkeypatch.setenv("AWS_BATCH_JOB_ID", "0987654321")
     record = logging.LogRecord(
@@ -27,10 +28,8 @@ def test_JSONFormatter(monkeypatch):
     result_str = formatter.format(record)
     result = json.loads(result_str)
     expect = {
-        "batch": {
-            "runner": f"bclaw_runner v{VERSION}",
-            "job_id": "0987654321",
-        },
+        "batch_job_id": "0987654321",
+        "bclaw_version": f"v1.2.3",
         "branch": "99",
         "function": "test_pathname.test_func",
         "job_file": {

bclaw_runner/tests/test_repo.py

@@ -300,7 +300,6 @@ def test_download_inputs_empty_inputs(monkeypatch, mock_buckets):
     assert len(result) == 0
 
 
-# todo: recursive glob
 def test_outputerator(monkeypatch, tmp_path, caplog):
     monkeypatch.setenv("BC_STEP_NAME", "test_step")
     repo = Repository(f"s3://{TEST_BUCKET}/repo/path")

bclaw_runner/tests/test_runner_main.py

@@ -311,7 +311,7 @@ def test_cli(capsys, requests_mock, mock_ec2_instance, monkeypatch, argv, expect
     monkeypatch.setenv("BC_WORKFLOW_NAME", "testWorkflowName")
     monkeypatch.setenv("BC_STEP_NAME", "test:step:name")
     monkeypatch.setenv("BC_JOB_NAME", "test*job")
-    monkeypatch.setenv("BC_S3_REQUEST_ID", "12345ELVISLIVES")
+    monkeypatch.setenv("BC_VERSION", "v1.2.3")
     monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
     monkeypatch.setattr("sys.argv", argv.split())
     monkeypatch.setattr("bclaw_runner.src.runner.runner_main.main", fake_main)

bclaw_runner/tests/test_string_subs.py

@@ -129,11 +129,13 @@ def test_substitute_recursion():
 
 @pytest.mark.parametrize("original, expect", [
     ("docker.io/library/single:${sub}", "docker.io/library/single:tag"),
+    ("no_${a}_registry:${sub}", "no_eh_registry:tag"),
+    ("no_registry:no_subs", "no_registry:no_subs"),
     ("public.ecr.aws/docker/library/multi:${a}_${b}_${c}", "public.ecr.aws/docker/library/multi:eh_bee_sea"),
     ("123456789012.dkr.ecr.us-east-1.amazonaws.com/no:subs", "123456789012.dkr.ecr.us-east-1.amazonaws.com/no:subs"),
     ("123456789012.dkr.ecr.us-east-1.amazonaws.com/no_tags", "123456789012.dkr.ecr.us-east-1.amazonaws.com/no_tags"),
     ("myregistryhost:5000/fedora/httpd:${sub}", "myregistryhost:5000/fedora/httpd:tag"),  # https://docs.docker.com/engine/reference/commandline/tag/#tag-an-image-for-a-private-repository
-    ("probably:${a}/highly/${b}/illegal/${c}:${sub}", "probably:${a}/highly/${b}/illegal/${c}:tag"),
+    ("probably:${a}/highly/${b}/illegal/${c}:${sub}", "probably:${a}/highly/${b}/illegal/sea:tag"),
 ])
 def test_substitute_image_tag(original, expect):
     spec = {

buildspec.yml

@@ -1,77 +1,97 @@
 version: 0.2
 
 env:
+  variables:
+    DOCKER_BUILDKIT: 1
   exported-variables:
     - CORE_STACK_NAME
+    - SOURCE_VERSION
 
 phases:
   install:
     runtime-versions:
       python: 3.9
     commands:
-      - pip install aws-sam-cli
       - pip install -r lambda/tests/requirements.txt
 
   pre_build:
     commands:
-      - pytest -s -vvv lambda/tests/
-      - export CORE_STACK_NAME=${INSTALLATION_NAME}-core
-      - export ECR_REPO_NAME=${CORE_STACK_NAME}/bclaw_runner
+      - PYTHONPATH=${CODEBUILD_SRC_DIR}/lambda/src/common/python:$PYTHONPATH pytest -s -vvv lambda/tests/
       - export ACCOUNT_ID=$(echo $CODEBUILD_BUILD_ARN | cut -d':' -f5)
-      - aws ecr create-repository --repository-name $ECR_REPO_NAME || true
-      - >
-        export ECR_REPO_URI=$(aws ecr describe-repositories
-        --repository-name $ECR_REPO_NAME
-        --query "repositories[0].repositoryUri"
-        --output text)
+      - export CORE_STACK_NAME=${INSTALLATION_NAME}-core
+      - export SOURCE_VERSION=$(git describe --tags)
+      - export BATCH_FILTER=$(aws logs describe-subscription-filters --log-group-name "/aws/batch/job" --query "subscriptionFilters[].filterName" --output text || true)
 
   build:
     commands:
-      - cd bclaw_runner
-      - docker build --target test -f Dockerfile.alpine .
-      - docker build -t $ECR_REPO_URI -f Dockerfile.alpine .
-
-      - cd $CODEBUILD_SRC_DIR
       - sam build -b ./build -s . -t cloudformation/bc_core.yaml
       - >
-        sam package
+        sam deploy
         --template-file build/template.yaml
+        --stack-name ${CORE_STACK_NAME}
         --s3-bucket ${RESOURCE_BUCKET_NAME}
         --s3-prefix lambda
-        --output-template-file build/packaged.yaml
-      - >
-        sam deploy
-        --template-file build/packaged.yaml
-        --stack-name ${CORE_STACK_NAME}
         --capabilities CAPABILITY_NAMED_IAM
-        --s3-prefix lambda
         --no-fail-on-empty-changeset
+        --tags "bclaw:version=${SOURCE_VERSION}"
         --parameter-overrides
-        AmiId=${AMI_ID_SSM}
+        AmiId=${AMI_ID:-auto}
         CompilerMacroName=${COMPILER_MACRO_NAME}
+        ExistingBatchSubscriptionFilter="${BATCH_FILTER:-none}"
         InstallationName=${INSTALLATION_NAME}
         LauncherBucketName=${LAUNCHER_BUCKET_NAME}
         LogRetentionDays=${LOG_RETENTION_DAYS}
+        LoggingDestinationArn=${LOGGING_DESTINATION}
         MaxvCpus=${MAX_VCPUS}
         MinvCpus=${MIN_VCPUS}
         ResourceBucketName=${RESOURCE_BUCKET_NAME}
         RootVolumeSize=${ROOT_VOLUME_SIZE}
-        RunnerImageURI=${ECR_REPO_URI}
         ScratchVolumeSize=${SCRATCH_VOLUME_SIZE}
         SecurityGroups=${SECURITY_GROUPS}
+        SourceVersion=${SOURCE_VERSION}
         Subnets=${SUBNETS}
         Uniqifier=$(date | md5sum | head -c 16)
         VpcId=${VPC_ID}
 
+      - export RUNNER_IMAGE_TAG=$(aws cloudformation describe-stacks --query "Stacks[?StackName=='$CORE_STACK_NAME'][].Outputs[?OutputKey=='RunnerImageUri'].OutputValue" --output text)
+      - echo $RUNNER_IMAGE_TAG
+      - export LAUNCHER_IMAGE_TAG=$(aws cloudformation describe-stacks --query "Stacks[?StackName=='$CORE_STACK_NAME'][].Outputs[?OutputKey=='JobLauncherImageUri'].OutputValue" --output text)
+      - echo $LAUNCHER_IMAGE_TAG
+
+      - cd $CODEBUILD_SRC_DIR/bclaw_runner
+      - >
+        docker build 
+        --build-arg BUILDKIT_INLINE_CACHE=1
+        --target test -f Dockerfile.alpine .
+      - >
+        docker build 
+        --build-arg BUILDKIT_INLINE_CACHE=1
+        --build-arg BC_VERSION_ARG=${SOURCE_VERSION}
+        -t $RUNNER_IMAGE_TAG -f Dockerfile.alpine .
+
+      - cd $CODEBUILD_SRC_DIR/lambda
+      - > 
+        docker build 
+        --build-arg BUILDKIT_INLINE_CACHE=1
+        --target test -f src/job_launcher/Dockerfile .
+      - >
+        docker build
+        --build-arg BUILDKIT_INLINE_CACHE=1
+        --build-arg BC_VERSION_ARG=${SOURCE_VERSION}
+        -t $LAUNCHER_IMAGE_TAG -f src/job_launcher/Dockerfile .
+
   post_build:
     commands:
       - >
         aws ecr get-login-password --region ${AWS_REGION} |
         docker login --username AWS --password-stdin ${ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com
-      - docker push ${ECR_REPO_URI}
+
+      # don't crash on tag collision
+      - docker push ${RUNNER_IMAGE_TAG} || true
+      - docker push ${LAUNCHER_IMAGE_TAG} || true
 
 artifacts:
   discard-paths: yes
   files:
     - cloudformation/wf_launcher.yaml
-    - cloudformation/wf_notifications.yaml
+    - cloudformation/wf_deploy.yaml