From 1049772cad9a127ed073e840b2f322fe84de97b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Feb 2022 19:15:55 +0000 Subject: [PATCH 01/23] Bump boto3 from 1.20.48 to 1.20.50 Bumps [boto3](https://github.com/boto/boto3) from 1.20.48 to 1.20.50. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.20.48...1.20.50) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index b0705e38c..73f266d1e 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -3,7 +3,7 @@ -r requirements-apps-scale-cluster.txt -r requirements-apps-start-execution.txt -r requirements-apps-update-db.txt -boto3==1.20.48 +boto3==1.20.50 jinja2==3.0.3 moto==1.3.14 pytest==7.0.0 From 1aa03c57872978e188c72f4b0be3c12c7007f012 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Feb 2022 19:16:29 +0000 Subject: [PATCH 02/23] Bump pytest from 7.0.0 to 7.0.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.0.0 to 7.0.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.0.0...7.0.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 73f266d1e..5c64822fd 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -6,7 +6,7 @@ boto3==1.20.50 jinja2==3.0.3 moto==1.3.14 -pytest==7.0.0 +pytest==7.0.1 PyYAML==6.0 responses==0.18.0 flake8==4.0.1 From 64188fdc3aac8dd606e6a19121ddace244bcfd22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Feb 2022 20:01:42 +0000 Subject: [PATCH 03/23] Bump boto3 from 1.20.50 to 1.20.53 Bumps [boto3](https://github.com/boto/boto3) from 1.20.50 to 1.20.53. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.20.50...1.20.53) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 5c64822fd..b0060c609 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -3,7 +3,7 @@ -r requirements-apps-scale-cluster.txt -r requirements-apps-start-execution.txt -r requirements-apps-update-db.txt -boto3==1.20.50 +boto3==1.20.53 jinja2==3.0.3 moto==1.3.14 pytest==7.0.1 From 128c84e728dd2d0fc6d3eb78123bf6ca3ff947bf Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 11 Feb 2022 16:45:25 -0900 Subject: [PATCH 04/23] Pass tests for check_processing_time --- Makefile | 3 +- .../src/check_processing_time.py | 4 +++ tests/test_check_processing_time.py | 33 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 apps/check-processing-time/src/check_processing_time.py create mode 100644 tests/test_check_processing_time.py diff --git a/Makefile b/Makefile index 242a15cc6..0c2bc8e5e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ API = ${PWD}/apps/api/src +CHECK_PROCESSING_TIME = ${PWD}/apps/check-processing-time/src GET_FILES = ${PWD}/apps/get-files/src PROCESS_NEW_GRANULES = ${PWD}/apps/process-new-granules/src SCALE_CLUSTER = ${PWD}/apps/scale-cluster/src @@ -6,7 +7,7 @@ START_EXECUTION = ${PWD}/apps/start-execution/src UPDATE_DB = ${PWD}/apps/update-db/src UPLOAD_LOG = ${PWD}/apps/upload-log/src DYNAMO = ${PWD}/lib/dynamo -export PYTHONPATH = ${API}:${GET_FILES}:${PROCESS_NEW_GRANULES}:${SCALE_CLUSTER}:${START_EXECUTION}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} +export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${PROCESS_NEW_GRANULES}:${SCALE_CLUSTER}:${START_EXECUTION}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} build: render diff --git a/apps/check-processing-time/src/check_processing_time.py b/apps/check-processing-time/src/check_processing_time.py new file mode 100644 index 000000000..7604a154d --- /dev/null +++ b/apps/check-processing-time/src/check_processing_time.py @@ -0,0 +1,4 @@ +def lambda_handler(event, context): + event['Attempts'].sort(key=lambda attempt: attempt['StartedAt']) + final_attempt = event['Attempts'][-1] + return final_attempt['StoppedAt'] - final_attempt['StartedAt'] diff --git a/tests/test_check_processing_time.py b/tests/test_check_processing_time.py new file mode 100644 index 000000000..25eb31f01 --- /dev/null +++ b/tests/test_check_processing_time.py @@ -0,0 +1,33 @@ +import check_processing_time + + +def test_single_attempt(): + event = { + 'Attempts': [{'Container': {}, 'StartedAt': 1644423555778, 'StatusReason': '', 'StoppedAt': 1644425055200}] + } + response = check_processing_time.lambda_handler(event, None) + assert response == 1644425055200 - 1644423555778 + + +def test_multiple_attempts(): + event = { + 'Attempts': [ + {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331}, + {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015} + ] + } + response = check_processing_time.lambda_handler(event, None) + assert response == 1644611472015 - 1644610107570 + + +def test_unsorted_attempts(): + # I'm not sure if the lambda would ever be given unsorted attempts, but it seems worth testing that it doesn't + # depend on the list to be sorted. + event = { + 'Attempts': [ + {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015}, + {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331} + ] + } + response = check_processing_time.lambda_handler(event, None) + assert response == 1644611472015 - 1644610107570 From 35d6954138c3012614bf4891520a90cb6d19968c Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 14 Feb 2022 11:54:24 -0900 Subject: [PATCH 05/23] Change check_processing_time input structure --- .../src/check_processing_time.py | 5 ++-- tests/test_check_processing_time.py | 24 ++++++++++++------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/apps/check-processing-time/src/check_processing_time.py b/apps/check-processing-time/src/check_processing_time.py index 7604a154d..1966642e3 100644 --- a/apps/check-processing-time/src/check_processing_time.py +++ b/apps/check-processing-time/src/check_processing_time.py @@ -1,4 +1,5 @@ def lambda_handler(event, context): - event['Attempts'].sort(key=lambda attempt: attempt['StartedAt']) - final_attempt = event['Attempts'][-1] + results = event['processing_results'] + results['Attempts'].sort(key=lambda attempt: attempt['StartedAt']) + final_attempt = results['Attempts'][-1] return final_attempt['StoppedAt'] - final_attempt['StartedAt'] diff --git a/tests/test_check_processing_time.py b/tests/test_check_processing_time.py index 25eb31f01..f3a6181bf 100644 --- a/tests/test_check_processing_time.py +++ b/tests/test_check_processing_time.py @@ -3,7 +3,9 @@ def test_single_attempt(): event = { - 'Attempts': [{'Container': {}, 'StartedAt': 1644423555778, 'StatusReason': '', 'StoppedAt': 1644425055200}] + 'processing_results': { + 'Attempts': [{'Container': {}, 'StartedAt': 1644423555778, 'StatusReason': '', 'StoppedAt': 1644425055200}] + } } response = check_processing_time.lambda_handler(event, None) assert response == 1644425055200 - 1644423555778 @@ -11,10 +13,12 @@ def test_single_attempt(): def test_multiple_attempts(): event = { - 'Attempts': [ - {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331}, - {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015} - ] + 'processing_results': { + 'Attempts': [ + {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331}, + {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015} + ] + } } response = check_processing_time.lambda_handler(event, None) assert response == 1644611472015 - 1644610107570 @@ -24,10 +28,12 @@ def test_unsorted_attempts(): # I'm not sure if the lambda would ever be given unsorted attempts, but it seems worth testing that it doesn't # depend on the list to be sorted. event = { - 'Attempts': [ - {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015}, - {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331} - ] + 'processing_results': { + 'Attempts': [ + {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015}, + {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331} + ] + } } response = check_processing_time.lambda_handler(event, None) assert response == 1644611472015 - 1644610107570 From 45569c8e73c925df3386c29c5ced3a667e3c0f68 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 14 Feb 2022 12:48:14 -0900 Subject: [PATCH 06/23] Check processing time for failed results --- .../src/check_processing_time.py | 15 +++++-- tests/test_check_processing_time.py | 45 ++++++++++++------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/apps/check-processing-time/src/check_processing_time.py b/apps/check-processing-time/src/check_processing_time.py index 1966642e3..68da7964e 100644 --- a/apps/check-processing-time/src/check_processing_time.py +++ b/apps/check-processing-time/src/check_processing_time.py @@ -1,5 +1,14 @@ +import json + + +def get_time_from_attempts(attempts): + attempts.sort(key=lambda attempt: attempt['StartedAt']) + final_attempt = attempts[-1] + return final_attempt['StoppedAt'] - final_attempt['StartedAt'] + + def lambda_handler(event, context): results = event['processing_results'] - results['Attempts'].sort(key=lambda attempt: attempt['StartedAt']) - final_attempt = results['Attempts'][-1] - return final_attempt['StoppedAt'] - final_attempt['StartedAt'] + if 'Attempts' in results: + return get_time_from_attempts(results['Attempts']) + return get_time_from_attempts(json.loads(results['Cause'])['Attempts']) diff --git a/tests/test_check_processing_time.py b/tests/test_check_processing_time.py index f3a6181bf..9eb0ad7cb 100644 --- a/tests/test_check_processing_time.py +++ b/tests/test_check_processing_time.py @@ -2,16 +2,32 @@ def test_single_attempt(): - event = { - 'processing_results': { - 'Attempts': [{'Container': {}, 'StartedAt': 1644423555778, 'StatusReason': '', 'StoppedAt': 1644425055200}] - } - } - response = check_processing_time.lambda_handler(event, None) - assert response == 1644425055200 - 1644423555778 + attempts = [{'Container': {}, 'StartedAt': 1644423555778, 'StatusReason': '', 'StoppedAt': 1644425055200}] + result = check_processing_time.get_time_from_attempts(attempts) + assert result == 1644425055200 - 1644423555778 def test_multiple_attempts(): + attempts = [ + {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331}, + {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015} + ] + result = check_processing_time.get_time_from_attempts(attempts) + assert result == 1644611472015 - 1644610107570 + + +def test_unsorted_attempts(): + # I'm not sure if the lambda would ever be given unsorted attempts, but it seems worth testing that it doesn't + # depend on the list to be sorted. + attempts = [ + {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015}, + {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331} + ] + result = check_processing_time.get_time_from_attempts(attempts) + assert result == 1644611472015 - 1644610107570 + + +def test_lambda_handler_with_normal_results(): event = { 'processing_results': { 'Attempts': [ @@ -24,16 +40,15 @@ def test_multiple_attempts(): assert response == 1644611472015 - 1644610107570 -def test_unsorted_attempts(): - # I'm not sure if the lambda would ever be given unsorted attempts, but it seems worth testing that it doesn't - # depend on the list to be sorted. +def test_lambda_handler_with_failed_results(): event = { 'processing_results': { - 'Attempts': [ - {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015}, - {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331} - ] + 'Error': 'States.TaskFailed', + 'Cause': '{"Attempts": [' + '{"Container": {}, "StartedAt": 1643834765893, "StatusReason": "", "StoppedAt": 1643834766455}, ' + '{"Container": {}, "StartedAt": 1643834888866, "StatusReason": "", "StoppedAt": 1643834889448}, ' + '{"Container": {}, "StartedAt": 1643834907858, "StatusReason": "", "StoppedAt": 1643834908466}]}' } } response = check_processing_time.lambda_handler(event, None) - assert response == 1644611472015 - 1644610107570 + assert response == 1643834908466 - 1643834907858 From ea7244738db887c206a11c76065d54dd74cd3584 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 14 Feb 2022 14:37:14 -0900 Subject: [PATCH 07/23] Add CHECK_PROCESSING_TIME step to workflow --- .../check-processing-time-cf.yml.j2 | 84 +++++++++++++++++++ apps/step-function.json.j2 | 24 +++++- apps/workflow-cf.yml.j2 | 10 +++ 3 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 apps/check-processing-time/check-processing-time-cf.yml.j2 diff --git a/apps/check-processing-time/check-processing-time-cf.yml.j2 b/apps/check-processing-time/check-processing-time-cf.yml.j2 new file mode 100644 index 000000000..499027702 --- /dev/null +++ b/apps/check-processing-time/check-processing-time-cf.yml.j2 @@ -0,0 +1,84 @@ +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + + PermissionsBoundaryPolicyArn: + Type: String + Default: "" + + SecurityGroupId: + Type: String + + SubnetIds: + Type: CommaDelimitedList + +Conditions: + + UsePermissionsBoundary: !Not [!Equals [!Ref PermissionsBoundaryPolicyArn, ""]] + + LambdasInVpc: !Not [!Equals [!Ref SecurityGroupId, ""]] + +Outputs: + + LambdaArn: + Value: !GetAtt Lambda.Arn + +Resources: + + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub "/aws/lambda/${Lambda}" + RetentionInDays: 90 + + Role: + Type: {{ 'Custom::JplRole' if security_environment == 'JPL' else 'AWS::IAM::Role' }} + Properties: + {% if security_environment == 'JPL' %} + ServiceToken: !ImportValue Custom::JplRole::ServiceToken + Path: /account-managed/hyp3/ + {% endif %} + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + Action: sts:AssumeRole + Principal: + Service: lambda.amazonaws.com + Effect: Allow + PermissionsBoundary: !If [UsePermissionsBoundary, !Ref PermissionsBoundaryPolicyArn, !Ref AWS::NoValue] + ManagedPolicyArns: + - !If [LambdasInVpc, arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole, !Ref AWS::NoValue] + - !Ref Policy + + Policy: + Type: {{ 'Custom::JplPolicy' if security_environment == 'JPL' else 'AWS::IAM::ManagedPolicy' }} + Properties: + {% if security_environment == 'JPL' %} + ServiceToken: !ImportValue Custom::JplPolicy::ServiceToken + Path: /account-managed/hyp3/ + {% endif %} + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - logs:CreateLogStream + - logs:PutLogEvents + Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*" + + Lambda: + Type: AWS::Lambda::Function + Properties: + Code: src/ + Handler: check_processing_time.lambda_handler + MemorySize: 128 + Role: !GetAtt Role.Arn + Runtime: python3.8 + Timeout: 30 + VpcConfig: + !If + - LambdasInVpc + - SecurityGroupIds: + - !Ref SecurityGroupId + SubnetIds: !Ref SubnetIds + - !Ref AWS::NoValue diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 7526aefe8..0ee45ff31 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -181,6 +181,24 @@ } ], "ResultPath": "$.results.get_files", + "Next": "CHECK_PROCESSING_TIME" + }, + "CHECK_PROCESSING_TIME": { + "Type": "Task", + "Resource": "${CheckProcessingTimeLambdaArn}", + "Parameters": { + "processing_results.$": "$.results.processing_results" + }, + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "JOB_FAILED", + "ResultPath": "$.results.processing_time_in_seconds" + } + ], + "ResultPath": "$.results.processing_time_in_seconds", "Next": "CHECK_STATUS" }, "CHECK_STATUS": { @@ -202,7 +220,8 @@ "browse_images.$": "$.results.get_files.browse_images", "thumbnail_images.$": "$.results.get_files.thumbnail_images", "logs.$": "$.results.get_files.logs", - "expiration_time.$": "$.results.get_files.expiration_time" + "expiration_time.$": "$.results.get_files.expiration_time", + "processing_time_in_seconds.$": "$.results.processing_time_in_seconds" }, "Retry": [ { @@ -222,7 +241,8 @@ "job_id.$": "$.job_id", "status_code": "FAILED", "logs.$": "$.results.get_files.logs", - "expiration_time.$": "$.results.get_files.expiration_time" + "expiration_time.$": "$.results.get_files.expiration_time", + "processing_time_in_seconds.$": "$.results.processing_time_in_seconds" }, "Retry": [ { diff --git a/apps/workflow-cf.yml.j2 b/apps/workflow-cf.yml.j2 index e2215306f..50d94497a 100644 --- a/apps/workflow-cf.yml.j2 +++ b/apps/workflow-cf.yml.j2 @@ -86,6 +86,7 @@ Resources: {% endfor %} UpdateDBLambdaArn: !GetAtt UpdateDB.Outputs.LambdaArn GetFilesLambdaArn: !GetAtt GetFiles.Outputs.LambdaArn + CheckProcessingTimeLambdaArn: !GetAtt CheckProcessingTime.Outputs.LambdaArn UploadLogLambdaArn: !GetAtt UploadLog.Outputs.LambdaArn StepFunctionRole: @@ -156,6 +157,15 @@ Resources: SubnetIds: !Join [",", !Ref SubnetIds] TemplateURL: get-files/get-files-cf.yml + CheckProcessingTime: + Type: AWS::CloudFormation::Stack + Properties: + Parameters: + PermissionsBoundaryPolicyArn: !Ref PermissionsBoundaryPolicyArn + SecurityGroupId: !Ref SecurityGroupId + SubnetIds: !Join [",", !Ref SubnetIds] + TemplateURL: check-processing-time/check-processing-time-cf.yml + StartExecution: Type: AWS::CloudFormation::Stack Properties: From 0b63ea75a081c9cf93d329e6be3f002e3b799083 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 14 Feb 2022 15:11:32 -0900 Subject: [PATCH 08/23] Set default processing time to 0 --- apps/step-function.json.j2 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 0ee45ff31..7f36648bd 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -1,6 +1,14 @@ { - "StartAt": "JOB_RUNNING", + "StartAt": "SET_DEFAULT_RESULTS", "States": { + "SET_DEFAULT_RESULTS": { + "Type": "Pass", + "Result": { + "processing_time_in_seconds": 0 + }, + "ResultPath": "$.results", + "Next": "JOB_RUNNING" + }, "JOB_RUNNING": { "Type": "Task", "Resource": "${UpdateDBLambdaArn}", From ee7883c0c0f83df06438ea6b0a49aa90b756ecad Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 14 Feb 2022 15:32:52 -0900 Subject: [PATCH 09/23] Add processing_time_in_seconds to API spec --- apps/api/src/hyp3_api/api-spec/openapi-spec.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml index 6c8084db5..cb76d0a9e 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml @@ -426,6 +426,8 @@ components: $ref: "#/components/schemas/list_of_urls" expiration_time: $ref: "#/components/schemas/datetime" + processing_time_in_seconds: + $ref: "#/components/schemas/seconds" priority: $ref: "#/components/schemas/priority" @@ -523,6 +525,10 @@ components: minimum: 0 maximum: 9999 + seconds: + type: integer + minimum: 0 + securitySchemes: EarthDataLogin: description: |- From 9e18f5d09bb6d3d71c29478dda8748c144e9ff2d Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 14 Feb 2022 15:55:16 -0900 Subject: [PATCH 10/23] Add processing time description in API spec --- apps/api/src/hyp3_api/api-spec/openapi-spec.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml index cb76d0a9e..dd030e130 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml @@ -427,7 +427,7 @@ components: expiration_time: $ref: "#/components/schemas/datetime" processing_time_in_seconds: - $ref: "#/components/schemas/seconds" + $ref: "#/components/schemas/processing_time_in_seconds" priority: $ref: "#/components/schemas/priority" @@ -525,7 +525,11 @@ components: minimum: 0 maximum: 9999 - seconds: + processing_time_in_seconds: + description: > + Run time in seconds for the final processing attempt (regardless of whether it succeeded). A value of zero + likely indicates that the job failed before reaching the processing step, although it may also indicate that the + job failed after reaching the processing step but before calculating the processing time. type: integer minimum: 0 From b90818589dcdce685bc7bdd0bf33be421c3980dd Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 14 Feb 2022 16:14:57 -0900 Subject: [PATCH 11/23] Update Changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b94cdb95..487d84fa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 2.9.0 +### Added +- Add `processing_time_in_seconds` to the `job` API schema to allow plugin developers to check processing time. + ## [2.8.4](https://github.com/ASFHyP3/hyp3/compare/v2.8.3...v2.8.4) ### Security - Encrypt Earthdata username and password using AWS Secrets Manager. From 1681fd4ef6db33ceca5681bf15a9b172d12304ed Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 14 Feb 2022 16:29:14 -0900 Subject: [PATCH 12/23] Allow step function to invoke processing time lambda --- apps/workflow-cf.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/workflow-cf.yml.j2 b/apps/workflow-cf.yml.j2 index 50d94497a..339724370 100644 --- a/apps/workflow-cf.yml.j2 +++ b/apps/workflow-cf.yml.j2 @@ -135,6 +135,7 @@ Resources: Resource: - !GetAtt UpdateDB.Outputs.LambdaArn - !GetAtt GetFiles.Outputs.LambdaArn + - !GetAtt CheckProcessingTime.Outputs.LambdaArn - !GetAtt UploadLog.Outputs.LambdaArn UpdateDB: From 15e8ee020fb1f9a9855cc48cf7f0ce274b60da38 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 14 Feb 2022 16:42:14 -0900 Subject: [PATCH 13/23] Update JPL-deployment.md --- docs/deployments/JPL-deployment.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/deployments/JPL-deployment.md b/docs/deployments/JPL-deployment.md index cc0d23733..e16aee4fa 100644 --- a/docs/deployments/JPL-deployment.md +++ b/docs/deployments/JPL-deployment.md @@ -4,6 +4,11 @@ HyP3 can be deployed into a JPL managed AWS commercial account as long as JPL's tooling is provided in the account and in same region as the deployment. Currently, the only regions supported are `us-west-1`, `us-west-2`, `us-east-1`, and `us-east-2`. +To request `roles-as-code` tooling be deployed in a JPL account, open a +[Cloud Team Service Desk](https://itsd-jira.jpl.nasa.gov/servicedesk/customer/portal/13) request here: +https://itsd-jira.jpl.nasa.gov/servicedesk/customer/portal/13/create/461?q=roles&q_time=1644889220558 + + For more information about `roles-as-code`, see: * https://wiki.jpl.nasa.gov/display/cloudcomputing/IAM+Roles+and+Policies * https://github.jpl.nasa.gov/cloud/roles-as-code/blob/master/Documentation.md From 7934a55f27ed7d58d2f181c379a9aa6be4aa60e3 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 15 Feb 2022 09:40:08 -0900 Subject: [PATCH 14/23] Don't set default params outside of main cf --- apps/check-processing-time/check-processing-time-cf.yml.j2 | 1 - apps/get-files/get-files-cf.yml.j2 | 1 - 2 files changed, 2 deletions(-) diff --git a/apps/check-processing-time/check-processing-time-cf.yml.j2 b/apps/check-processing-time/check-processing-time-cf.yml.j2 index 499027702..a4168ecd0 100644 --- a/apps/check-processing-time/check-processing-time-cf.yml.j2 +++ b/apps/check-processing-time/check-processing-time-cf.yml.j2 @@ -4,7 +4,6 @@ Parameters: PermissionsBoundaryPolicyArn: Type: String - Default: "" SecurityGroupId: Type: String diff --git a/apps/get-files/get-files-cf.yml.j2 b/apps/get-files/get-files-cf.yml.j2 index 75802e8f6..79c6e4578 100644 --- a/apps/get-files/get-files-cf.yml.j2 +++ b/apps/get-files/get-files-cf.yml.j2 @@ -7,7 +7,6 @@ Parameters: PermissionsBoundaryPolicyArn: Type: String - Default: "" SecurityGroupId: Type: String From d69ac2e06e87e92579a79b4d3843b19a18bb49fc Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 15 Feb 2022 09:49:02 -0900 Subject: [PATCH 15/23] Refactor processing time lambda handler --- apps/check-processing-time/src/check_processing_time.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/check-processing-time/src/check_processing_time.py b/apps/check-processing-time/src/check_processing_time.py index 68da7964e..28066e78b 100644 --- a/apps/check-processing-time/src/check_processing_time.py +++ b/apps/check-processing-time/src/check_processing_time.py @@ -10,5 +10,7 @@ def get_time_from_attempts(attempts): def lambda_handler(event, context): results = event['processing_results'] if 'Attempts' in results: - return get_time_from_attempts(results['Attempts']) - return get_time_from_attempts(json.loads(results['Cause'])['Attempts']) + attempts = results['Attempts'] + else: + attempts = json.loads(results['Cause'])['Attempts'] + return get_time_from_attempts(attempts) From 1b9472fedd06b9f81a2e1a6d0ed283860d312410 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Feb 2022 19:15:03 +0000 Subject: [PATCH 16/23] Bump boto3 from 1.20.53 to 1.21.0 Bumps [boto3](https://github.com/boto/boto3) from 1.20.53 to 1.21.0. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.20.53...1.21.0) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index b0060c609..ac7b148b9 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -3,7 +3,7 @@ -r requirements-apps-scale-cluster.txt -r requirements-apps-start-execution.txt -r requirements-apps-update-db.txt -boto3==1.20.53 +boto3==1.21.0 jinja2==3.0.3 moto==1.3.14 pytest==7.0.1 From 057148abe6a56b036c0416f556c1cb3e19acbdd0 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 15 Feb 2022 10:41:43 -0900 Subject: [PATCH 17/23] Convert processing time to seconds --- .../src/hyp3_api/api-spec/openapi-spec.yml | 2 +- .../src/check_processing_time.py | 2 +- tests/test_check_processing_time.py | 20 +++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml index dd030e130..704ed7309 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml @@ -530,7 +530,7 @@ components: Run time in seconds for the final processing attempt (regardless of whether it succeeded). A value of zero likely indicates that the job failed before reaching the processing step, although it may also indicate that the job failed after reaching the processing step but before calculating the processing time. - type: integer + type: number minimum: 0 securitySchemes: diff --git a/apps/check-processing-time/src/check_processing_time.py b/apps/check-processing-time/src/check_processing_time.py index 28066e78b..c000aec5d 100644 --- a/apps/check-processing-time/src/check_processing_time.py +++ b/apps/check-processing-time/src/check_processing_time.py @@ -4,7 +4,7 @@ def get_time_from_attempts(attempts): attempts.sort(key=lambda attempt: attempt['StartedAt']) final_attempt = attempts[-1] - return final_attempt['StoppedAt'] - final_attempt['StartedAt'] + return (final_attempt['StoppedAt'] - final_attempt['StartedAt']) / 1000 def lambda_handler(event, context): diff --git a/tests/test_check_processing_time.py b/tests/test_check_processing_time.py index 9eb0ad7cb..63fe507f6 100644 --- a/tests/test_check_processing_time.py +++ b/tests/test_check_processing_time.py @@ -2,29 +2,29 @@ def test_single_attempt(): - attempts = [{'Container': {}, 'StartedAt': 1644423555778, 'StatusReason': '', 'StoppedAt': 1644425055200}] + attempts = [{'Container': {}, 'StartedAt': 500, 'StatusReason': '', 'StoppedAt': 2800}] result = check_processing_time.get_time_from_attempts(attempts) - assert result == 1644425055200 - 1644423555778 + assert result == 2.3 def test_multiple_attempts(): attempts = [ - {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331}, - {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015} + {'Container': {}, 'StartedAt': 500, 'StatusReason': '', 'StoppedAt': 1000}, + {'Container': {}, 'StartedAt': 3000, 'StatusReason': '', 'StoppedAt': 8700} ] result = check_processing_time.get_time_from_attempts(attempts) - assert result == 1644611472015 - 1644610107570 + assert result == 5.7 def test_unsorted_attempts(): # I'm not sure if the lambda would ever be given unsorted attempts, but it seems worth testing that it doesn't # depend on the list to be sorted. attempts = [ - {'Container': {}, 'StartedAt': 1644610107570, 'StatusReason': '', 'StoppedAt': 1644611472015}, - {'Container': {}, 'StartedAt': 1644609403693, 'StatusReason': '', 'StoppedAt': 1644609919331} + {'Container': {}, 'StartedAt': 3000, 'StatusReason': '', 'StoppedAt': 8700}, + {'Container': {}, 'StartedAt': 500, 'StatusReason': '', 'StoppedAt': 1000} ] result = check_processing_time.get_time_from_attempts(attempts) - assert result == 1644611472015 - 1644610107570 + assert result == 5.7 def test_lambda_handler_with_normal_results(): @@ -37,7 +37,7 @@ def test_lambda_handler_with_normal_results(): } } response = check_processing_time.lambda_handler(event, None) - assert response == 1644611472015 - 1644610107570 + assert response == (1644611472015 - 1644610107570) / 1000 def test_lambda_handler_with_failed_results(): @@ -51,4 +51,4 @@ def test_lambda_handler_with_failed_results(): } } response = check_processing_time.lambda_handler(event, None) - assert response == 1643834908466 - 1643834907858 + assert response == (1643834908466 - 1643834907858) / 1000 From b5a636ff2dfebdcd6059f737683f7ce0e9278e91 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 15 Feb 2022 11:20:10 -0900 Subject: [PATCH 18/23] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0c2bc8e5e..5b22107cf 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ render: static: flake8 openapi-validate cfn-lint flake8: - flake8 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,start_execution,update_db,upload_log,dynamo,process_new_granules,scale_cluster apps tests lib + flake8 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,check_processing_time,start_execution,update_db,upload_log,dynamo,process_new_granules,scale_cluster apps tests lib openapi-validate: render prance validate --backend=openapi-spec-validator apps/api/src/hyp3_api/api-spec/openapi-spec.yml From dbd9c7703525ed1c01faa4e3fc51055ed7fe3c05 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Tue, 15 Feb 2022 12:50:00 -0900 Subject: [PATCH 19/23] convert float values to Decimals in dynamo.jobs.update_job --- lib/dynamo/dynamo/jobs.py | 7 +++++-- tests/test_dynamo/test_jobs.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index b2c691499..28f5a3995 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -115,8 +115,11 @@ def update_job(job): table = DYNAMODB_RESOURCE.Table(environ['JOBS_TABLE_NAME']) primary_key = 'job_id' key = {'job_id': job[primary_key]} - update_expression = 'SET {}'.format(','.join(f'{k}=:{k}' for k in job if k != primary_key)) - expression_attribute_values = {f':{k}': v for k, v in job.items() if k != primary_key} + + prepared_job = convert_floats_to_decimals(job) + update_expression = 'SET {}'.format(','.join(f'{k}=:{k}' for k in prepared_job if k != primary_key)) + expression_attribute_values = {f':{k}': v for k, v in prepared_job.items() if k != primary_key} + table.update_item( Key=key, UpdateExpression=update_expression, diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index ce3b0eb15..6499ab2d1 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -426,7 +426,7 @@ def test_update_job(tables): for item in table_items: tables.jobs_table.put_item(Item=item) - job = {'job_id': 'job1', 'status_code': 'status2'} + job = {'job_id': 'job1', 'status_code': 'status2', 'processing_time_in_seconds': 1.23} dynamo.jobs.update_job(job) response = tables.jobs_table.scan() @@ -437,6 +437,7 @@ def test_update_job(tables): 'user_id': 'user1', 'status_code': 'status2', 'request_time': '2000-01-01T00:00:00+00:00', + 'processing_time_in_seconds': Decimal('1.23'), }, ] assert response['Items'] == expected_response From 1b481a39754d86e35dc8d616ebb7679978b2f51c Mon Sep 17 00:00:00 2001 From: jzhu4 Date: Tue, 15 Feb 2022 13:30:39 -0900 Subject: [PATCH 20/23] delete environment hype-autorift in the deploy.yml --- .github/workflows/deploy.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c5e2c063a..db8dd86f5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -88,20 +88,6 @@ jobs: # security_environment: JPL # ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id - - environment: hyp3-autorift - domain: hyp3-autorift.asf.alaska.edu - template_bucket: cf-templates-igavixdzdy7k-us-west-2 - image_tag: latest - product_lifetime_in_days: 180 - quota: 0 - deploy_ref: refs/heads/main - job_files: job_spec/AUTORIFT_ITS_LIVE.yml - default_max_vcpus: 1600 - expanded_max_vcpus: 1600 - required_surplus: 0 - security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id - - environment: hyp3-autorift-eu domain: hyp3-autorift-eu.asf.alaska.edu template_bucket: cf-templates-autorift-eu-central-1 From 437c04104fa25d88f33777916a6af0eba1f20d1f Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 15 Feb 2022 15:27:46 -0900 Subject: [PATCH 21/23] Fix Changelog formatting --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 487d84fa4..2da84d204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## 2.9.0 +## [2.9.0] ### Added - Add `processing_time_in_seconds` to the `job` API schema to allow plugin developers to check processing time. From 36a8095aa3e5d0fde5b48b13942bc66c7f78b4a6 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 16 Feb 2022 09:12:00 -0900 Subject: [PATCH 22/23] Reorganize deployments --- .../workflows/{deploy.yml => deploy-daac.yml} | 58 +--------- .github/workflows/deploy-enterprise.yml | 107 ++++++++++++++++++ 2 files changed, 108 insertions(+), 57 deletions(-) rename .github/workflows/{deploy.yml => deploy-daac.yml} (66%) create mode 100644 .github/workflows/deploy-enterprise.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy-daac.yml similarity index 66% rename from .github/workflows/deploy.yml rename to .github/workflows/deploy-daac.yml index db8dd86f5..1e5f5a3ae 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy-daac.yml @@ -1,4 +1,4 @@ -name: Deploy to AWS +name: Deploy DAAC Stacks to AWS on: push: @@ -74,62 +74,6 @@ jobs: security_environment: EDC ami_id: image_id_ecs_amz2 - #- environment: hyp3-its-live - # domain: hyp3-its-live.asf.alaska.edu - # template_bucket: cf-templates-3o5lnspmwmzg-us-west-2 - # image_tag: latest - # product_lifetime_in_days: 180 - # quota: 0 - # deploy_ref: refs/heads/main - # job_files: job_spec/AUTORIFT_ITS_LIVE.yml - # default_max_vcpus: 1600 - # expanded_max_vcpus: 1600 - # required_surplus: 0 - # security_environment: JPL - # ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id - - - environment: hyp3-autorift-eu - domain: hyp3-autorift-eu.asf.alaska.edu - template_bucket: cf-templates-autorift-eu-central-1 - image_tag: latest - product_lifetime_in_days: 180 - quota: 0 - deploy_ref: refs/heads/main - job_files: job_spec/AUTORIFT_ITS_LIVE_EU.yml - default_max_vcpus: 1600 - expanded_max_vcpus: 1600 - required_surplus: 0 - security_environment: ASF - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id - - #- environment: hyp3-isce - # domain: hyp3-isce.asf.alaska.edu - # template_bucket: cf-templates-t790khv4btdq-us-west-2 - # image_tag: latest - # product_lifetime_in_days: 180 - # quota: 0 - # deploy_ref: refs/heads/main - # job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml - # default_max_vcpus: 1600 - # expanded_max_vcpus: 1600 - # required_surplus: 0 - # security_environment: ASF - # ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id - - #- environment: hyp3-tibet - # domain: hyp3-tibet.asf.alaska.edu - # template_bucket: cf-templates-ejaipnrxq7xg-us-west-2 - # image_tag: latest - # product_lifetime_in_days: 180 - # quota: 0 - # deploy_ref: refs/heads/main - # job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml - # default_max_vcpus: 1600 - # expanded_max_vcpus: 1600 - # required_surplus: 0 - # security_environment: ASF - # ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id - environment: name: ${{ matrix.environment }} url: https://${{ matrix.domain }} diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml new file mode 100644 index 000000000..75185edc7 --- /dev/null +++ b/.github/workflows/deploy-enterprise.yml @@ -0,0 +1,107 @@ +name: Deploy Enterprize Stacks to AWS + +on: + push: + branches: + - main + +jobs: + deploy: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - environment: hyp3-its-live + domain: hyp3-its-live.asf.alaska.edu + template_bucket: cf-templates-3o5lnspmwmzg-us-west-2 + image_tag: latest + product_lifetime_in_days: 180 + quota: 0 + job_files: job_spec/AUTORIFT_ITS_LIVE.yml + default_max_vcpus: 2640 + expanded_max_vcpus: 2640 + required_surplus: 0 + security_environment: JPL + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + + - environment: hyp3-autorift-eu + domain: hyp3-autorift-eu.asf.alaska.edu + template_bucket: cf-templates-autorift-eu-central-1 + image_tag: latest + product_lifetime_in_days: 180 + quota: 0 + job_files: job_spec/AUTORIFT_ITS_LIVE_EU.yml + default_max_vcpus: 1600 + expanded_max_vcpus: 1600 + required_surplus: 0 + security_environment: ASF + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + + - environment: hyp3-isce + domain: hyp3-isce.asf.alaska.edu + template_bucket: cf-templates-t790khv4btdq-us-west-2 + image_tag: latest + product_lifetime_in_days: 180 + quota: 0 + job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml + default_max_vcpus: 1600 + expanded_max_vcpus: 1600 + required_surplus: 0 + security_environment: ASF + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + + - environment: hyp3-tibet + domain: hyp3-tibet.asf.alaska.edu + template_bucket: cf-templates-ejaipnrxq7xg-us-west-2 + image_tag: latest + product_lifetime_in_days: 180 + quota: 0 + job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml + default_max_vcpus: 1600 + expanded_max_vcpus: 1600 + required_surplus: 0 + security_environment: ASF + ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + + environment: + name: ${{ matrix.environment }} + url: https://${{ matrix.domain }} + + steps: + - uses: actions/checkout@v2 + + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.V2_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.V2_AWS_SESSION_TOKEN }} + aws-region: ${{ secrets.AWS_REGION }} + + - uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - uses: ./.github/actions/deploy-hyp3 + with: + TEMPLATE_BUCKET: ${{ matrix.template_bucket }} + STACK_NAME: ${{ matrix.environment }} + DOMAIN_NAME: ${{ matrix.domain }} + CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }} + IMAGE_TAG: ${{ matrix.image_tag }} + PRODUCT_LIFETIME: ${{ matrix.product_lifetime_in_days }} + VPC_ID: ${{ secrets.VPC_ID }} + SUBNET_IDS: ${{ secrets.SUBNET_IDS }} + EDL_USERNAME: ${{ secrets.EDL_USERNAME }} + EDL_PASSWORD: ${{ secrets.EDL_PASSWORD }} + CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }} + MONTHLY_JOB_QUOTA_PER_USER: ${{ matrix.quota }} + JOB_FILES: ${{ matrix.job_files }} + BANNED_CIDR_BLOCKS: ${{ secrets.BANNED_CIDR_BLOCKS }} + DEFAULT_MAX_VCPUS: ${{ matrix.default_max_vcpus }} + EXPANDED_MAX_VCPUS: ${{ matrix.expanded_max_vcpus }} + MONTHLY_COMPUTE_BUDGET: ${{ secrets.MONTHLY_COMPUTE_BUDGET }} + REQUIRED_SURPLUS: ${{ matrix.required_surplus }} + PERMISSIONS_BOUNDARY_ARN: ${{ secrets.PERMISSIONS_BOUNDARY_ARN }} + SECURITY_ENVIRONMENT: ${{ matrix.security_environment }} + AMI_ID: ${{ matrix.ami_id }} From e5bc53121c15b471d03a2c89a8d570d6f3ff3d28 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 16 Feb 2022 09:46:57 -0900 Subject: [PATCH 23/23] fix spelling --- .github/workflows/deploy-enterprise.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml index 75185edc7..f3f9dfc61 100644 --- a/.github/workflows/deploy-enterprise.yml +++ b/.github/workflows/deploy-enterprise.yml @@ -1,4 +1,4 @@ -name: Deploy Enterprize Stacks to AWS +name: Deploy Enterprise Stacks to AWS on: push: