add second motoserver for secretsmanager

.flake8

@@ -2,4 +2,4 @@
 max-line-length = 88
 max-complexity = 18
 select = B,C,E,F,W,T4,B9
-ignore = E203, E266, E501, W503, F403, F401, E402, W605
+ignore = E203, E266, E501, W503, F403, E402, W605

app/models/orm/migrations/env.py

@@ -12,12 +12,12 @@
 from app.application import db
 
 # To include a model in migrations, add a line here.
-from app.models.orm.assets import Asset
-from app.models.orm.datasets import Dataset
-from app.models.orm.geostore import Geostore
-from app.models.orm.tasks import Task
-from app.models.orm.user_areas import UserArea
-from app.models.orm.versions import Version
+from app.models.orm.assets import Asset  # noqa: F401
+from app.models.orm.datasets import Dataset  # noqa: F401
+from app.models.orm.geostore import Geostore  # noqa: F401
+from app.models.orm.tasks import Task  # noqa: F401
+from app.models.orm.user_areas import UserArea  # noqa: F401
+from app.models.orm.versions import Version  # noqa: F401
 
 ###############################################################################
 

app/routes/datasets/queries.py

@@ -19,8 +19,7 @@
 from sqlalchemy.engine import RowProxy
 
 from ...application import db
-from ...crud import assets, versions
-from ...errors import RecordNotFoundError
+from ...crud import assets
 from ...models.enum.assets import AssetType
 from ...models.enum.geostore import GeostoreOrigin
 from ...models.enum.pg_admin_functions import (

app/settings/globals.py

@@ -148,3 +148,4 @@
 SQL_REQUEST_TIMEOUT = 58
 
 AWS_GCS_KEY_SECRET_ARN = config("AWS_GCS_KEY_SECRET_ARN", cast=str, default=None)
+AWS_SECRETSMANAGER_URL = config("AWS_SECRETSMANAGER_URL", cast=str, default=None)

app/tasks/raster_tile_cache_assets/__init__.py

@@ -1,4 +1,4 @@
-from .raster_tile_cache_assets import (
+from .raster_tile_cache_assets import (  # noqa: F401
     raster_tile_cache_asset,
     raster_tile_cache_validator,
 )

app/tasks/raster_tile_set_assets/__init__.py

@@ -1 +1 @@
-from .raster_tile_set_assets import raster_tile_set_asset
+from .raster_tile_set_assets import raster_tile_set_asset  # noqa: F401

batch/pixetl.dockerfile

@@ -1,4 +1,4 @@
-FROM globalforestwatch/pixetl:v1.3.9
+FROM globalforestwatch/pixetl:v1.3.10
 
 # Copy scripts
 COPY ./batch/scripts/ /opt/scripts/

batch/scripts/report_status.sh

@@ -24,9 +24,9 @@ ESC_COMMAND=$(echo -n "$*" | json_escape)
 # Also make sure we don't reveal any sensitive information
 # But we still want to know if the var was set
 ESC_OUTPUT="$(cat $OUTPUT_FILE \
-  | sed 's/^AWS_SECRET_ACCESS_KEY.*$/AWS_SECRET_ACCESS_KEY=\*\*\*/' \
+  | sed 's/^AWS_SECRET_ACCESS_KEY.*$/AWS_SECRET_ACCESS_KEY=\*\*\*/' \  # pragma: allowlist secret
   | sed 's/^AWS_ACCESS_KEY_ID.*$/AWS_ACCESS_KEY_ID=\*\*\*/' \
-  | sed 's/^PGPASSWORD.*$/PGPASSWORD=\*\*\*/' \
+  | sed 's/^PGPASSWORD.*$/PGPASSWORD=\*\*\*/' \  # pragma: allowlist secret
   | sed 's/^PGUSER.*$/PGUSER=\*\*\*/' \
   | sed 's/^PGDATABASE.*$/PGDATABASE=\*\*\*/' \
   | sed 's/^PGHOST.*$/PGHOST=\*\*\*/' \

docker-compose.test.yml

@@ -48,14 +48,16 @@ services:
       - PIXETL_CORES=4
       - PIXETL_MAX_MEM=3800
       - API_URL=http://app_test:9000
-      - S3_ENTRYPOINT_URL=http://motoserver:5000
+      - S3_ENTRYPOINT_URL=http://motoserver-s3:5000
       - SERVICE_ACCOUNT_TOKEN=testing
       - RASTER_ANALYSIS_LAMBDA_NAME=test_raster_analysis
       - AWS_GCS_KEY_SECRET_ARN=testing
+      - AWS_SECRETSMANAGER_URL=http://motoserver-secretsmanager:5001
     entrypoint: wait_for_postgres.sh pytest -vv --cov-report term --cov-report xml:/app/tests/cobertura.xml --cov=app
     depends_on:
       - test_database
-      - motoserver
+      - motoserver-s3
+      - motoserver-secretsmanager
 
   test_database:
     container_name: gfw-data-api-test-database
@@ -71,13 +73,21 @@ services:
       - test_database_data:/var/lib/postgresql/data
     restart: on-failure
 
-  motoserver:
-    container_name: motoserver
+  motoserver-s3:
+    container_name: motoserver-s3
     image: motoserver/moto:latest
     ports:
       - 5000:5000
     entrypoint: moto_server s3 -H 0.0.0.0
     restart: on-failure
 
+  motoserver-secretsmanager:
+    container_name: motoserver-secretsmanager
+    image: motoserver/moto:latest
+    ports:
+      - 5001:5001
+    entrypoint: moto_server secretsmanager -p 5001 -H 0.0.0.0
+    restart: on-failure
+
 volumes:
   test_database_data:

scripts/test

@@ -33,7 +33,8 @@ if [ "${BUILD}" = true ]; then
   docker-compose -f docker-compose.test.yml --project-name gfw-data-api_test build --no-cache app_test
 fi
 
+set +e
 docker-compose -f docker-compose.test.yml --project-name gfw-data-api_test run --rm --name app_test app_test tests/"$*"
 exit_code=$?
-docker-compose -f docker-compose.test.yml down
+docker-compose -f docker-compose.test.yml --project-name gfw-data-api_test down --remove-orphans
 exit $exit_code

tests/__init__.py

@@ -203,14 +203,18 @@ def add_job_definition(self, job_definition_name, docker_image, mount_tmp=False)
             "environment": [
                 {"name": "AWS_ACCESS_KEY_ID", "value": "testing"},
                 {"name": "AWS_SECRET_ACCESS_KEY", "value": "testing"},
-                {"name": "ENDPOINT_URL", "value": "http://motoserver:5000"},
+                {"name": "ENDPOINT_URL", "value": "http://motoserver-s3:5000"},
                 {"name": "DEBUG", "value": "1"},
                 {"name": "TILE_CACHE", "value": TILE_CACHE_BUCKET},
                 {"name": "DATA_LAKE", "value": DATA_LAKE_BUCKET},
                 {"name": "AWS_HTTPS", "value": "NO"},
-                {"name": "AWS_S3_ENDPOINT", "value": "motoserver:5000"},
+                {"name": "AWS_S3_ENDPOINT", "value": "motoserver-s3:5000"},
                 {"name": "AWS_VIRTUAL_HOSTING", "value": "FALSE"},
                 {"name": "GDAL_DISABLE_READDIR_ON_OPEN", "value": "YES"},
+                {
+                    "name": "AWS_SECRETSMANAGER_URL",
+                    "value": "http://motoserver-secretsmanager:5001",
+                },
             ],
             "volumes": [
                 {"host": {"sourcePath": f"{ROOT}/tests/fixtures/aws"}, "name": "aws"},

tests/conftest.py

@@ -19,7 +19,9 @@
 from app.settings.globals import (
     AURORA_JOB_QUEUE,
     AURORA_JOB_QUEUE_FAST,
+    AWS_GCS_KEY_SECRET_ARN,
     AWS_REGION,
+    AWS_SECRETSMANAGER_URL,
     DATA_LAKE_BUCKET,
     DATA_LAKE_JOB_QUEUE,
     GDAL_PYTHON_JOB_DEFINITION,
@@ -234,7 +236,7 @@ def flush_request_list(httpd):
 def copy_fixtures():
     # Upload file to mocked S3 bucket
     s3_client = boto3.client(
-        "s3", region_name=AWS_REGION, endpoint_url="http://motoserver:5000"
+        "s3", region_name=AWS_REGION, endpoint_url="http://motoserver-s3:5000"
     )
 
     s3_client.create_bucket(Bucket=BUCKET)
@@ -312,3 +314,18 @@ def create_lambda(func_str):
     yield create_lambda
 
     aws_mock.stop_services()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def secrets():
+
+    secret_client = boto3.client(
+        "secretsmanager", region_name=AWS_REGION, endpoint_url=AWS_SECRETSMANAGER_URL
+    )
+    secret_client.create_secret(
+        Name=AWS_GCS_KEY_SECRET_ARN,
+        SecretString="foosecret",  # pragma: allowlist secret
+    )
+    yield
+
+    secret_client.delete_secret(SecretId=AWS_GCS_KEY_SECRET_ARN)

tests/fixtures/aws/config

@@ -1,6 +1,6 @@
 [default]
 s3 =
-    endpoint_url = http://motoserver:5000
+    endpoint_url = http://motoserver-s3:5000
 
 [plugins]
 endpoint = awscli_plugin_endpoint

tests/routes/test_assets.py

@@ -92,7 +92,7 @@ async def test_auxiliary_raster_asset(async_client, batch_client, httpd):
     auxiliary_grid = "90/9984"
 
     s3_client = boto3.client(
-        "s3", region_name=AWS_REGION, endpoint_url="http://motoserver:5000"
+        "s3", region_name=AWS_REGION, endpoint_url="http://motoserver-s3:5000"
     )
 
     pixetl_output_files = [
@@ -192,7 +192,7 @@ async def test_auxiliary_vector_asset(async_client, batch_client, httpd):
     version = "v1.1.1"
 
     s3_client = boto3.client(
-        "s3", region_name=AWS_REGION, endpoint_url="http://motoserver:5000"
+        "s3", region_name=AWS_REGION, endpoint_url="http://motoserver-s3:5000"
     )
 
     pixetl_output_files = [

tests/utils.py

@@ -240,7 +240,7 @@ async def check_tasks_status(async_client, logs, asset_ids) -> None:
 
 def upload_fake_data(dtype, dtype_name, no_data, prefix):
     s3_client = boto3.client(
-        "s3", region_name=AWS_REGION, endpoint_url="http://motoserver:5000"
+        "s3", region_name=AWS_REGION, endpoint_url="http://motoserver-s3:5000"
     )
 
     data_file_name = "0000000000-0000000000.tif"