diff --git a/.flake8 b/.flake8 index e2b167eec..5826386b6 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ max-line-length = 88 max-complexity = 18 select = B,C,E,F,W,T4,B9 -ignore = E203, E266, E501, W503, F403, F401, E402, W605 \ No newline at end of file +ignore = E203, E266, E501, W503, F403, E402, W605 \ No newline at end of file diff --git a/app/models/orm/migrations/env.py b/app/models/orm/migrations/env.py index f7f3a50eb..327db8639 100644 --- a/app/models/orm/migrations/env.py +++ b/app/models/orm/migrations/env.py @@ -12,12 +12,12 @@ from app.application import db # To include a model in migrations, add a line here. -from app.models.orm.assets import Asset -from app.models.orm.datasets import Dataset -from app.models.orm.geostore import Geostore -from app.models.orm.tasks import Task -from app.models.orm.user_areas import UserArea -from app.models.orm.versions import Version +from app.models.orm.assets import Asset # noqa: F401 +from app.models.orm.datasets import Dataset # noqa: F401 +from app.models.orm.geostore import Geostore # noqa: F401 +from app.models.orm.tasks import Task # noqa: F401 +from app.models.orm.user_areas import UserArea # noqa: F401 +from app.models.orm.versions import Version # noqa: F401 ############################################################################### diff --git a/app/routes/datasets/queries.py b/app/routes/datasets/queries.py index dd8ea24d0..35d1a9e65 100644 --- a/app/routes/datasets/queries.py +++ b/app/routes/datasets/queries.py @@ -19,8 +19,7 @@ from sqlalchemy.engine import RowProxy from ...application import db -from ...crud import assets, versions -from ...errors import RecordNotFoundError +from ...crud import assets from ...models.enum.assets import AssetType from ...models.enum.geostore import GeostoreOrigin from ...models.enum.pg_admin_functions import ( diff --git a/app/settings/globals.py b/app/settings/globals.py index 7d1fa5563..9a7725720 100644 --- a/app/settings/globals.py +++ b/app/settings/globals.py @@ -148,3 +148,4 @@ SQL_REQUEST_TIMEOUT = 58 AWS_GCS_KEY_SECRET_ARN = config("AWS_GCS_KEY_SECRET_ARN", cast=str, default=None) +AWS_SECRETSMANAGER_URL = config("AWS_SECRETSMANAGER_URL", cast=str, default=None) diff --git a/app/tasks/raster_tile_cache_assets/__init__.py b/app/tasks/raster_tile_cache_assets/__init__.py index 47c838f4f..fc2ceb516 100644 --- a/app/tasks/raster_tile_cache_assets/__init__.py +++ b/app/tasks/raster_tile_cache_assets/__init__.py @@ -1,4 +1,4 @@ -from .raster_tile_cache_assets import ( +from .raster_tile_cache_assets import ( # noqa: F401 raster_tile_cache_asset, raster_tile_cache_validator, ) diff --git a/app/tasks/raster_tile_set_assets/__init__.py b/app/tasks/raster_tile_set_assets/__init__.py index 6f2be59c8..fb2424109 100644 --- a/app/tasks/raster_tile_set_assets/__init__.py +++ b/app/tasks/raster_tile_set_assets/__init__.py @@ -1 +1 @@ -from .raster_tile_set_assets import raster_tile_set_asset +from .raster_tile_set_assets import raster_tile_set_asset # noqa: F401 diff --git a/batch/pixetl.dockerfile b/batch/pixetl.dockerfile index bd50e6df2..a3e0a69b6 100644 --- a/batch/pixetl.dockerfile +++ b/batch/pixetl.dockerfile @@ -1,4 +1,4 @@ -FROM globalforestwatch/pixetl:v1.3.9 +FROM globalforestwatch/pixetl:v1.3.10 # Copy scripts COPY ./batch/scripts/ /opt/scripts/ diff --git a/batch/scripts/report_status.sh b/batch/scripts/report_status.sh index 1b6d09f64..2fb7f828f 100755 --- a/batch/scripts/report_status.sh +++ b/batch/scripts/report_status.sh @@ -24,9 +24,9 @@ ESC_COMMAND=$(echo -n "$*" | json_escape) # Also make sure we don't reveal any sensitive information # But we still want to know if the var was set ESC_OUTPUT="$(cat $OUTPUT_FILE \ - | sed 's/^AWS_SECRET_ACCESS_KEY.*$/AWS_SECRET_ACCESS_KEY=\*\*\*/' \ + | sed 's/^AWS_SECRET_ACCESS_KEY.*$/AWS_SECRET_ACCESS_KEY=\*\*\*/' \ # pragma: allowlist secret | sed 's/^AWS_ACCESS_KEY_ID.*$/AWS_ACCESS_KEY_ID=\*\*\*/' \ - | sed 's/^PGPASSWORD.*$/PGPASSWORD=\*\*\*/' \ + | sed 's/^PGPASSWORD.*$/PGPASSWORD=\*\*\*/' \ # pragma: allowlist secret | sed 's/^PGUSER.*$/PGUSER=\*\*\*/' \ | sed 's/^PGDATABASE.*$/PGDATABASE=\*\*\*/' \ | sed 's/^PGHOST.*$/PGHOST=\*\*\*/' \ diff --git a/docker-compose.test.yml b/docker-compose.test.yml index a099ee91d..3fe189303 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -48,14 +48,16 @@ services: - PIXETL_CORES=4 - PIXETL_MAX_MEM=3800 - API_URL=http://app_test:9000 - - S3_ENTRYPOINT_URL=http://motoserver:5000 + - S3_ENTRYPOINT_URL=http://motoserver-s3:5000 - SERVICE_ACCOUNT_TOKEN=testing - RASTER_ANALYSIS_LAMBDA_NAME=test_raster_analysis - AWS_GCS_KEY_SECRET_ARN=testing + - AWS_SECRETSMANAGER_URL=http://motoserver-secretsmanager:5001 entrypoint: wait_for_postgres.sh pytest -vv --cov-report term --cov-report xml:/app/tests/cobertura.xml --cov=app depends_on: - test_database - - motoserver + - motoserver-s3 + - motoserver-secretsmanager test_database: container_name: gfw-data-api-test-database @@ -71,13 +73,21 @@ services: - test_database_data:/var/lib/postgresql/data restart: on-failure - motoserver: - container_name: motoserver + motoserver-s3: + container_name: motoserver-s3 image: motoserver/moto:latest ports: - 5000:5000 entrypoint: moto_server s3 -H 0.0.0.0 restart: on-failure + motoserver-secretsmanager: + container_name: motoserver-secretsmanager + image: motoserver/moto:latest + ports: + - 5001:5001 + entrypoint: moto_server secretsmanager -p 5001 -H 0.0.0.0 + restart: on-failure + volumes: test_database_data: diff --git a/scripts/test b/scripts/test index a93de42ad..f05be6c38 100755 --- a/scripts/test +++ b/scripts/test @@ -33,7 +33,8 @@ if [ "${BUILD}" = true ]; then docker-compose -f docker-compose.test.yml --project-name gfw-data-api_test build --no-cache app_test fi +set +e docker-compose -f docker-compose.test.yml --project-name gfw-data-api_test run --rm --name app_test app_test tests/"$*" exit_code=$? -docker-compose -f docker-compose.test.yml down +docker-compose -f docker-compose.test.yml --project-name gfw-data-api_test down --remove-orphans exit $exit_code diff --git a/tests/__init__.py b/tests/__init__.py index cccb9238d..096c2b1a9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -203,14 +203,18 @@ def add_job_definition(self, job_definition_name, docker_image, mount_tmp=False) "environment": [ {"name": "AWS_ACCESS_KEY_ID", "value": "testing"}, {"name": "AWS_SECRET_ACCESS_KEY", "value": "testing"}, - {"name": "ENDPOINT_URL", "value": "http://motoserver:5000"}, + {"name": "ENDPOINT_URL", "value": "http://motoserver-s3:5000"}, {"name": "DEBUG", "value": "1"}, {"name": "TILE_CACHE", "value": TILE_CACHE_BUCKET}, {"name": "DATA_LAKE", "value": DATA_LAKE_BUCKET}, {"name": "AWS_HTTPS", "value": "NO"}, - {"name": "AWS_S3_ENDPOINT", "value": "motoserver:5000"}, + {"name": "AWS_S3_ENDPOINT", "value": "motoserver-s3:5000"}, {"name": "AWS_VIRTUAL_HOSTING", "value": "FALSE"}, {"name": "GDAL_DISABLE_READDIR_ON_OPEN", "value": "YES"}, + { + "name": "AWS_SECRETSMANAGER_URL", + "value": "http://motoserver-secretsmanager:5001", + }, ], "volumes": [ {"host": {"sourcePath": f"{ROOT}/tests/fixtures/aws"}, "name": "aws"}, diff --git a/tests/conftest.py b/tests/conftest.py index c855d793e..6793769ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,9 @@ from app.settings.globals import ( AURORA_JOB_QUEUE, AURORA_JOB_QUEUE_FAST, + AWS_GCS_KEY_SECRET_ARN, AWS_REGION, + AWS_SECRETSMANAGER_URL, DATA_LAKE_BUCKET, DATA_LAKE_JOB_QUEUE, GDAL_PYTHON_JOB_DEFINITION, @@ -234,7 +236,7 @@ def flush_request_list(httpd): def copy_fixtures(): # Upload file to mocked S3 bucket s3_client = boto3.client( - "s3", region_name=AWS_REGION, endpoint_url="http://motoserver:5000" + "s3", region_name=AWS_REGION, endpoint_url="http://motoserver-s3:5000" ) s3_client.create_bucket(Bucket=BUCKET) @@ -312,3 +314,18 @@ def create_lambda(func_str): yield create_lambda aws_mock.stop_services() + + +@pytest.fixture(scope="session", autouse=True) +def secrets(): + + secret_client = boto3.client( + "secretsmanager", region_name=AWS_REGION, endpoint_url=AWS_SECRETSMANAGER_URL + ) + secret_client.create_secret( + Name=AWS_GCS_KEY_SECRET_ARN, + SecretString="foosecret", # pragma: allowlist secret + ) + yield + + secret_client.delete_secret(SecretId=AWS_GCS_KEY_SECRET_ARN) diff --git a/tests/fixtures/aws/config b/tests/fixtures/aws/config index bfcf7a13e..3cf780134 100644 --- a/tests/fixtures/aws/config +++ b/tests/fixtures/aws/config @@ -1,6 +1,6 @@ [default] s3 = - endpoint_url = http://motoserver:5000 + endpoint_url = http://motoserver-s3:5000 [plugins] endpoint = awscli_plugin_endpoint \ No newline at end of file diff --git a/tests/routes/test_assets.py b/tests/routes/test_assets.py index 3242acdf9..260142f13 100644 --- a/tests/routes/test_assets.py +++ b/tests/routes/test_assets.py @@ -92,7 +92,7 @@ async def test_auxiliary_raster_asset(async_client, batch_client, httpd): auxiliary_grid = "90/9984" s3_client = boto3.client( - "s3", region_name=AWS_REGION, endpoint_url="http://motoserver:5000" + "s3", region_name=AWS_REGION, endpoint_url="http://motoserver-s3:5000" ) pixetl_output_files = [ @@ -192,7 +192,7 @@ async def test_auxiliary_vector_asset(async_client, batch_client, httpd): version = "v1.1.1" s3_client = boto3.client( - "s3", region_name=AWS_REGION, endpoint_url="http://motoserver:5000" + "s3", region_name=AWS_REGION, endpoint_url="http://motoserver-s3:5000" ) pixetl_output_files = [ diff --git a/tests/utils.py b/tests/utils.py index f05d81b37..ffc3ef884 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -240,7 +240,7 @@ async def check_tasks_status(async_client, logs, asset_ids) -> None: def upload_fake_data(dtype, dtype_name, no_data, prefix): s3_client = boto3.client( - "s3", region_name=AWS_REGION, endpoint_url="http://motoserver:5000" + "s3", region_name=AWS_REGION, endpoint_url="http://motoserver-s3:5000" ) data_file_name = "0000000000-0000000000.tif"