reduce Lambda asset size (#74)

* use uv for dependency management * run cdk synth in CI to check asset package size * update minimum python version to 3.9 * strip debug symbols from compiled C/C++ code in Lambda docker build
developmentseed · Jan 13, 2025 · 0f31041 · 0f31041
1 parent 52f7f8a
commit 0f31041
Show file tree

Hide file tree

Showing 10 changed files with 3,036 additions and 125 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -29,40 +29,44 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
+      - uses: actions/checkout@v4
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.5.*" 
+          enable-cache: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install -e .["test"]
+          uv sync --all-extras
 
       - name: run pre-commit
         if: ${{ matrix.python-version == env.LATEST_PY_VERSION }}
         run: |
-          python -m pip install pre-commit
-          pre-commit run --all-files
+          uv run pre-commit run --all-files
 
       - name: Run tests
-        run: python -m pytest --cov titiler.xarray --cov-report term-missing -s -vv
+        run: uv run pytest
 
   deploy:
     needs: [tests]
     runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')
+    env:
+      UV_PYTHON: 3.12
 
     defaults:
       run:
         working-directory: infrastructure/aws
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v2
@@ -76,23 +80,39 @@ jobs:
         with:
           node-version: 18
 
-      - name: Install cdk
-        run: npm install -g aws-cdk
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          python-version: '3.x'
+          version: "0.5.*" 
+          enable-cache: true
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install -r requirements-cdk.txt
+          uv sync --only-group deployment
+          uv run npm install
+
+      - name: CDK Synth
+        run: uv run --only-group deployment npm run cdk -- synth
 
+      - name: Check Asset Sizes
+        run: |
+          MAX_SIZE_BYTES=262144000  # 262 MB in bytes
+          for dir in cdk.out/asset.*; do
+            if [ -d "$dir" ]; then
+              size=$(du -sb "$dir" | cut -f1)
+              if [ "$size" -gt $MAX_SIZE_BYTES ]; then
+                echo "Directory $dir exceeds 262 MB with size $size bytes (max: $MAX_SIZE_BYTES bytes)."
+                exit 1 # Exit with failure if any asset directory is too large
+              fi
+              echo "Asset directory $dir size: $size bytes"
+            fi
+          done
+          echo "All asset directories are within size limits."
+      
       # Build and deploy to the development environment whenever there is a push to main or dev
       - name: Build & Deploy Development
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: npm run cdk -- deploy titiler-xarray-development --require-approval never
+        run: uv run npm run cdk -- deploy titiler-xarray-development --require-approval never
         env:
           TITILER_XARRAY_PYTHONWARNINGS: ignore
           TITILER_XARRAY_DEBUG: True
@@ -102,7 +122,7 @@ jobs:
       # Build and deploy to production deployment whenever there a new tag is pushed
       - name: Build & Deploy Production
         if: startsWith(github.ref, 'refs/tags/v')
-        run: npm run cdk -- deploy titiler-xarray-production --require-approval never
+        run: uv run npm run cdk -- deploy titiler-xarray-production --require-approval never
         env:
           TITILER_XARRAY_PYTHONWARNINGS: ignore
           TITILER_XARRAY_DEBUG: True

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.12.1
+    rev: v0.23
     hooks:
       - id: validate-pyproject
 

diff --git a/README.md b/README.md
@@ -21,21 +21,27 @@ uvicorn titiler.multidim.main:app --reload
 To access the docs, visit <http://127.0.0.1:8000/api.html>.
 ![](https://github.com/developmentseed/titiler-xarray/assets/10407788/4368546b-5b60-4cd5-86be-fdd959374b17)
 
-## Testing
+## Development
 
 Tests use data generated locally by using `tests/fixtures/generate_test_*.py` scripts.
 
+Install the package using [`uv`](https://docs.astral.sh/uv/getting-started/installation/) with all development dependencies:
+
+```bash
+uv sync
+uv run pre-commit install
+```
+
 To run all the tests:
 
 ```bash
-python -m pip install -e ".[test]"
-python -m pytest 
+uv run pytest
 ```
 
 To run just one test:
 
 ```bash
-python -m pytest tests/test_app.py::test_get_info 
+uv run pytest tests/test_app.py::test_get_info 
 ```
 
 ## VEDA Deployment
@@ -56,19 +62,14 @@ The following steps detail how to to setup and deploy the CDK stack from your lo
     # Download titiler repo
     git clone https://github.com/developmentseed/titiler-xarray.git
 
-    # Create a virtual environment
-    python -m pip install --upgrade virtualenv
-    virtualenv infrastructure/aws/.venv
-    source infrastructure/aws/.venv/bin/activate
-
-    # install cdk dependencies
-    python -m pip install -r infrastructure/aws/requirements-cdk.txt
+    # Install with the deployment dependencies
+    uv sync --group deployment
 
     # Install node dependency
-    npm --prefix infrastructure/aws install
+    uv run npm --prefix infrastructure/aws install
 
     # Deploys the CDK toolkit stack into an AWS environment
-    npm --prefix infrastructure/aws run cdk -- bootstrap
+    uv run npm --prefix infrastructure/aws run cdk -- bootstrap
 
     # or to a specific region and or using AWS profile
     AWS_DEFAULT_REGION=us-west-2 AWS_REGION=us-west-2 AWS_PROFILE=myprofile npm --prefix infrastructure/aws run cdk -- bootstrap
@@ -81,16 +82,16 @@ The following steps detail how to to setup and deploy the CDK stack from your lo
 3. Pre-Generate CFN template
 
     ```bash
-    npm --prefix infrastructure/aws run cdk -- synth  # Synthesizes and prints the CloudFormation template for this stack
+    uv run npm --prefix infrastructure/aws run cdk -- synth  # Synthesizes and prints the CloudFormation template for this stack
     ```
 
 4. Deploy
 
     ```bash
-    STACK_STAGE=staging npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-staging
+    STACK_STAGE=staging uv run npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-staging
 
     # Deploy in specific region
-    AWS_DEFAULT_REGION=us-west-2 AWS_REGION=us-west-2 AWS_PROFILE=smce-veda STACK_STAGE=production  npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-production
+    AWS_DEFAULT_REGION=us-west-2 AWS_REGION=us-west-2 AWS_PROFILE=smce-veda STACK_STAGE=production  uv run npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-production
     ```
 
 **Important**

diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py
@@ -2,18 +2,18 @@
 
 from typing import Dict, List, Optional
 
-import pydantic
+from pydantic_settings import BaseSettings
 
 
-class StackSettings(pydantic.BaseSettings):
+class StackSettings(BaseSettings):
     """Application settings"""
 
     name: str = "titiler-xarray"
     stage: str = "production"
 
-    owner: Optional[str]
-    client: Optional[str]
-    project: Optional[str]
+    owner: Optional[str] = None
+    client: Optional[str] = None
+    project: Optional[str] = None
 
     additional_env: Dict = {}
 
@@ -31,7 +31,7 @@ class StackSettings(pydantic.BaseSettings):
 
     # The maximum of concurrent executions you want to reserve for the function.
     # Default: - No specific limit - account limit.
-    max_concurrent: Optional[int]
+    max_concurrent: Optional[int] = None
     alarm_email: Optional[str] = ""
 
     class Config:

diff --git a/infrastructure/aws/lambda/Dockerfile b/infrastructure/aws/lambda/Dockerfile
@@ -1,6 +1,7 @@
-ARG PYTHON_VERSION=3.10
+ARG PYTHON_VERSION=3.11
 
 FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 WORKDIR /tmp
 
@@ -18,8 +19,7 @@ COPY src/titiler/ src/titiler/
 # and becaise we NEED to remove both boto3 and botocore to save space for the package
 # we have to force using old package version that seems `almost` compatible with Lambda env botocore
 # https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
-RUN pip install --upgrade pip
-RUN pip install . "mangum>=0.10.0" "botocore==1.29.76" "aiobotocore==2.5.0" -t /asset --no-binary pydantic
+RUN uv pip install --compile-bytecode . "mangum>=0.10.0" "botocore==1.29.76" "aiobotocore==2.5.0" --target /asset --no-binary pydantic
 
 # Reduce package size and remove useless files
 RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
@@ -30,8 +30,11 @@ RUN rm -rdf /asset/numpy/doc/ /asset/bin /asset/geos_license /asset/Misc
 RUN rm -rdf /asset/boto3*
 RUN rm -rdf /asset/botocore*
 
-# Remove system dependencies
-RUN yum remove -y gcc-c++
+# Strip debug symbols from compiled C/C++ code (except for numpy.libs!)
+RUN cd /asset && \
+  find . -type f -name '*.so*' \
+  -not -path "./numpy.libs/*" \
+  -exec strip --strip-unneeded {} \;
 
 COPY infrastructure/aws/lambda/handler.py /asset/handler.py
 

diff --git a/infrastructure/aws/requirements-cdk.txt b/infrastructure/aws/requirements-cdk.txt
diff --git a/pyproject.toml b/pyproject.toml
@@ -2,7 +2,7 @@
 name = "titiler-multidim"
 description = "TiTiler application extension for titiler.xarray."
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 authors = [
     {name = "Vincent Sarago", email = "vincent@developmentseed.com"},
 ]
@@ -28,43 +28,43 @@ dependencies = [
     "titiler.core>=0.19.0,<0.20",
     "titiler.xarray>=0.19.0,<0.20",
     "cftime",
-    "h5netcdf",
-    "numpy<2.0.0",
-    "xarray",
-    "rioxarray",
-    "zarr>=2,<3",
-    "fakeredis",
+    "fastapi",
     "fsspec",
-    "s3fs",
-    "aiohttp",
-    "requests",
+    "h5netcdf",
+    "numpy",
     "pydantic-settings~=2.0",
     "pydantic>=2.4,<3.0",
-    "pandas==1.5.3",
     "redis",
-    "fastapi>=0.108.0,<0.109.0",
-    "starlette>=0.29.0,<0.33.0",
+    "rioxarray",
+    "s3fs",
+    "xarray",
+    "zarr>=2,<3",
 ]
 
 [project.optional-dependencies]
-test = [
-    "pytest",
-    "pytest-cov",
-    "pytest-asyncio",
-    "httpx<0.28",
-    "yappi",
+server = [
+    "uvicorn"
 ]
+
+[dependency-groups]
 dev = [
-    "dask>=2023.5.0",
+    "fakeredis>=2.23.5",
+    "httpx",
     "ipython>=8.12.3",
     "netcdf4>=1.7.2",
-    "pre-commit",
-]
-debug = [
-    "yappi"
+    "pre-commit>=3.5.0",
+    "pytest-asyncio>=0.24.0",
+    "pytest-cov>=5.0.0",
+    "pytest>=8.3.2",
+    "yappi>=1.6.0",
 ]
-server = [
-    "uvicorn"
+deployment = [
+    "aws-cdk-aws-apigatewayv2-alpha==2.76.0a0",
+    "aws-cdk-aws-apigatewayv2-integrations-alpha==2.76.0a0",
+    "aws-cdk-lib==2.76.0",
+    "constructs>=10.4.2",
+    "pydantic-settings~=2.0",
+    "python-dotenv>=1.0.1",
 ]
 
 [project.urls]
@@ -119,6 +119,7 @@ requires = ["pdm-backend"]
 build-backend = "pdm.backend"
 
 
+
 [tool.pdm.version]
 source = "file"
 path = "src/titiler/multidim/__init__.py"