Merge pull request #315 from natrimmer/perf_caches_in_gh_actions

Performance: GitHub Action caching
hotosm · Feb 8, 2025 · b216228 · b216228
2 parents a588e49 + 2121dcc
commit b216228
Show file tree

Hide file tree

Showing 7 changed files with 114 additions and 45 deletions.
diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml
@@ -30,11 +30,14 @@ jobs:
         # needed because the postgres container does not provide a healthcheck
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 2
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out code
+        uses: actions/checkout@v4
+
       - name: Set up Python 3.8
         uses: actions/setup-python@v5
         with:
           python-version: 3.8
+          cache: "pip"
 
       - name: Clone Ramp
         run: git clone https://github.com/kshitijrajsharma/ramp-code-fAIr.git ramp-code
@@ -76,27 +79,26 @@ jobs:
           redis-cli ping
 
       - name: Install Dependencies
+        working-directory: ./backend
         run: |
-          cd backend/
           pip install -r requirements.txt
           pip install coverage
           pip install factory-boy
 
       - name: Create env
+        working-directory: ./backend
         run: |
-          cd backend/
           export DATABASE_URL=postgis://admin:password@localhost:5432/ai
           export RAMP_HOME="/home/runner/work/fAIr/fAIr"
           export TRAINING_WORKSPACE="/home/runner/work/fAIr/fAIr/backend/training"
 
       - name: Run celery worker
-        run: |
-          cd backend/
-          celery -A aiproject worker --loglevel=debug &
+        working-directory: ./backend
+        run: celery -A aiproject worker --loglevel=debug &
+
       - name: Run flower dashboard
-        run: |
-          cd backend/
-          celery -A aiproject --broker=redis://localhost:6379/ flower  &
+        working-directory: ./backend
+        run: celery -A aiproject --broker=redis://localhost:6379/ flower  &
 
       - name: Fix gdal array
         run: |
@@ -115,8 +117,8 @@ jobs:
           OSM_CLIENT_ID: ${{ secrets.OSM_CLIENT_ID }}
           OSM_CLIENT_SECRET: ${{ secrets.OSM_CLIENT_SECRET }}
           OSM_SECRET_KEY: ${{ secrets.OSM_SECRET_KEY }}
+        working-directory: ./backend
         run: |
-          cd backend/
           python manage.py makemigrations
           python manage.py makemigrations core
           python manage.py makemigrations login
@@ -130,9 +132,6 @@ jobs:
           OSM_CLIENT_ID: ${{ secrets.OSM_CLIENT_ID }}
           OSM_CLIENT_SECRET: ${{ secrets.OSM_CLIENT_SECRET }}
           OSM_SECRET_KEY: ${{ secrets.OSM_SECRET_KEY }}
-
-        run: |
-          cd backend/
-          coverage run manage.py test tests
-
+        working-directory: ./backend
+        run: coverage run manage.py test tests
 # enable coverage report with this :           coverage report
diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml
@@ -29,29 +29,40 @@ jobs:
       contents: read
       packages: write
     steps:
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for API Docker
         id: meta_api
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_api
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Build and push API Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: backend/
           file: backend/Dockerfile_CPU
           push: false
           tags: ${{ steps.meta_api.outputs.tags }}
           labels: ${{ steps.meta_api.outputs.labels }}
+          cache-from: type=gha,scope=api,timeout=20m
+          cache-to: type=gha,mode=max,scope=api,timeout=20m,ignore-error=true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
   build-and-push-worker-image:
     needs: build-and-push-api-image
@@ -60,26 +71,37 @@ jobs:
       contents: read
       packages: write
     steps:
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Worker Docker
         id: meta_worker
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_worker
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Build and push Worker Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: backend/
           file: backend/Dockerfile
           push: false
           tags: ${{ steps.meta_worker.outputs.tags }}
           labels: ${{ steps.meta_worker.outputs.labels }}
+          cache-from: type=gha,scope=docker-worker,timeout=20m
+          cache-to: type=gha,mode=min,scope=docker-worker,timeout=20m,ignore-error=true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/docker_publish_image.yml b/.github/workflows/docker_publish_image.yml
@@ -24,29 +24,40 @@ jobs:
       contents: read
       packages: write
     steps:
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for API Docker
         id: meta_api
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_api
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Build and push API Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: backend/
           file: backend/Dockerfile.API
           push: true
           tags: ${{ steps.meta_api.outputs.tags }}
           labels: ${{ steps.meta_api.outputs.labels }}
+          cache-from: type=gha,scope=api,timeout=20m
+          cache-to: type=gha,mode=max,scope=api,timeout=20m,ignore-error=true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
   build-and-push-worker-image:
     needs: build-and-push-api-image
@@ -55,26 +66,37 @@ jobs:
       contents: read
       packages: write
     steps:
+      - name: Remove unnecessary files
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Worker Docker
         id: meta_worker
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_worker
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Build and push Worker Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: backend/
           file: backend/Dockerfile
           push: true
           tags: ${{ steps.meta_worker.outputs.tags }}
           labels: ${{ steps.meta_worker.outputs.labels }}
+          cache-from: type=gha,scope=docker-worker,timeout=20m
+          cache-to: type=gha,mode=min,scope=docker-worker,timeout=20m,ignore-error=true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/frontend_build.yml b/.github/workflows/frontend_build.yml
@@ -31,12 +31,22 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Install dependencies
-        run: |
-          cd frontend/
-          npm install
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        id: npm-cache
+        with:
+          path: |
+            frontend/node_modules
+            ~/.npm
+          key: ${{ runner.os }}-npm-${{ hashFiles('frontend/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-npm-
+
+      - name: Install Node.js dependencies
+        if: steps.npm-cache.outputs.cache-hit != 'true'
+        working-directory: ./frontend
+        run: npm install
 
       - name: Build
-        run: |
-          cd frontend/
-          npm run build
+        working-directory: ./frontend
+        run: npm run build
diff --git a/.github/workflows/frontend_build_push.yml b/.github/workflows/frontend_build_push.yml
@@ -3,9 +3,6 @@ name: Frontend Build and upload to S3
 on:
   release:
     types: [released]
-    paths:
-      - "frontend/**"
-      - ".github/workflows/frontend_build_push.yml"
   workflow_dispatch:
 
 permissions:
@@ -16,6 +13,7 @@ jobs:
   build_and_upload:
     runs-on: ubuntu-latest
     environment: Production
+
     env:
       CI: false
 
@@ -24,16 +22,33 @@ jobs:
         node-version: [20]
 
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - name: Check out Git repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Install dependencies
-        run: cd frontend/ && npm install
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        id: npm-cache
+        with:
+          path: |
+            frontend/node_modules
+            ~/.npm
+          key: ${{ runner.os }}-npm-${{ hashFiles('frontend/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-npm-
+
+      - name: Install Node.js dependencies
+        if: steps.npm-cache.outputs.cache-hit != 'true'
+        working-directory: ./frontend
+        run: npm ci
 
       - name: Build frontend
-        run: cd frontend/ && npm run build
+        working-directory: ./frontend
+        run: npm run build
         env:
           VITE_BASE_API_URL: ${{ vars.VITE_BASE_API_URL }}
           VITE_MATOMO_ID: ${{ vars.VITE_MATOMO_ID }}
@@ -49,4 +64,5 @@ jobs:
           role-session-name: fAIrGithub
 
       - name: Upload to S3
-        run: cd frontend/dist && aws s3 sync . s3://${{ vars.FRONTEND_BUCKET }}/
+        working-directory: ./frontend/dist
+        run: aws s3 sync . s3://${{ vars.FRONTEND_BUCKET }}/
diff --git a/.gitignore b/.gitignore
@@ -21,8 +21,6 @@ postgres-data/*
 frontend/build
 
 # misc
-.DS_Store
-frontend/.DS_Store
 frontend/.env.local
 frontend/.env.development.local
 frontend/.env.test.local
@@ -52,3 +50,5 @@ trainings/*
 backend/.env
 backend/config.txt
 backend/postgres-data
+
+**/.DS_Store
diff --git a/backend/.DS_Store b/backend/.DS_Store