fix: wait for deduplication before extract and publish (#184)

anct-cartographie-nationale · Feb 5, 2024 · 3a69026 · 3a69026
1 parent 3959620
commit 3a69026
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 139 deletions.
diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml
@@ -47,22 +47,18 @@ jobs:
           ]
     runs-on: ubuntu-latest
     environment: production
-
     steps:
       - name: Checkout mednum-cli repository
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
           node-version: '16.x'
           cache: 'yarn'
-
       - name: Install dependencies
         run: yarn
-
       - name: 'Create env file'
         run: |
           touch .env
@@ -73,48 +69,32 @@ jobs:
           echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env
           echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env
           echo DATA_INCLUSION_API_KEY="${{ secrets.DATA_INCLUSION_API_KEY }}" >> .env
-
       - name: Transform
         run: yarn transformer.${{ matrix.source }}
-
       - name: Publish
         run: yarn publier.${{ matrix.source }}
-
       - name: Deduplicate
         run: yarn dedupliquer.${{ matrix.source }}
 
-  extract-and-publish:
-    name: Extract and publish to data.gouv
-    strategy:
-      matrix:
-        source:
-          [
-            extract-eure-et-loir,
-            extract-mednum-hub-antilles,
-            extract-numi,
-            extract-savoie,
-            extract-vendee
-          ]
+  deduplicate:
+    name: Merge all duplicates between sources
     runs-on: ubuntu-latest
     environment: production
     needs:
       - transform-and-publish
-
+      - extract-and-publish
     steps:
-      - name: Checkout mednum-cli repository
+      - name: Checkout client-application repository
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
           node-version: '16.x'
           cache: 'yarn'
-
       - name: Install dependencies
         run: yarn
-
       - name: 'Create env file'
         run: |
           touch .env
@@ -123,36 +103,38 @@ jobs:
           echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env
           echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env
           echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env
+          echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env
+      - name: Deduplicate
+        run: yarn dedupliquer.all
 
-      - name: Extract
-        run: yarn extract.${{ matrix.source }}
-
-      - name: Publish
-        run: yarn publier.${{ matrix.source }}
-
-  deduplicate:
-    name: Merge all duplicates between sources
+  extract-and-publish:
+    name: Extract and publish to data.gouv
+    strategy:
+      matrix:
+        source:
+          [
+            extract-eure-et-loir,
+            extract-mednum-hub-antilles,
+            extract-numi,
+            extract-savoie,
+            extract-vendee
+          ]
     runs-on: ubuntu-latest
     environment: production
     needs:
-      - transform-and-publish
-      - extract-and-publish
-
+      - deduplicate
     steps:
-      - name: Checkout client-application repository
+      - name: Checkout mednum-cli repository
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
           node-version: '16.x'
           cache: 'yarn'
-
       - name: Install dependencies
         run: yarn
-
       - name: 'Create env file'
         run: |
           touch .env
@@ -161,10 +143,10 @@ jobs:
           echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env
           echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env
           echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env
-          echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env
-
-      - name: Deduplicate
-        run: yarn dedupliquer.all
+      - name: Extract
+        run: yarn extract.${{ matrix.source }}
+      - name: Publish
+        run: yarn publier.${{ matrix.source }}
 
   publish-to-data-gouv:
     name: Get lieux from API and publish to data.gouv
@@ -176,21 +158,17 @@ jobs:
       - name: Wait for deduplication end
         run: sleep 300s
         shell: bash
-
       - name: Checkout mednum-cli repository
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
           node-version: '16.x'
           cache: 'yarn'
-
       - name: Install dependencies
         run: yarn
-
       - name: 'Create env file'
         run: |
           touch .env
@@ -199,10 +177,8 @@ jobs:
           echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env
           echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env
           echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env
-
       - name: Extract
         run: yarn extract.all
-
       - name: Publish
         run: yarn publier.all
 
@@ -215,12 +191,10 @@ jobs:
       - name: Sleep for 5 minutes
         run: sleep 300s
         shell: bash
-
       - name: Prepare lieux inclusion numerique file
         run: |
           mkdir mkdir -p client-application/assets/data
           curl https://cartographie.societenumerique.gouv.fr/api/v0/lieux-inclusion-numerique | gunzip >> ./client-application/assets/data/lieux-inclusion-numerique.json
-
       - name: Publish to S3
         uses: kersvers/s3-sync-with-cloudfront-invalidation@v1.0.0
         env: