From 3a690269df06020847996cd6d4f946c6e417ab17 Mon Sep 17 00:00:00 2001 From: Marc Gavanier Date: Mon, 5 Feb 2024 08:42:18 +0100 Subject: [PATCH] fix: wait for deduplication before extract and publish (#184) --- .github/workflows/nightly-publish.yml | 76 ++++++----------- .github/workflows/release.yml | 112 ++++++++++---------------- .github/workflows/validate.yml | 17 ---- 3 files changed, 66 insertions(+), 139 deletions(-) diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml index e574e081..9fad1570 100644 --- a/.github/workflows/nightly-publish.yml +++ b/.github/workflows/nightly-publish.yml @@ -47,22 +47,18 @@ jobs: ] runs-on: ubuntu-latest environment: production - steps: - name: Checkout mednum-cli repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: 'Create env file' run: | touch .env @@ -73,48 +69,32 @@ jobs: echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env echo DATA_INCLUSION_API_KEY="${{ secrets.DATA_INCLUSION_API_KEY }}" >> .env - - name: Transform run: yarn transformer.${{ matrix.source }} - - name: Publish run: yarn publier.${{ matrix.source }} - - name: Deduplicate run: yarn dedupliquer.${{ matrix.source }} - extract-and-publish: - name: Extract and publish to data.gouv - strategy: - matrix: - source: - [ - extract-eure-et-loir, - extract-mednum-hub-antilles, - extract-numi, - extract-savoie, - extract-vendee - ] + deduplicate: + name: Merge all duplicates between sources runs-on: ubuntu-latest environment: production needs: - transform-and-publish - + - extract-and-publish steps: - - name: Checkout mednum-cli repository + - name: Checkout client-application repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: 'Create env file' run: | touch .env @@ -123,36 +103,38 @@ jobs: echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env + echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env + - name: Deduplicate + run: yarn dedupliquer.all - - name: Extract - run: yarn extract.${{ matrix.source }} - - - name: Publish - run: yarn publier.${{ matrix.source }} - - deduplicate: - name: Merge all duplicates between sources + extract-and-publish: + name: Extract and publish to data.gouv + strategy: + matrix: + source: + [ + extract-eure-et-loir, + extract-mednum-hub-antilles, + extract-numi, + extract-savoie, + extract-vendee + ] runs-on: ubuntu-latest environment: production needs: - - transform-and-publish - - extract-and-publish - + - deduplicate steps: - - name: Checkout client-application repository + - name: Checkout mednum-cli repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: 'Create env file' run: | touch .env @@ -161,10 +143,10 @@ jobs: echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env - echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env - - - name: Deduplicate - run: yarn dedupliquer.all + - name: Extract + run: yarn extract.${{ matrix.source }} + - name: Publish + run: yarn publier.${{ matrix.source }} publish-to-data-gouv: name: Get lieux from API and publish to data.gouv @@ -176,21 +158,17 @@ jobs: - name: Wait for deduplication end run: sleep 300s shell: bash - - name: Checkout mednum-cli repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: 'Create env file' run: | touch .env @@ -199,10 +177,8 @@ jobs: echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env - - name: Extract run: yarn extract.all - - name: Publish run: yarn publier.all @@ -215,12 +191,10 @@ jobs: - name: Sleep for 5 minutes run: sleep 300s shell: bash - - name: Prepare lieux inclusion numerique file run: | mkdir mkdir -p client-application/assets/data curl https://cartographie.societenumerique.gouv.fr/api/v0/lieux-inclusion-numerique | gunzip >> ./client-application/assets/data/lieux-inclusion-numerique.json - - name: Publish to S3 uses: kersvers/s3-sync-with-cloudfront-invalidation@v1.0.0 env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1ba21d6a..42959024 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,22 +52,18 @@ jobs: ] runs-on: ubuntu-latest environment: production - steps: - name: Checkout mednum-cli repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: 'Create env file' run: | touch .env @@ -78,63 +74,13 @@ jobs: echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env echo DATA_INCLUSION_API_KEY="${{ secrets.DATA_INCLUSION_API_KEY }}" >> .env - - name: Transform run: yarn transformer.${{ matrix.source }} - - name: Publish run: yarn publier.${{ matrix.source }} - - name: Internal deduplication run: yarn dedupliquer.${{ matrix.source }} - extract-and-publish: - name: Extract and publish to data.gouv - strategy: - matrix: - source: - [ - extract-eure-et-loir, - extract-mednum-hub-antilles, - extract-numi, - extract-savoie, - extract-vendee - ] - runs-on: ubuntu-latest - environment: production - needs: - - transform-and-publish - - steps: - - name: Checkout mednum-cli repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '16.x' - cache: 'yarn' - - - name: Install dependencies - run: yarn - - - name: 'Create env file' - run: | - touch .env - echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env - echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env - echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env - echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env - echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env - - - name: Extract - run: yarn extract.${{ matrix.source }} - - - name: Publish - run: yarn publier.${{ matrix.source }} - build: uses: romain-cambonie/serenity-workflows/.github/workflows/_build-and-upload-artifact.reusable.yml@master @@ -143,23 +89,19 @@ jobs: runs-on: ubuntu-latest needs: - build - steps: - name: Checkout mednum-cli repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: 'lts/*' - - name: Download release artifact uses: actions/download-artifact@v3 with: name: bundle - - name: Link the semantic release config and release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -176,22 +118,18 @@ jobs: needs: - transform-and-publish - extract-and-publish - steps: - name: Checkout client-application repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: 'Create env file' run: | touch .env @@ -201,10 +139,50 @@ jobs: echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env - - name: Deduplicate run: yarn dedupliquer.all + extract-and-publish: + name: Extract and publish to data.gouv + strategy: + matrix: + source: + [ + extract-eure-et-loir, + extract-mednum-hub-antilles, + extract-numi, + extract-savoie, + extract-vendee + ] + runs-on: ubuntu-latest + environment: production + needs: + - deduplicate + steps: + - name: Checkout mednum-cli repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '16.x' + cache: 'yarn' + - name: Install dependencies + run: yarn + - name: 'Create env file' + run: | + touch .env + echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env + echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env + echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env + echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env + echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env + - name: Extract + run: yarn extract.${{ matrix.source }} + - name: Publish + run: yarn publier.${{ matrix.source }} + publish-to-data-gouv: name: Get lieux from API and publish to data.gouv runs-on: ubuntu-latest @@ -215,21 +193,17 @@ jobs: - name: Wait for deduplication end run: sleep 300s shell: bash - - name: Checkout mednum-cli repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: 'Create env file' run: | touch .env @@ -238,10 +212,8 @@ jobs: echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env - - name: Extract run: yarn extract.all - - name: Publish run: yarn publier.all @@ -254,12 +226,10 @@ jobs: - name: Sleep for 5 minutes run: sleep 300s shell: bash - - name: Prepare lieux inclusion numerique file run: | mkdir mkdir -p client-application/assets/data curl https://cartographie.societenumerique.gouv.fr/api/v0/lieux-inclusion-numerique | gunzip >> ./client-application/assets/data/lieux-inclusion-numerique.json - - name: Publish to S3 uses: kersvers/s3-sync-with-cloudfront-invalidation@v1.0.0 env: diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 9ebd328f..a9d8c95a 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -25,25 +25,20 @@ jobs: commitlint: name: CommitLint runs-on: ubuntu-latest - steps: - name: Checkout client-application repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: commitlint run: yarn lint.commit - validation-matrix: uses: romain-cambonie/serenity-workflows/.github/workflows/_validation-matrix.reusable.yml@master with: @@ -90,22 +85,18 @@ jobs: ] runs-on: ubuntu-latest environment: demo - steps: - name: Checkout client-application repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: 'Create env file' run: | touch .env @@ -115,10 +106,8 @@ jobs: echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env echo DATA_INCLUSION_API_KEY="${{ secrets.DATA_INCLUSION_API_KEY }}" >> .env echo MAX_TRANSFORM=1000 >> .env - - name: Transform run: yarn transformer.${{ matrix.source }} - - name: Publish run: yarn publier.${{ matrix.source }} @@ -136,22 +125,18 @@ jobs: ] runs-on: ubuntu-latest environment: demo - steps: - name: Checkout client-application repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '16.x' cache: 'yarn' - - name: Install dependencies run: yarn - - name: 'Create env file' run: | touch .env @@ -159,9 +144,7 @@ jobs: echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env - - name: Extract run: yarn extract.${{ matrix.source }} - - name: Publish run: yarn publier.${{ matrix.source }}