Merge branch 'develop' into openalex-importer

desci-labs · Jan 8, 2025 · 6528ac9 · 6528ac9
2 parents 0376584 + 69c7bf2
commit 6528ac9
Show file tree

Hide file tree

Showing 333 changed files with 34,455 additions and 7,642 deletions.
diff --git a/.ceramicDev.config.json b/.ceramicDev.config.json
@@ -1,5 +1,5 @@
 {
-  "anchor": { },
+  "anchor": {},
   "http-api": {
     "cors-allowed-origins": [
       ".*"
@@ -16,8 +16,8 @@
     "log-level": 2
   },
   "metrics": {
-    "metrics-exporter-enabled": false,
-    "metrics-port": 9090
+    "prometheus-exporter-enabled": true,
+    "prometheus-exporter-port": 9464
   },
   "network": {
     "name": "inmemory"
@@ -33,3 +33,4 @@
     "models": []
   }
 }
+
diff --git a/.env.example b/.env.example
@@ -2,9 +2,11 @@ NODE_ENV=dev
 
 PORT=5420
 
+# Kubo RPC endpoints
 IPFS_NODE_URL=http://host.docker.internal:5001
-PUBLIC_IPFS_RESOLVER=https://ipfs.io
+PUBLIC_IPFS_RESOLVER=http://host.docker.internal:5002
 
+# Kubo path gateway endpoint
 IPFS_RESOLVER_OVERRIDE=http://host.docker.internal:8089/ipfs
 
 ### Database - Postgres
@@ -111,15 +113,18 @@ RUN=1
 # Enable google api functionalities
 GOOGLE_CLIENT_ID=
 GOOGLE_CLIENT_SECRET= # Unnecessary for now, not doing serverside 2step
-GOOGLE_DEV_API_KEY= # Unnecessary for now, not doing serverside 2step
+GOOGLE_DEV_API_KEY=   # Unnecessary for now, not doing serverside 2step
 
 ## Configure RPC nodes (open an issue/ping us to access DeSci Labs' nodes)
 ETHEREUM_RPC_URL=http://host.docker.internal:8545
 
 # Use this for Sepolia testnet
 # ETHEREUM_RPC_URL=https://eth-sepolia.g.alchemy.com/v2/demo
 
-DPID_URL_OVERRIDE=https://dev-beta.dpid.org
+# Use resolver in compose cluster locally
+DPID_URL_OVERRIDE=http://host.docker.internal:5460
+# Use public dev resolver
+# DPID_URL_OVERRIDE=https://dev-beta.dpid.org
 
 # Set to true if you want to mute the publish worker in local dev
 MUTE_PUBLISH_WORKER=false
@@ -129,18 +134,17 @@ MAX_LOCK_TIME=3600 # 1 hour
 DOI_PREFIX=10.62891
 CROSSREF_DOI_URL=https://doi.org
 
-# Cross ref api 
+# Cross ref api
 CROSSREF_METADATA_API=https://test.crossref.org/servlet/deposit
 CROSSREF_ADMIN_API=https://test.crossref.org
 CROSSREF_EMAIL=
 CROSSREF_LOGIN=
+CROSSREF_LOGIN_ROLE=
 CROSSREF_PASSWORD=
 
 # Cross ref notification callback envs
 CROSSREF_NOTIFY_ENDPOINT=endpoint
 
-# automated metadata
-
 # Automated metadata
 AUTOMATED_METADATA_API=http://host.docker.internal:5005
 AUTOMATED_METADATA_API_KEY=
@@ -158,6 +162,9 @@ ES_DB_NAME=
 ES_DB_USER=
 ES_DB_PASSWORD=
 
-
-### open Alex Database - Postgres 
+### open Alex Database - Postgres
 OPEN_ALEX_DATABASE_URL=postgresql://username:password@host/database?schema=openalex
+
+CLOUDFLARE_WORKER_API=http://host.docker.internal:5445
+CLOUDFLARE_WORKER_API_SECRET=auth-token
+ENABLE_WORKERS_API=true
diff --git a/.env.test b/.env.test
@@ -27,7 +27,7 @@ REDIS_PORT=6379
 REDIS_URL=redis://host.docker.internal:6379
 
 # LOCAL DEV
-# http://localhost:1984 
+# http://localhost:1984
 #
 # LIVE:
 # host: 'arweave.net',
@@ -77,7 +77,7 @@ REPO_SERVER_URL=http://host.docker.internal:5485
 DOI_PREFIX=10.62891
 CROSSREF_DOI_URL=https://doi.org
 
-# Cross ref api 
+# Cross ref api
 CROSSREF_METADATA_API=https://test.crossref.org/servlet/deposit
 CROSSREF_ADMIN_API=https://test.crossref.org
 CROSSREF_API_KEY=
@@ -90,4 +90,8 @@ CROSSREF_NOTIFY_ENDPOINT=endpoint
 
 # Automated metadata
 AUTOMATED_METADATA_API=
-AUTOMATED_METADATA_API_KEY=
+AUTOMATED_METADATA_API_KEY=
+
+CLOUDFLARE_WORKER_API=http://host.docker.internal:5446
+CLOUDFLARE_WORKER_API_SECRET=auth-token
+ENABLE_WORKERS_API=true
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
@@ -9,6 +9,7 @@ on:
       - desci-server/**
       - desci-repo/**
       - desci-contracts/**
+      - sync-server/**
       - Dockerfile
 
 name: Test desci-server
@@ -22,15 +23,22 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   CONTAINER_IMAGE: desci-server
   DOCKER_BUILDKIT: 1
+  SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
 
 jobs:
   build-and-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     # container:
     #   image: ubuntu:latest
     #   options: --user 1001
 
     steps:
+      - name: Delete huge unnecessary tools folder
+        run: |
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
+          rm -rf /opt/hostedtoolcache
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
+
       - name: Check out repository
         uses: actions/checkout@v4
 
@@ -48,16 +56,38 @@ jobs:
             desci-repo/yarn.lock
 
       - name: Install dependencies
-        run: cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn --ignore-engines && cd ../desci-repo && yarn
+        run: |
+          cd desci-models && npm i -g yarn && yarn && yarn build && cd ../desci-server && yarn --ignore-engines && cd ../desci-repo && yarn
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
 
       - name: Stub contract
         run: |
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-research-object.json
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-dpid.json
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
+
+      - name: Set up Sentry CLI
+        run: |
+          npm install -g @sentry/cli
+          sentry-cli --version
+          sentry-cli login --auth-token $SENTRY_AUTH_TOKEN
+          # append to .env
+          echo -e "\nSENTRY_AUTH_TOKEN=$SENTRY_AUTH" >> desci-server/.env
+          cd desci-server && yarn build
+
+      # - name: Set up sync server
+      #   run: |
+      #     cd sync-server && yarn --ignore-engines && ./scripts/build.sh test
+      #     if [ $? -ne 0 ]; then
+      #       exit 1
+      #     fi
+      #     echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
 
       - name: Run tests
         run: |
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
           cd desci-server && export DOCKER_BUILDKIT=1 && yarn --ignore-engines && yarn test
           if [ $? -ne 0 ]; then
             exit 1
           fi
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
diff --git a/.github/workflows/build-isolated-media-server.yaml b/.github/workflows/build-isolated-media-server.yaml
@@ -27,7 +27,7 @@ env:
 jobs:
   build-and-push:
     name: Build and deploy
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: hashicorp/setup-terraform@v1
       - name: Checkout

diff --git a/.github/workflows/build-nodes-media.yaml b/.github/workflows/build-nodes-media.yaml
@@ -26,7 +26,7 @@ env:
 jobs:
   build-and-push:
     name: Build and deploy
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: hashicorp/setup-terraform@v1
       - name: Checkout

diff --git a/.github/workflows/build-repo-server.yaml b/.github/workflows/build-repo-server.yaml
@@ -28,7 +28,7 @@ jobs:
   build-and-push:
     # needs: build-and-test
     name: Build and deploy
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: hashicorp/setup-terraform@v1
       - name: Checkout

diff --git a/.github/workflows/build-server.yaml b/.github/workflows/build-server.yaml
@@ -23,12 +23,13 @@ env:
   AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
   CONTAINER_IMAGE: desci-server
   DOCKER_BUILDKIT: 1
 
 jobs:
   build-and-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     # container:
     #   image: ubuntu:latest
     #   options: --user 1001
@@ -37,6 +38,12 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
 
+      - name: Delete huge unnecessary tools folder
+        run: |
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
+          rm -rf /opt/hostedtoolcache
+          echo "DISK USE:"; find / -maxdepth 1 -mindepth 1 -type d -exec du -hs {} \; 2>/dev/null
+
       - name: Set up the environment
         # Replace this line with the appropriate setup for your project
         # Examples:
@@ -57,6 +64,14 @@ jobs:
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-research-object.json
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-dpid.json
 
+      - name: Set up Sentry CLI
+        run: |
+          npm install -g @sentry/cli
+          sentry-cli --version
+          sentry-cli login --auth-token $SENTRY_AUTH_TOKEN
+          # append to .env
+          echo -e "\nSENTRY_AUTH_TOKEN=$SENTRY_AUTH" >> desci-server/.env
+
       - name: Run tests
         run: |
           cd desci-server && export DOCKER_BUILDKIT=1 && yarn --ignore-engines && yarn test
@@ -68,7 +83,7 @@ jobs:
   build-and-push-images:
     # we build and push for every commit, even if tests pass, that way when tests pass deployment is short (run test + build in parallel)
     name: Build and push images
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: hashicorp/setup-terraform@v1
       - name: Checkout
@@ -113,6 +128,14 @@ jobs:
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-research-object.json
           echo "{\"proxies\":[{\"address\":\"\"}]}" > desci-server/src/desci-contracts-config/unknown-dpid.json
 
+      - name: Set up Sentry CLI
+        run: |
+          npm install -g @sentry/cli
+          sentry-cli --version
+          sentry-cli login --auth-token $SENTRY_AUTH_TOKEN
+          # append to .env
+          echo -e "\nSENTRY_AUTH_TOKEN=$SENTRY_AUTH" >> desci-server/.env
+
       - name: Build and tag the image (DEV)
         if: github.ref == 'refs/heads/develop'
         run: |
@@ -169,7 +192,7 @@ jobs:
     needs:
       - build-and-test
       - build-and-push-images
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: hashicorp/setup-terraform@v1
       - name: Checkout

diff --git a/.github/workflows/deploy-sync-server.yaml b/.github/workflows/deploy-sync-server.yaml
@@ -0,0 +1,52 @@
+name: Deploy Worker
+on:
+  push:
+    paths:
+      - .github/workflows/deploy-sync-server.yaml
+      - sync-server/**
+    branches: # array of glob patterns matching against refs/heads. Optional; defaults to all
+      - main # triggers on pushes that contain changes
+      - develop
+env:
+  API_TOKEN: ${{ secrets.API_TOKEN }}
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      # - name: Show github ref
+      #   run: |
+      #     echo ${{ github.ref }}
+      #     echo ${{ github.ref_name }}
+      #     echo ${{ github.head_ref }}
+      # - name: Build & Deploy Worker (Dev)
+      #   if: github.head_ref == 'tay/automerge-party-flagged'
+      #   uses: cloudflare/wrangler-action@v3
+      #   with:
+      #     apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+      #     accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+      #     environment: staging
+      #     workingDirectory: sync-server
+      #     wranglerVersion: 3.95.0
+      - name: Build & Deploy Worker (Staging)
+        if: github.ref == 'refs/heads/develop'
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          environment: staging
+          workingDirectory: sync-server
+          wranglerVersion: 3.95.0
+          secrets: API_TOKEN
+
+      - name: Build & Deploy Worker (Production)
+        if: github.ref == 'refs/heads/main'
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          environment: production
+          workingDirectory: sync-server
+          wranglerVersion: 3.95.0
+          secrets: API_TOKEN
diff --git a/.github/workflows/run-migrations-dev.yaml b/.github/workflows/run-migrations-dev.yaml
@@ -22,6 +22,14 @@ jobs:
         uses: unfor19/install-aws-cli-action@v1
         with:
           version: 1
+          workdir: ${{ runner.temp }}/aws_cli_install
+          rootdir: ${{ runner.temp }}/aws_cli
+          installrootdir: ${{ runner.temp }}/aws
+
+      - name: Add AWS tools to PATH
+        run: |
+          echo "${{ runner.temp }}/aws/bin" >> $GITHUB_PATH
+          echo "${{ runner.temp }}/aws_cli/bin" >> $GITHUB_PATH
 
       - uses: prepor/action-aws-iam-authenticator@master
       - run: aws-iam-authenticator version