Merge pull request #274 from desci-labs/m0ar/reverse-proxy-service

Add `reverse-proxy` for hiding sensitive endpoints
desci-labs · Apr 17, 2024 · 5be4f56 · 5be4f56
2 parents 2395429 + 058f412
commit 5be4f56
Show file tree

Hide file tree

Showing 13 changed files with 3,230 additions and 1 deletion.
diff --git a/.github/workflows/build-reverse-proxy.yaml b/.github/workflows/build-reverse-proxy.yaml
@@ -0,0 +1,112 @@
+# build.yml
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - reverse-proxy/**
+    branches:
+      - develop
+      - main
+
+# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
+env:
+  AWS_DEFAULT_REGION: us-east-2
+  AWS_DEFAULT_OUTPUT: json
+  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CONTAINER_IMAGE: reverse-proxy
+  DOCKER_BUILDKIT: 1
+
+jobs:
+  build-and-push:
+    name: Build and deploy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Add steps here like linting, testing, minification, etc.
+      - id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+        with:
+          version: 1
+
+      - uses: prepor/action-aws-iam-authenticator@master
+      - run: aws-iam-authenticator version
+
+      - name: Install Kubectl
+        run: |
+          #$(curl -Ls https://dl.k8s.io/release/stable.txt)
+          version=v1.23.6
+          echo "using kubectl@$version"
+          curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          mkdir $HOME/.kube
+          sudo apt-get update
+          sudo apt-get install less
+          echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
+          aws sts get-caller-identity
+          kubectl describe deployments
+
+      - name: Build and tag the image (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          # Build and tag the image
+          docker build \
+            -t $CONTAINER_IMAGE-dev:latest \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev \
+             ./$CONTAINER_IMAGE
+
+      - name: Build and tag the image (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Build and tag the image
+          docker build \
+            -t $CONTAINER_IMAGE-prod:latest \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod \
+             ./$CONTAINER_IMAGE
+
+      - name: Push (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE-dev:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest
+
+      - name: Push (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE-prod:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE-prod:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:latest
+
+      - name: Deploy to EKS (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: | # defaults to latest kubectl binary version
+          kubectl apply -f $CONTAINER_IMAGE/kubernetes/deployment_dev.yaml
+          kubectl set image deployment/reverse-proxy-dev reverse-proxy-dev=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} --record
+
+      - name: Deploy to EKS (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: | # defaults to latest kubectl binary version
+          kubectl apply -f $CONTAINER_IMAGE/kubernetes/deployment_prod.yaml
+          kubectl set image deployment/reverse-proxy-prod reverse-proxy-prod=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }} --record
+
+      - name: Verify EKS Deployment (DEV)
+        if: github.ref == 'refs/heads/develop'
+        run: |
+          kubectl rollout status deployment/reverse-proxy-dev
+
+      - name: Verify EKS Deployment (PROD)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          kubectl rollout status deployment/reverse-proxy-prod
diff --git a/.github/workflows/deploy-staging-services.yaml b/.github/workflows/deploy-staging-services.yaml
@@ -257,3 +257,61 @@ jobs:
       - name: Verify EKS Deployment (STAGING)'
         run: |
           kubectl rollout status deployment/desci-server-staging
+
+  build-reverse-proxy:
+    name: Build and deploy reverse-proxy STAGING
+    runs-on: ubuntu-latest
+    steps:
+      - uses: hashicorp/setup-terraform@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Add steps here like linting, testing, minification, etc.
+      - id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+        with:
+          version: 1
+
+      - uses: prepor/action-aws-iam-authenticator@master
+      - run: aws-iam-authenticator version
+
+      - name: Install Kubectl
+        run: |
+          #$(curl -Ls https://dl.k8s.io/release/stable.txt)
+          version=v1.23.6
+          echo "using kubectl@$version"
+          curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl
+          chmod +x kubectl
+          mv kubectl /usr/local/bin
+          mkdir $HOME/.kube
+          sudo apt-get update
+          sudo apt-get install less
+          echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config
+          aws sts get-caller-identity
+          kubectl describe deployments
+
+      - name: Build and tag the image (STAGING)
+        run: |
+          # Build and tag the image
+          docker build \
+            -t $CONTAINER_IMAGE-staging:latest \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging \
+             ./$CONTAINER_IMAGE
+
+      - name: Push (STAGING)
+        run: |
+          # Push image to AWS ECR
+          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
+          docker tag $CONTAINER_IMAGE-staging:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:${{ github.sha }}
+          docker tag $CONTAINER_IMAGE-staging:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:latest
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:${{ github.sha }}
+          docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:latest
+      
+      - name: Deploy to EKS (STAGING)
+        run: | # defaults to latest kubectl binary version
+          kubectl apply -f $CONTAINER_IMAGE/deployment.yaml
+          kubectl set image deployment/reverse-proxy-staging reverse-proxy-staging=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-staging:${{ github.sha }} --record
+
+      - name: Verify EKS Deployment (STAGING)
+        run: |
+          kubectl rollout status deployment/reverse-proxy-staging
diff --git a/README.md b/README.md
@@ -77,9 +77,13 @@ This is a NodeJS backend that manages draft Nodes. It maintains a user auth, ver
 
 ## **nodes-lib**
 
-A library for programmatically interacting with Nodes, basically allowing automation of actions possible in the webapp. See separate documentation in the [README](./nodes-lib/README.md)
+A library for programmatically interacting with Nodes, basically allowing automation of actions possible in the webapp. See separate documentation in the [README](./nodes-lib/README.md).
 <br><br>
 
+## **reverse-proxy-service**
+
+A tiny service for proxying route segments to given target URL's, allowing hiding many target destinations under aliases on one domain. See docs in [README](./reverse-proxy-service/README.md).
+
 ## **desci-art-viewer**
 
 Nobody knows why this is still here, but it implements a React+Three.js 3d torus that plays [Conway's Game of Life](https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life) on the surface of the torus. We were totally inspired by [this gif on Wikipedia](https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life#/media/File:Trefoil_knot_conways_game_of_life.gif) and it only seems to work on Mac/Linux right now, YMMV.

diff --git a/reverse-proxy/.gitignore b/reverse-proxy/.gitignore
@@ -0,0 +1,2 @@
+node_modules
+dist
diff --git a/reverse-proxy/Dockerfile b/reverse-proxy/Dockerfile
@@ -0,0 +1,34 @@
+FROM node:20.12.1-alpine3.18 AS base
+
+RUN apk update && apk add --no-cache dumb-init
+
+WORKDIR /usr/src/app
+
+COPY package*.json .
+
+####################
+FROM base as builder
+
+RUN --mount=type=cache,target=/usr/src/app/.npm \
+  npm set cache /usr/src/app/.npm && \
+  npm ci
+
+COPY . .
+
+RUN npm run build
+
+#######################
+FROM base AS production
+
+RUN --mount=type=cache,target=/usr/src/app/.npm \
+  npm set cache /usr/src/app/.npm && \
+  npm ci --only=production
+
+ENV NODE_ENV production
+USER node
+
+COPY --chown=node:node --from=builder /usr/src/app/dist ./dist
+
+EXPOSE 5678
+
+CMD [ "dumb-init", "node", "dist/index.js"]
diff --git a/reverse-proxy/README.md b/reverse-proxy/README.md
@@ -0,0 +1,33 @@
+# Reverse Proxy Service
+Allows mapping paths to other URLs, useful for hiding target services. In particular, where the target URL contains authentication information which might otherwise leak.
+
+## Configuration
+Set one or more environment variables matching this pattern, where `EXAMPLE` is the path that will redirect to the target URL:
+
+```shell
+PROXY_MAPPING_EXAMPLE=https://example.com
+```
+
+So, in general, the pattern is `PROXY_MAPPING_[redirect key]=[redirect target]`. The redirect key is removed from the request, but the rest stays. Here are some examples of that that means in practice:
+
+| Path            | Resolves to              |
+|-----------------|--------------------------|
+| `/example`      | https://example.com      |
+| `/example/cats` | https://example.com/cats |
+
+## Running
+Run in development mode:
+```shell
+PROXY_MAPPING_EXAMPLE=https://example.com npm run dev
+```
+
+Run in docker:
+```shell
+docker build . -t proxy-service
+docker run --network=host -e PROXY_MAPPING_EXAMPLE="https://example.com" proxy-service
+```
+
+Regardless of which mode, the service listens on port 5678. It can be tested by sending a request:
+```shell
+curl http://localhost:5678/example
+```
diff --git a/reverse-proxy/kubernetes/deployment_dev.yaml b/reverse-proxy/kubernetes/deployment_dev.yaml
@@ -0,0 +1,93 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: public-reverse-proxy-dev-service
+  labels:
+    App: ReverseProxyDev
+spec:
+  type: LoadBalancer
+  selector:
+    App: ReverseProxyDev
+  ports:
+    - protocol: TCP
+      name: api
+      port: 80
+      targetPort: 5678
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: reverse-proxy-dev-ingress
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+spec:
+  rules:
+    - host: reverse-proxy-dev.desci.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: public-reverse-proxy-dev
+                port:
+                  number: 80
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: reverse-proxy-dev
+  labels:
+    App: ReverseProxyDev
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      App: ReverseProxyDev
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject: 'true'
+        vault.hashicorp.com/agent-inject-status: 'update'
+        vault.hashicorp.com/role: app-vault-reader
+        vault.hashicorp.com/agent-inject-secret-config: secrets/desci-server/dev/db
+        vault.hashicorp.com/agent-inject-template-config: |
+          {{- with secret "secrets/desci-server/dev/" -}}
+          echo "sourcing";
+          export PROXY_MAPPING_RPC_SEPOLIA={{ .Data.PROXY_MAPPING_RPC_SEPOLIA }}
+          export PROXY_MAPPING_RPC_MAINNET={{ .Data.PROXY_MAPPING_RPC_MAINNET }}
+
+          {{- end -}}
+      labels:
+        App: ReverseProxyDev
+    spec:
+      containers:
+        - image: 523044037273.dkr.ecr.us-east-2.amazonaws.com/reverse-proxy-dev:latest
+          name: reverse-proxy-dev
+          command: ['/bin/bash', '-c']
+          args:
+            - echo "SOURCING ENV"; source /vault/secrets/config; node ./dist/index.js;
+          ports:
+            - containerPort: 5678
+              name: api
+          resources:
+            limits:
+              cpu: '1'
+              memory: 2Gi
+            requests:
+              cpu: '1'
+              memory: 2Gi
+          # restart pod after failureThreshold*periodSeconds total seconds
+          livenessProbe:
+            httpGet:
+              path: /healthcheck
+              port: api
+            failureThreshold: 10
+            periodSeconds: 3
+      serviceAccountName: 'vault-auth'