Added Cloudflare Workers KV storage support, and improved k8s support

.gitignore

@@ -1,2 +1,3 @@
 .direnv/
 .envrc
+secrets.yml

.utils/apply

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ "$1" = "all" ]; then
+  TARGETS=$(kubectx)
+else
+  TARGETS=$1
+fi
+
+echo "==================================================="
+while IFS= read -r line
+do
+  echo "Deploying to ${line}"
+  echo "==================================================="
+
+  kubectx "$line" > /dev/null
+  cat k8s.yml | envsubst | kubectl apply -n ${NAMESPACE} -f -
+
+  echo ""
+  echo "==================================================="
+done <<< $TARGETS

.utils/delete

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ "$1" = "all" ]; then
+  TARGETS=$(kubectx)
+else
+  TARGETS=$1
+fi
+
+echo "==================================================="
+while IFS= read -r line
+do
+  echo "Deploying to ${line}"
+  echo "==================================================="
+
+  kubectx "$line" > /dev/null
+  cat k8s.yml | envsubst | kubectl delete -n ${NAMESPACE} -f -
+
+  echo ""
+  echo "==================================================="
+done <<< $TARGETS

.utils/rollout

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ "$1" = "all" ]; then
+  TARGETS=$(kubectx)
+else
+  TARGETS=$1
+fi
+
+echo "==================================================="
+while IFS= read -r line
+do
+  echo "Rolling to ${line}"
+  echo "==================================================="
+
+  kubectx "$line" > /dev/null
+  kubectl rollout restart deployment/${NAME} -n ${NAMESPACE} && kubectl rollout status deployment/${NAME} -n ${NAMESPACE}
+
+  echo ""
+  echo "==================================================="
+done <<< $TARGETS

Dockerfile

@@ -0,0 +1,19 @@
+FROM alpine:latest
+MAINTAINER Matthew Gall <docker@matthewgall.com>
+
+RUN apk add --update \
+	build-base \
+	python3 \
+	python3-dev \
+	py-pip \
+	openssl-dev \
+	libffi-dev \
+	&& rm -rf /var/cache/apk/*
+
+WORKDIR /app
+COPY . /app
+
+RUN pip3 install --upgrade pip && \
+    pip3 install -r /app/requirements.txt
+
+CMD ["python3", "/app/app.py"]

Makefile

@@ -0,0 +1,22 @@
+export NAME?=drandarchive
+export NAMESPACE?=drandarchive
+export IMAGE?=matthewgall/drandarchive.com
+export COLO:=$(shell kubectx -c)
+
+.PHONY: apply apply-all
+apply:
+	@.utils/apply ${COLO}
+apply-all:
+	@.utils/apply all
+
+.PHONY: delete delete-all
+delete:
+	@.utils/delete ${COLO}
+delete-all:
+	@.utils/delete all
+
+.PHONY: rollout rollout-all
+rollout:
+	@.utils/rollout ${COLO}
+rollout-all:
+	@.utils/rollout all

app.py

@@ -13,7 +13,6 @@ def backfill():
         log.debug("Processing round: {}".format(i))
         get_round(i)
 
-
 def get_round(i):
     try:
         r = requests.get('https://{}/public/{}'.format(args.drand, i), timeout=2).json()
@@ -33,17 +32,46 @@ def get_latest_round():
     except:
         return None
 
+def sendto_cf(d):
+    try:
+        r = requests.put(
+            "https://api.cloudflare.com/client/v4/accounts/{}/storage/kv/namespaces/{}/values/{}".format(
+                args.cf_account,
+                args.cf_namespace,
+                "{}.{}".format(args.drand.replace(".", ""), d['round'])
+            ),
+            headers={
+                'Authorization': "Bearer {}".format(args.cf_token)
+            },
+            data=json.dumps(d)
+        )
+        print(r.json())
+    except:
+        return None
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser()
 
     # drand server
-    parser.add_argument("-d", "--drand", help="drand server to archive", default="drand.cloudflare.com")
+    parser.add_argument("-d", "--drand", help="drand server to archive", default=os.getenv("DRAND_SERVER", "drand.cloudflare.com"))
 
     # modes
     parser.add_argument("-b", "--backfill", help="backfill workers KV with the results of all rounds", action="store_true")
     parser.add_argument("-sv", "--server", help="run a server to update the archive as live", action="store_true")
 
+    # config
+    parser.add_argument("-dl", "--delay", help="pause between each check", default=os.getenv("DELAY", "5"))
+
+    # storage
+    ## cloudflare (workers kv)
+    parser.add_argument("--cf", help="store data using Cloudflare Workers KV", action="store_true")
+    parser.add_argument("--cf-account", help="cloudflare account id", default=os.getenv("CLOUDFLARE_ACCOUNT", ""))
+    parser.add_argument("--cf-token", help="cloudflare api token", default=os.getenv("CLOUDFLARE_TOKEN", ""))
+    parser.add_argument("--cf-namespace", help="cloudflare kv namespace", default=os.getenv("CLOUDFLARE_NAMESPACE", ""))
+
+    ## s3
+
     # Verbose mode
     parser.add_argument("--verbose", "-v", help="increase output verbosity", action="store_true")
     args = parser.parse_args()
@@ -73,11 +101,13 @@ def get_latest_round():
                 d = get_round(r)
 
                 # Publish it
-
+                if args.cf:
+                    sendto_cf(d)
+
                 # Update the pointer
                 l = d['round']
             else:
                 log.info("Round has not proceeded yet, still at round {}".format(r))
 
             # And wait for the next round
-            time.sleep(3)
+            time.sleep(int(args.delay))

k8s.yml

@@ -0,0 +1,81 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ${NAMESPACE}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${NAME}-drandcloudflarecom-backfill
+  labels:
+    app: ${NAME}-drandcloudflarecom-backfill
+spec:
+  template:
+    spec:
+      containers:
+      - name: ${NAME}-drandcloudflarecom-backfill
+        image: matthewgall/drandarchive:latest
+        args: ["--backfill", "--cf"]
+        env:
+          - name: DRAND_SERVER
+            value: drand.cloudflare.com
+          - name: DELAY
+            value: "120"
+          - name: CLOUDFLARE_ACCOUNT
+            valueFrom:
+              secretKeyRef:
+                name: ${NAME}-secrets
+                key: cf-account
+          - name: CLOUDFLARE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: ${NAME}-secrets
+                key: cf-token
+          - name: CLOUDFLARE_NAMESPACE
+            valueFrom:
+              secretKeyRef:
+                name: ${NAME}-secrets
+                key: cf-namespace
+      restartPolicy: Never
+  backoffLimit: 4
+--
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${NAME}-drandcloudflarecom-server
+  labels:
+    app: ${NAME}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ${NAME}-drandcloudflarecom-server
+  template:
+    metadata:
+      labels:
+        app: ${NAME}-drandcloudflarecom-server
+    spec:
+      containers:
+      - name: ${NAME}-drandcloudflarecom-server
+        image: matthewgall/drandarchive.com:latest
+        args: ["--sv", "--cf"]
+        env:
+          - name: DRAND_SERVER
+            value: drand.cloudflare.com
+          - name: DELAY
+            value: "10"
+          - name: CLOUDFLARE_ACCOUNT
+            valueFrom:
+              secretKeyRef:
+                name: ${NAME}-secrets
+                key: cf-account
+          - name: CLOUDFLARE_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: ${NAME}-secrets
+                key: cf-token
+          - name: CLOUDFLARE_NAMESPACE
+            valueFrom:
+              secretKeyRef:
+                name: ${NAME}-secrets
+                key: cf-namespace