Implement VK with token refresh

Author: dciangot

.github/workflows/build_images.yaml

@@ -37,6 +37,38 @@ jobs:
           file: ./Dockerfile.vk
           platforms: linux/amd64, linux/arm64
 
+  virtual-kubelet-refrest-token:
+    runs-on: ubuntu-latest
+    #env:
+    #  DOCKER_TARGET_PLATFORM: linux/arm64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set env
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Get Repo Owner
+        id: get_repo_owner
+        run: echo ::set-output name=repo_owner::$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')
+      - name: Build container base image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          outputs: "type=registry,push=true"
+          tags: |
+            ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/virtual-kubelet-inttw-refresh:${{ env.RELEASE_VERSION }}
+          file: ./Dockerfile.refresh-token
+          platforms: linux/amd64, linux/arm64
+
   interlink:
     runs-on: ubuntu-latest
     #env:

Dockerfile.refresh-token

@@ -0,0 +1,7 @@
+FROM python:3.10
+
+RUN pip3 install requests
+
+COPY ./scripts/refresh.py /opt/refresh.py
+
+ENTRYPOINT ["python3", "/opt/refresh.py"]

docs/itwinctl.sh

@@ -2,7 +2,7 @@
 
 #export INTERLINKCONFIGPATH="$PWD/kustomizations/InterLinkConfig.yaml"
 
-VERSION="${VERSION:-0.0.4}"
+VERSION="${VERSION:-0.0.5}"
 
 OS=$(uname -s)
 
@@ -30,13 +30,14 @@ AUTHORIZED_GROUPS="${AUTHORIZED_GROUPS:-intw}"
 AUTHORIZED_AUD="${AUTHORIZED_AUD:-intertw-vk}"
 API_HTTP_PORT="${API_HTTP_PORT:-8080}"
 API_HTTPS_PORT="${API_HTTPS_PORT:-443}"
-INTERLINKPORT="${INTERLINKPORT:-3000}"
-INTERLINKURL="${INTERLINKURL:-http://0.0.0.0}"
-INTERLINKPORT="${INTERLINKPORT:-3000}"
-INTERLINKURL="${INTERLINKURL:-http://0.0.0.0}"
-INTERLINKCONFIGPATH="${INTERLINKCONFIGPATH:-$HOME/.config/interlink/InterLinkConfig.yaml}"
+export INTERLINKPORT="${INTERLINKPORT:-3000}"
+export INTERLINKURL="${INTERLINKURL:-http://0.0.0.0}"
+export INTERLINKPORT="${INTERLINKPORT:-3000}"
+export INTERLINKURL="${INTERLINKURL:-http://0.0.0.0}"
+export INTERLINKCONFIGPATH="${INTERLINKCONFIGPATH:-$HOME/.config/interlink/InterLinkConfig.yaml}"
+export SBATCHPATH="${SBATCHPATH:-/usr/bin/sbatch}"
+export SCANCELPATH="${SCANCELPATH:-/usr/bin/scancel}"
 
-export INTERLINKCONFIGPATH=$INTERLINKCONFIGPATH
 
 install () {
     mkdir -p $HOME/.local/interlink/logs || exit 1
@@ -119,4 +120,6 @@ case "$1" in
         stop
         start
         ;;
+    uninstall)
+        rm -r $HOME/.local/interlink
 esac

kustomizations/InterLinkConfig.yaml

@@ -1,9 +1,10 @@
+VKTokenFile: "/opt/interlink/token"
 InterlinkURL: "http://localhost"
 SidecarURL: "http://localhost"
 InterlinkPort: "3000"
 SidecarService: "slurm" #docker, slurm
 SbatchPath: "/usr/bin/sbatch"
-ScancelPath: "/usr/bin/sbatch"
+ScancelPath: "/usr/bin/scancel"
 CommandPrefix: ""
 Tsocks: false
 TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so"

kustomizations/deployment.yaml

@@ -43,6 +43,8 @@ spec:
               fieldPath: status.podIP
         - name: INTERLINKCONFIGPATH
           value: "/etc/interlink/InterLinkConfig.yaml"
+        - name: VKTOKENFILE
+          value: "/opt/interlink/token"
         - name: VK_CONFIG_PATH
           value: "/etc/interlink/knoc-cfg.json"
         volumeMounts:
@@ -52,6 +54,40 @@ spec:
         - name: config-json
           mountPath: /etc/interlink/knoc-cfg.json
           subPath: knoc-cfg.json
+        - name: token
+          mountPath: /opt/interlink
+        resources:
+          limits:
+            cpu: 2000m
+            memory: 2Gi
+          requests:
+            cpu: 2000m
+            memory: 2Gi
+      - name: refresh-token
+        image: docker.io/surax98/vk:latest
+        imagePullPolicy: Always
+        env:
+        - name: IAM_SERVER
+          value: "https://dodas-iam.cloud.cnaf.infn.it/"
+        # TODO load env IAM client from secret
+        - name: IAM_CLIENT_ID
+          value: "DUMMY"
+        - name: IAM_CLIENT_SECRET
+          value: "DUMMY"
+        - name: IAM_VK_AUD
+          value: intertw-vk
+        - name: TOKEN_PATH
+          value: /opt/interlink/token
+        resources:
+          limits:
+            cpu: 2000m
+            memory: 2Gi
+          requests:
+            cpu: 2000m
+            memory: 2Gi
+        volumeMounts:
+        - name: token
+          mountPath: /opt/interlink
       serviceAccountName: knoc
       volumes:
       - name: config
@@ -62,5 +98,6 @@ spec:
         configMap:
           # Provide the name of the ConfigMap you want to mount.
           name: test-vk-config
-
+      - name: token
+        emptyDir: {}
 

pkg/common/func.go

@@ -102,6 +102,19 @@ func NewInterLinkConfig() {
 			InterLinkConfigInst.Tsockspath = path
 		}
 
+		if os.Getenv("VKTOKENFILE") != "" {
+			path := os.Getenv("VKTOKENFILE")
+			if _, err := os.Stat(path); err != nil {
+				log.Println("File " + path + " doesn't exist. You can set a custom path by exporting VKTOKENFILE. Exiting...")
+				os.Exit(-1)
+			}
+
+			InterLinkConfigInst.VKTokenFile = path
+		} else {
+			path = "/tmp/token"
+			InterLinkConfigInst.VKTokenFile = path
+		}
+
 		InterLinkConfigInst.set = true
 	}
 }

pkg/common/types.go

@@ -29,6 +29,7 @@ type Request struct {
 }
 
 type InterLinkConfig struct {
+	VKTokenFile    string `yaml:"VKTokenFile"`
 	Interlinkurl   string `yaml:"InterlinkURL"`
 	Sidecarurl     string `yaml:"SidecarURL"`
 	Sbatchpath     string `yaml:"SbatchPath"`

pkg/virtualkubelet/execute.go

@@ -4,8 +4,10 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"strings"
 
 	common "github.com/CARV-ICS-FORTH/knoc/common"
@@ -18,7 +20,7 @@ import (
 
 var NoReq uint8
 
-func createRequest(pod commonIL.Request) []byte {
+func createRequest(pod commonIL.Request, token string) []byte {
 	var returnValue, _ = json.Marshal(commonIL.PodStatus{PodStatus: commonIL.UNKNOWN})
 
 	bodyBytes, err := json.Marshal(pod)
@@ -29,6 +31,7 @@ func createRequest(pod commonIL.Request) []byte {
 		log.L.Error(err)
 	}
 
+	req.Header.Add("Authorization", "Bearer "+token)
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		log.L.Error(err)
@@ -41,7 +44,7 @@ func createRequest(pod commonIL.Request) []byte {
 	return returnValue
 }
 
-func deleteRequest(pod commonIL.Request) []byte {
+func deleteRequest(pod commonIL.Request, token string) []byte {
 	var returnValue, _ = json.Marshal(commonIL.PodStatus{PodStatus: commonIL.UNKNOWN})
 
 	bodyBytes, err := json.Marshal(pod)
@@ -51,6 +54,7 @@ func deleteRequest(pod commonIL.Request) []byte {
 		log.L.Error(err)
 	}
 
+	req.Header.Add("Authorization", "Bearer "+token)
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		log.L.Error(err)
@@ -63,7 +67,7 @@ func deleteRequest(pod commonIL.Request) []byte {
 	return returnValue
 }
 
-func statusRequest(podsList commonIL.Request) []byte {
+func statusRequest(podsList commonIL.Request, token string) []byte {
 	var returnValue []byte
 	var response []commonIL.StatusResponse
 
@@ -76,6 +80,8 @@ func statusRequest(podsList commonIL.Request) []byte {
 
 	log.L.Println(string(bodyBytes))
 
+	req.Header.Add("Authorization", "Bearer "+token)
+
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		log.L.Error(err)
@@ -91,26 +97,32 @@ func RemoteExecution(p *VirtualKubeletProvider, ctx context.Context, mode int8,
 	var req commonIL.Request
 	req.Pods = map[string]*v1.Pod{pod.Name: pod}
 
+	b, err := os.ReadFile(commonIL.InterLinkConfigInst.VKTokenFile) // just pass the file name
+	if err != nil {
+		fmt.Print(err)
+	}
+	token := string(b)
+
 	switch mode {
 	case common.CREATE:
 		//v1.Pod used only for secrets and volumes management; TO BE IMPLEMENTED
-		returnVal := createRequest(req)
+		returnVal := createRequest(req, token)
 		log.L.Println(string(returnVal))
 		break
 
 	case common.DELETE:
 		if NoReq > 0 {
 			NoReq--
 		} else {
-			returnVal := deleteRequest(req)
+			returnVal := deleteRequest(req, token)
 			log.L.Println(string(returnVal))
 		}
 		break
 	}
 	return nil
 }
 
-func checkPodsStatus(p *VirtualKubeletProvider, ctx context.Context) {
+func checkPodsStatus(p *VirtualKubeletProvider, ctx context.Context, token string) {
 	if len(p.pods) == 0 {
 		return
 	}
@@ -119,7 +131,7 @@ func checkPodsStatus(p *VirtualKubeletProvider, ctx context.Context) {
 	var PodsList commonIL.Request
 	PodsList.Pods = p.pods
 
-	returnVal = statusRequest(PodsList)
+	returnVal = statusRequest(PodsList, token)
 	json.Unmarshal(returnVal, &ret)
 
 	for podIndex, podStatus := range ret.PodStatus {

pkg/virtualkubelet/virtualkubelet.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"io/ioutil"
 	"math/rand"
+	"os"
 	"time"
 
 	"github.com/CARV-ICS-FORTH/knoc"
@@ -573,6 +574,12 @@ func (p *VirtualKubeletProvider) statusLoop(ctx context.Context) {
 		<-t.C
 	}
 
+	b, err := os.ReadFile(commonIL.InterLinkConfigInst.VKTokenFile) // just pass the file name
+	if err != nil {
+		fmt.Print(err)
+	}
+	token := string(b)
+
 	for {
 		t.Reset(5 * time.Second)
 		select {
@@ -581,7 +588,12 @@ func (p *VirtualKubeletProvider) statusLoop(ctx context.Context) {
 		case <-t.C:
 		}
 
-		checkPodsStatus(p, ctx)
+		b, err = os.ReadFile(commonIL.InterLinkConfigInst.VKTokenFile) // just pass the file name
+		if err != nil {
+			fmt.Print(err)
+		}
+		token = string(b)
+		checkPodsStatus(p, ctx, token)
 	}
 }
 

scripts/refresh.py

@@ -0,0 +1,56 @@
+#! /bin/env python3
+"""
+Functions and scripts to sync OIDC identities on user accounts
+"""
+
+import os
+import json
+import time
+import logging
+import requests
+
+if __name__ == '__main__':
+    """
+    sync OIDC identities on user accounts
+    """
+    try:
+        iam_server = os.environ.get(
+            "IAM_SERVER", "https://cms-auth.web.cern.ch/")
+        iam_client_id = os.environ.get("IAM_CLIENT_ID")
+        iam_client_secret = os.environ.get("IAM_CLIENT_SECRET")
+        audience = os.environ.get("IAM_VK_AUD")
+        output_file = os.environ.get("TOKEN_PATH", "/opt/interlink/token")
+    except Exception as ex:
+        print(ex)
+        exit(1)
+
+    token = None
+
+    while True:
+        try:
+            request_data = {
+                "client_id": iam_client_id,
+                "client_secret": iam_client_secret,
+                "grant_type": "client_credentials",
+                "username": "not_needed",
+                "password": "not_needed",
+                "scope": "openid profile email iam",
+                "aud": audience
+            }
+            r = requests.post(iam_server+"token", data=request_data)
+            response = json.loads(r.text)
+
+            #print(iam_client_id, iam_client_secret, response)
+            token = response['access_token']
+
+            logging.info("Token retrieved")
+
+            with open(output_file, "w") as text_file:
+                text_file.write(token)
+
+            logging.info(f"Token written in {output_file}")
+
+        except Exception as e:
+            logging.warn("ERROR oidc get token: {}".format(e))
+        
+        time.sleep(600)