Feature/is alive check and api error handling (#10)

* UPDATE: is alive check to change when unable to connect to k8s

* UPDATE: to handle api exception

collector/nais_stream.py

@@ -1,6 +1,9 @@
 import re
 
 from typing import Callable
+
+from kubernetes.client import ApiException
+
 from collector.nais import init_nais_logging
 
 logger = init_nais_logging()
@@ -18,24 +21,32 @@ def __str__(self):
     def resource_version(self):
         return self._resource_version
 
-class NaisStream:
 
+class NaisStream:
     def __init__(self, callback_function: Callable, v1, w):
         self.callback_function = callback_function
         self.v1 = v1
         self.w = w
 
     def watch(self, **kwargs):
-        for event in self.w.stream(self.v1.list_cluster_custom_object,
-                                   group="nais.io",
-                                   version="v1alpha1",
-                                   plural="applications",
-                                   **kwargs):
-            if event["type"] not in ["ERROR", "DELETED"]:
-                self.callback_function(event)
-            elif event["type"] == "ERROR" and event["object"]["code"] == 410:
-                logger.warning("")
-                logger.warning(event)
-                logger.warning("")
-                resource_version = event["object"]["message"].split('(')
+        try:
+            for event in self.w.stream(self.v1.list_cluster_custom_object,
+                                       group="nais.io",
+                                       version="v1alpha1",
+                                       plural="applications",
+                                       **kwargs):
+                if event["type"] not in ["ERROR", "DELETED"]:
+                    self.callback_function(event)
+                elif event["type"] == "ERROR" and event["object"]["code"] == 410:
+                    logger.warning("")
+                    logger.warning(event)
+                    logger.warning("")
+                    resource_version = event["object"]["message"].split('(')
+                    raise TooOldResourceVersionError(resource_version[1][:-1])
+        except ApiException as api_exc:
+            logger.warning(f"\n\nAPI exception: {api_exc.status}: Reason: {api_exc.reason}\n\n")
+            if api_exc.status == 410:
+                resource_version = api_exc.reason.split('(')
                 raise TooOldResourceVersionError(resource_version[1][:-1])
+            else:
+                raise api_exc

main.py

@@ -5,13 +5,16 @@
 import backoff
 # noinspection PyPackageRequirements
 from fastapi import FastAPI
+from starlette.responses import JSONResponse
+from starlette import status
 from collector.nais import init_nais_logging
 from collector.kube_api import watch_nais_apps
 from kubernetes import client, config
 
 # initiating logging
 logger = init_nais_logging()
 app = FastAPI()
+is_alive = True
 
 
 @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=10)
@@ -43,7 +46,11 @@ def init_kube_client():
 
 
 def watch_nais_task() -> None:
-    watch_nais_apps(watch_nais_callback)
+    try:
+        watch_nais_apps(watch_nais_callback)
+    except:
+        global is_alive
+        is_alive = False
 
 
 @app.on_event('startup')
@@ -71,7 +78,11 @@ def root():
 
 @app.get('/is-alive')
 def is_healthy():
-    return 'OK'
+    if is_alive:
+        return 'OK'
+    else:
+        return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                            content={"Error": "K8s stream stopped"})
 
 
 @app.get('/is-ready')