Add optional json logging (#64)

* introduce pythonjsonlogger * correct library name and make json conditional * update prints to use logger * correct library name in code * change import of pythonjsonlogger * cast json logging env var as cannot be unmarshaled to boolean * add logger instead of prints * replace remaining prints with logger and fix linter issues * add missing positional argument * move positional argument * change log level and rename function * robot warning as info with details * bump harborapi version
steadforce · Nov 27, 2024 · 8558afb · 8558afb
1 parent 06c71d5
commit 8558afb
Show file tree

Hide file tree

Showing 13 changed files with 166 additions and 124 deletions.
diff --git a/dev_requirements.txt b/dev_requirements.txt
@@ -1,2 +1,3 @@
 harborapi
+python-json-logger
 chevron
diff --git a/requirements.txt b/requirements.txt
@@ -1,2 +1,3 @@
-harborapi==0.25.3
+harborapi==0.26.1
+python-json-logger==2.0.7
 chevron==0.14.0
diff --git a/src/configuration.py b/src/configuration.py
@@ -7,13 +7,13 @@
 oidc_endpoint = os.environ.get("OIDC_ENDPOINT")
 
 
-async def sync_harbor_configuration(client, path):
+async def sync_harbor_configuration(client, path, logger):
     """Synchronize the harbor configuration
 
     The configurations file, if existent, will be applied to harbor.
     """
 
-    print("SYNCING HARBOR CONFIGURATION")
+    logger.info("Syncing harbor configuration")
     harbor_config = json.load(open(path))
     harbor_config = Configurations(**harbor_config)
     harbor_config.oidc_client_secret = oidc_client_secret

diff --git a/src/garbage_collection_schedule.py b/src/garbage_collection_schedule.py
@@ -1,19 +1,19 @@
 import json
 
 
-async def sync_garbage_collection_schedule(client, path):
+async def sync_garbage_collection_schedule(client, path, logger):
     """Synchronize the garbage collection and its schedule
 
     The garbage collection and its schedule from the garbage collection
     schedule file, if existent, will be updated and applied to harbor.
     """
 
-    print("SYNCING GARBAGE COLLECTION SCHEDULE")
+    logger.info("Syncing garbage collection schedule")
     garbage_collection_schedule = json.load(open(path))
     try:
         await client.get_gc_schedule()
-        print("Updating garbage collection schedule")
+        logger.info("Updating garbage collection schedule")
         await client.update_gc_schedule(garbage_collection_schedule)
     except Exception as e:
-        print("Creating garbage collection schedule")
+        logger.info("Creating garbage collection schedule")
         await client.create_gc_schedule(garbage_collection_schedule)
diff --git a/src/harbor.py b/src/harbor.py
@@ -10,12 +10,14 @@
 """
 
 
-from harborapi import HarborAsyncClient
 import argparse
 import os
 import asyncio
+import logging
 
-from utils import wait_until_healthy, sync_admin_password, check_file_exists
+from harborapi import HarborAsyncClient
+from pythonjsonlogger import jsonlogger
+from utils import wait_until_healthy, sync_admin_password, file_exists
 from configuration import sync_harbor_configuration
 from registries import sync_registries
 from projects import sync_projects
@@ -31,6 +33,16 @@
 new_admin_password = os.environ.get("ADMIN_PASSWORD_NEW")
 api_url = os.environ.get("HARBOR_API_URL")
 config_folder_path = os.environ.get("CONFIG_FOLDER_PATH")
+json_logging = os.environ.get("JSON_LOGGING", "False") == "True"
+
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+if json_logging:
+    handler = logging.StreamHandler()
+    formatter = jsonlogger.JsonFormatter()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
 
 
 async def main() -> None:
@@ -44,56 +56,52 @@ async def main() -> None:
         verify=False,
     )
 
-    # Wait for healthy harbor
-    print("WAITING FOR HEALTHY HARBOR")
-    await wait_until_healthy(client)
-    print("")
+    logger.info("Waiting for healthy harbor")
+    await wait_until_healthy(client, logger)
 
-    # Update admin password
-    print("UPDATE ADMIN PASSWORD")
-    await sync_admin_password(client)
-    print("")
+    logger.info("Update admin password")
+    await sync_admin_password(client, logger)
 
     path = config_folder_path + "/configurations.json"
-    if check_file_exists(path):
-        await sync_harbor_configuration(client, path)
+    if file_exists(path, logger):
+        await sync_harbor_configuration(client, path, logger)
 
     path = config_folder_path + "/registries.json"
-    if check_file_exists(path):
-        await sync_registries(client, path)
+    if file_exists(path, logger):
+        await sync_registries(client, path, logger)
 
     path = config_folder_path + "/projects.json"
-    if check_file_exists(path):
-        await sync_projects(client, path)
+    if file_exists(path, logger):
+        await sync_projects(client, path, logger)
 
     path = config_folder_path + "/project-members.json"
-    if check_file_exists(path):
-        await sync_project_members(client, path)
+    if file_exists(path, logger):
+        await sync_project_members(client, path, logger)
 
     path = config_folder_path + "/robots.json"
-    if check_file_exists(path):
-        await sync_robot_accounts(client, path)
+    if file_exists(path, logger):
+        await sync_robot_accounts(client, path, logger)
 
     path = config_folder_path + "/webhooks.json"
-    if check_file_exists(path):
-        await sync_webhooks(client, path)
+    if file_exists(path, logger):
+        await sync_webhooks(client, path, logger)
 
     path = config_folder_path + "/purge-job-schedule.json"
-    if check_file_exists(path):
-        await sync_purge_job_schedule(client, path)
+    if file_exists(path, logger):
+        await sync_purge_job_schedule(client, path, logger)
 
     path = config_folder_path + "/garbage-collection-schedule.json"
-    if check_file_exists(path):
-        await sync_garbage_collection_schedule(client, path)
+    if file_exists(path, logger):
+        await sync_garbage_collection_schedule(client, path, logger)
 
     path = config_folder_path + "/retention-policies.json"
-    if check_file_exists(path):
-        await sync_retention_policies(client, path)
+    if file_exists(path, logger):
+        await sync_retention_policies(client, path, logger)
 
 
 def parse_args():
     parser = argparse.ArgumentParser(
-        description="""Harbor Day 2 configurator to sync harbor configs""",
+        description="""Harbor Day 2 operator to sync harbor configs""",
     )
     args = parser.parse_args()
     return args

diff --git a/src/project_members.py b/src/project_members.py
@@ -11,18 +11,21 @@ class ProjectRole(Enum):
     MAINTAINER = 4
 
 
-async def sync_project_members(client, path):
+async def sync_project_members(client, path, logger):
     """Synchronize all project members
 
     All project members and their roles from the project members file,
     if existent, will be updated and applied to harbor.
     """
 
-    print("SYNCING PROJECT MEMBERS")
+    logger.info("Syncing project members")
     project_members_config = json.load(open(path))
     for project in project_members_config:
         project_name = project["project_name"]
-        print(f'PROJECT: "{project_name}"')
+        logger.info(
+            "Syncing project members of project",
+            extra={"project": project_name}
+        )
 
         current_members = await client.get_project_members(
             project_name_or_id=project_name,
@@ -42,9 +45,12 @@ async def sync_project_members(client, path):
             if current_member.entity_name not in [
                 target_member.entity_name for target_member in target_members
             ]:
-                print(
-                    f'- Removing "{current_member.entity_name}" from project'
-                    f' "{project_name}"'
+                logger.info(
+                    "Removing member from project",
+                    extra={
+                        "member": current_member.entity_name,
+                        "project": project_name
+                    }
                 )
                 await client.remove_project_member(
                     project_name_or_id=project_name,
@@ -56,17 +62,23 @@ async def sync_project_members(client, path):
             member_id = get_member_id(current_members, member.entity_name)
             # Sync existing members' project role
             if member_id:
-                print(f'- Syncing project role of "{member.entity_name}"')
+                logger.info(
+                    "Syncing project role of member",
+                    extra={"member": member.entity_name}
+                )
                 await client.update_project_member_role(
                     project_name_or_id=project_name,
                     member_id=member_id,
                     role=member.role_id,
                 )
             # Add new member
             else:
-                print(
-                    f'- Adding new member "{member.entity_name}" to project'
-                    f' "{project_name}"'
+                logger.info(
+                    "Adding new member to project",
+                    extra={
+                        "member": member.entity_name,
+                        "project": project_name
+                    }
                 )
                 try:
                     await client.add_project_member_user(
@@ -75,7 +87,10 @@ async def sync_project_members(client, path):
                         role_id=member.role_id,
                     )
                 except NotFound:
-                    print(
-                        f'  => ERROR: User "{member.entity_name}" not found.'
-                        " Make sure the user has logged in at least once."
+                    logger.info(
+                        "User not found",
+                        extra={
+                            "member": member.entity_name,
+                            "hint": "Make sure user has logged in"
+                        }
                     )
diff --git a/src/projects.py b/src/projects.py
@@ -3,14 +3,14 @@
 from utils import fill_template
 
 
-async def sync_projects(client, path):
+async def sync_projects(client, path, logger):
     """Synchronize all projects
 
     All projects from the project file, if existent,
     will be updated and applied to harbor.
     """
 
-    print("SYNCING PROJECTS")
+    logger.info("Syncing projects")
     target_projects_string = await fill_template(client, path)
     target_projects = json.loads(target_projects_string)
     current_projects = await client.get_projects(limit=None)
@@ -29,28 +29,34 @@ async def sync_projects(client, path):
                 limit=None
             )
             if len(repositories) == 0:
-                print(
-                    f'- Deleting project "{current_project.name}" since it is'
-                    " empty and not defined in config files"
+                logger.info(
+                    "Deleting project",
+                    extra={"project": current_project.name}
                 )
                 await client.delete_project(
                     project_name_or_id=current_project.name
                 )
             else:
-                print(
-                    f'- Deletion of project "{current_project.name}" not'
-                    " possible since it is not empty"
+                logger.info(
+                    "Deletion of project not possible as not empty",
+                    extra={"project": current_project.name}
                 )
 
     # Modify existing projects or create new ones
     for target_project in target_projects:
         # Modify existing project
         if target_project["project_name"] in current_project_names:
-            print(f'- Syncing project "{target_project["project_name"]}"')
+            logger.info(
+                "Syncing project",
+                extra={"project": target_project["project_name"]}
+            )
             await client.update_project(
                 project_name_or_id=current_project.name, project=target_project
             )
         # Create new project
         else:
-            print(f'- Creating new project "{target_project["project_name"]}"')
+            logger.info(
+                "Creating new project",
+                extra={"project": target_project["project_name"]}
+            )
             await client.create_project(project=target_project)
diff --git a/src/purge_job_schedule.py b/src/purge_job_schedule.py
@@ -1,19 +1,19 @@
 import json
 
 
-async def sync_purge_job_schedule(client, path):
+async def sync_purge_job_schedule(client, path, logger):
     """Synchronize the purge job and its schedule
 
     The purge job and its schedule from the purge job schedule file,
     if existent, will be updated and applied to harbor.
     """
 
-    print("SYNCING PURGE JOB SCHEDULE")
+    logger.info("Syncing purge job schedule")
     purge_job_schedule = json.load(open(path))
     try:
         await client.get_purge_job_schedule()
-        print("Updating purge job schedule")
+        logger.info("Updating purge job schedule")
         await client.update_purge_job_schedule(purge_job_schedule)
     except Exception as e:
-        print("Creating purge job schedule")
+        logger.info("Creating purge job schedule")
         await client.create_purge_job_schedule(purge_job_schedule)
diff --git a/src/registries.py b/src/registries.py
@@ -1,14 +1,14 @@
 import json
 
 
-async def sync_registries(client, path):
+async def sync_registries(client, path, logger):
     """Synchronize all registries
 
     All registries from the registries file, if existent,
     will be updated and applied to harbor.
     """
 
-    print("SYNCING REGISTRIES")
+    logger.info("Syncing registries")
     target_registries = json.load(open(path))
     current_registries = await client.get_registries(limit=None)
     current_registry_names = [
@@ -24,24 +24,28 @@ async def sync_registries(client, path):
     # Delete all registries not defined in config file
     for current_registry in current_registries:
         if current_registry.name not in target_registry_names:
-            print(
-                f'- Deleting registry "{current_registry.name}" since it is'
-                " not defined in config files"
+            logger.info(
+                "Deleting registry as it is not defined in config",
+                extra={"registry": current_registry.name}
             )
             await client.delete_registry(id=current_registry.id)
 
     # Modify existing registries or create new ones
     for target_registry in target_registries:
         # Modify existing registry
-        if target_registry["name"] in current_registry_names:
+        target_registry_name = target_registry["name"]
+        if target_registry_name in current_registry_names:
             registry_id = current_registry_id[
-                current_registry_names.index(target_registry["name"])
+                current_registry_names.index(target_registry_name)
             ]
-            print(f'- Syncing registry "{target_registry["name"]}"')
+            logger.info(
+                "Syncing registy",
+                extra={"registry": target_registry_name}
+            )
             await client.update_registry(
                 id=registry_id, registry=target_registry
             )
         # Create new registry
         else:
-            print(f'- Creating new registry "{target_registry["name"]}"')
+            logger.info("Creating new registry", extra={target_registry_name})
             await client.create_registry(registry=target_registry)