diff --git a/dev_requirements.txt b/dev_requirements.txt index 0379643..0ebd0ec 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,2 +1,3 @@ harborapi +python-json-logger chevron diff --git a/requirements.txt b/requirements.txt index 4042836..78d6ac9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -harborapi==0.25.3 +harborapi==0.26.1 +python-json-logger==2.0.7 chevron==0.14.0 diff --git a/src/configuration.py b/src/configuration.py index b311981..0a54283 100644 --- a/src/configuration.py +++ b/src/configuration.py @@ -7,13 +7,13 @@ oidc_endpoint = os.environ.get("OIDC_ENDPOINT") -async def sync_harbor_configuration(client, path): +async def sync_harbor_configuration(client, path, logger): """Synchronize the harbor configuration The configurations file, if existent, will be applied to harbor. """ - print("SYNCING HARBOR CONFIGURATION") + logger.info("Syncing harbor configuration") harbor_config = json.load(open(path)) harbor_config = Configurations(**harbor_config) harbor_config.oidc_client_secret = oidc_client_secret diff --git a/src/garbage_collection_schedule.py b/src/garbage_collection_schedule.py index 1b6ecdc..d995cd1 100644 --- a/src/garbage_collection_schedule.py +++ b/src/garbage_collection_schedule.py @@ -1,19 +1,19 @@ import json -async def sync_garbage_collection_schedule(client, path): +async def sync_garbage_collection_schedule(client, path, logger): """Synchronize the garbage collection and its schedule The garbage collection and its schedule from the garbage collection schedule file, if existent, will be updated and applied to harbor. """ - print("SYNCING GARBAGE COLLECTION SCHEDULE") + logger.info("Syncing garbage collection schedule") garbage_collection_schedule = json.load(open(path)) try: await client.get_gc_schedule() - print("Updating garbage collection schedule") + logger.info("Updating garbage collection schedule") await client.update_gc_schedule(garbage_collection_schedule) except Exception as e: - print("Creating garbage collection schedule") + logger.info("Creating garbage collection schedule") await client.create_gc_schedule(garbage_collection_schedule) diff --git a/src/harbor.py b/src/harbor.py index 6d433d7..8660e91 100644 --- a/src/harbor.py +++ b/src/harbor.py @@ -10,12 +10,14 @@ """ -from harborapi import HarborAsyncClient import argparse import os import asyncio +import logging -from utils import wait_until_healthy, sync_admin_password, check_file_exists +from harborapi import HarborAsyncClient +from pythonjsonlogger import jsonlogger +from utils import wait_until_healthy, sync_admin_password, file_exists from configuration import sync_harbor_configuration from registries import sync_registries from projects import sync_projects @@ -31,6 +33,16 @@ new_admin_password = os.environ.get("ADMIN_PASSWORD_NEW") api_url = os.environ.get("HARBOR_API_URL") config_folder_path = os.environ.get("CONFIG_FOLDER_PATH") +json_logging = os.environ.get("JSON_LOGGING", "False") == "True" + + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +if json_logging: + handler = logging.StreamHandler() + formatter = jsonlogger.JsonFormatter() + handler.setFormatter(formatter) + logger.addHandler(handler) async def main() -> None: @@ -44,56 +56,52 @@ async def main() -> None: verify=False, ) - # Wait for healthy harbor - print("WAITING FOR HEALTHY HARBOR") - await wait_until_healthy(client) - print("") + logger.info("Waiting for healthy harbor") + await wait_until_healthy(client, logger) - # Update admin password - print("UPDATE ADMIN PASSWORD") - await sync_admin_password(client) - print("") + logger.info("Update admin password") + await sync_admin_password(client, logger) path = config_folder_path + "/configurations.json" - if check_file_exists(path): - await sync_harbor_configuration(client, path) + if file_exists(path, logger): + await sync_harbor_configuration(client, path, logger) path = config_folder_path + "/registries.json" - if check_file_exists(path): - await sync_registries(client, path) + if file_exists(path, logger): + await sync_registries(client, path, logger) path = config_folder_path + "/projects.json" - if check_file_exists(path): - await sync_projects(client, path) + if file_exists(path, logger): + await sync_projects(client, path, logger) path = config_folder_path + "/project-members.json" - if check_file_exists(path): - await sync_project_members(client, path) + if file_exists(path, logger): + await sync_project_members(client, path, logger) path = config_folder_path + "/robots.json" - if check_file_exists(path): - await sync_robot_accounts(client, path) + if file_exists(path, logger): + await sync_robot_accounts(client, path, logger) path = config_folder_path + "/webhooks.json" - if check_file_exists(path): - await sync_webhooks(client, path) + if file_exists(path, logger): + await sync_webhooks(client, path, logger) path = config_folder_path + "/purge-job-schedule.json" - if check_file_exists(path): - await sync_purge_job_schedule(client, path) + if file_exists(path, logger): + await sync_purge_job_schedule(client, path, logger) path = config_folder_path + "/garbage-collection-schedule.json" - if check_file_exists(path): - await sync_garbage_collection_schedule(client, path) + if file_exists(path, logger): + await sync_garbage_collection_schedule(client, path, logger) path = config_folder_path + "/retention-policies.json" - if check_file_exists(path): - await sync_retention_policies(client, path) + if file_exists(path, logger): + await sync_retention_policies(client, path, logger) def parse_args(): parser = argparse.ArgumentParser( - description="""Harbor Day 2 configurator to sync harbor configs""", + description="""Harbor Day 2 operator to sync harbor configs""", ) args = parser.parse_args() return args diff --git a/src/project_members.py b/src/project_members.py index 4d575d2..6b5f47b 100644 --- a/src/project_members.py +++ b/src/project_members.py @@ -11,18 +11,21 @@ class ProjectRole(Enum): MAINTAINER = 4 -async def sync_project_members(client, path): +async def sync_project_members(client, path, logger): """Synchronize all project members All project members and their roles from the project members file, if existent, will be updated and applied to harbor. """ - print("SYNCING PROJECT MEMBERS") + logger.info("Syncing project members") project_members_config = json.load(open(path)) for project in project_members_config: project_name = project["project_name"] - print(f'PROJECT: "{project_name}"') + logger.info( + "Syncing project members of project", + extra={"project": project_name} + ) current_members = await client.get_project_members( project_name_or_id=project_name, @@ -42,9 +45,12 @@ async def sync_project_members(client, path): if current_member.entity_name not in [ target_member.entity_name for target_member in target_members ]: - print( - f'- Removing "{current_member.entity_name}" from project' - f' "{project_name}"' + logger.info( + "Removing member from project", + extra={ + "member": current_member.entity_name, + "project": project_name + } ) await client.remove_project_member( project_name_or_id=project_name, @@ -56,7 +62,10 @@ async def sync_project_members(client, path): member_id = get_member_id(current_members, member.entity_name) # Sync existing members' project role if member_id: - print(f'- Syncing project role of "{member.entity_name}"') + logger.info( + "Syncing project role of member", + extra={"member": member.entity_name} + ) await client.update_project_member_role( project_name_or_id=project_name, member_id=member_id, @@ -64,9 +73,12 @@ async def sync_project_members(client, path): ) # Add new member else: - print( - f'- Adding new member "{member.entity_name}" to project' - f' "{project_name}"' + logger.info( + "Adding new member to project", + extra={ + "member": member.entity_name, + "project": project_name + } ) try: await client.add_project_member_user( @@ -75,7 +87,10 @@ async def sync_project_members(client, path): role_id=member.role_id, ) except NotFound: - print( - f' => ERROR: User "{member.entity_name}" not found.' - " Make sure the user has logged in at least once." + logger.info( + "User not found", + extra={ + "member": member.entity_name, + "hint": "Make sure user has logged in" + } ) diff --git a/src/projects.py b/src/projects.py index c471940..87fa79c 100644 --- a/src/projects.py +++ b/src/projects.py @@ -3,14 +3,14 @@ from utils import fill_template -async def sync_projects(client, path): +async def sync_projects(client, path, logger): """Synchronize all projects All projects from the project file, if existent, will be updated and applied to harbor. """ - print("SYNCING PROJECTS") + logger.info("Syncing projects") target_projects_string = await fill_template(client, path) target_projects = json.loads(target_projects_string) current_projects = await client.get_projects(limit=None) @@ -29,28 +29,34 @@ async def sync_projects(client, path): limit=None ) if len(repositories) == 0: - print( - f'- Deleting project "{current_project.name}" since it is' - " empty and not defined in config files" + logger.info( + "Deleting project", + extra={"project": current_project.name} ) await client.delete_project( project_name_or_id=current_project.name ) else: - print( - f'- Deletion of project "{current_project.name}" not' - " possible since it is not empty" + logger.info( + "Deletion of project not possible as not empty", + extra={"project": current_project.name} ) # Modify existing projects or create new ones for target_project in target_projects: # Modify existing project if target_project["project_name"] in current_project_names: - print(f'- Syncing project "{target_project["project_name"]}"') + logger.info( + "Syncing project", + extra={"project": target_project["project_name"]} + ) await client.update_project( project_name_or_id=current_project.name, project=target_project ) # Create new project else: - print(f'- Creating new project "{target_project["project_name"]}"') + logger.info( + "Creating new project", + extra={"project": target_project["project_name"]} + ) await client.create_project(project=target_project) diff --git a/src/purge_job_schedule.py b/src/purge_job_schedule.py index 0bd1ab4..96c5dcc 100644 --- a/src/purge_job_schedule.py +++ b/src/purge_job_schedule.py @@ -1,19 +1,19 @@ import json -async def sync_purge_job_schedule(client, path): +async def sync_purge_job_schedule(client, path, logger): """Synchronize the purge job and its schedule The purge job and its schedule from the purge job schedule file, if existent, will be updated and applied to harbor. """ - print("SYNCING PURGE JOB SCHEDULE") + logger.info("Syncing purge job schedule") purge_job_schedule = json.load(open(path)) try: await client.get_purge_job_schedule() - print("Updating purge job schedule") + logger.info("Updating purge job schedule") await client.update_purge_job_schedule(purge_job_schedule) except Exception as e: - print("Creating purge job schedule") + logger.info("Creating purge job schedule") await client.create_purge_job_schedule(purge_job_schedule) diff --git a/src/registries.py b/src/registries.py index 0f8abce..b8a8058 100644 --- a/src/registries.py +++ b/src/registries.py @@ -1,14 +1,14 @@ import json -async def sync_registries(client, path): +async def sync_registries(client, path, logger): """Synchronize all registries All registries from the registries file, if existent, will be updated and applied to harbor. """ - print("SYNCING REGISTRIES") + logger.info("Syncing registries") target_registries = json.load(open(path)) current_registries = await client.get_registries(limit=None) current_registry_names = [ @@ -24,24 +24,28 @@ async def sync_registries(client, path): # Delete all registries not defined in config file for current_registry in current_registries: if current_registry.name not in target_registry_names: - print( - f'- Deleting registry "{current_registry.name}" since it is' - " not defined in config files" + logger.info( + "Deleting registry as it is not defined in config", + extra={"registry": current_registry.name} ) await client.delete_registry(id=current_registry.id) # Modify existing registries or create new ones for target_registry in target_registries: # Modify existing registry - if target_registry["name"] in current_registry_names: + target_registry_name = target_registry["name"] + if target_registry_name in current_registry_names: registry_id = current_registry_id[ - current_registry_names.index(target_registry["name"]) + current_registry_names.index(target_registry_name) ] - print(f'- Syncing registry "{target_registry["name"]}"') + logger.info( + "Syncing registy", + extra={"registry": target_registry_name} + ) await client.update_registry( id=registry_id, registry=target_registry ) # Create new registry else: - print(f'- Creating new registry "{target_registry["name"]}"') + logger.info("Creating new registry", extra={target_registry_name}) await client.create_registry(registry=target_registry) diff --git a/src/retention_policies.py b/src/retention_policies.py index 0224840..62c155c 100644 --- a/src/retention_policies.py +++ b/src/retention_policies.py @@ -3,14 +3,14 @@ from utils import fill_template -async def sync_retention_policies(client, path): +async def sync_retention_policies(client, path, logger): """Synchronize the retention policies The retention policies from the retention policies file, if existent, will be updated and applied to harbor. """ - print('SYNCING RETENTION POLICIES') + logger.info("Syncing retention policies") retention_policies_string = await fill_template(client, path) retention_policies = json.loads(retention_policies_string) for retention_policy in retention_policies: @@ -23,11 +23,15 @@ async def sync_retention_policies(client, path): project_retention_id, retention_policy ) - print(f"Updating retention policy for project with id " - f"{retention_scope}") + logger.info( + "Updating retention policy", + extra={"project_id": retention_scope} + ) except Exception as e: - print(f"Creating retention policy for project with id " - f"{retention_scope}") + logger.info( + "Creating retention policy", + extra={"project_id": retention_scope} + ) await client.create_retention_policy( retention_policy ) diff --git a/src/robot_accounts.py b/src/robot_accounts.py index 90505d7..20218f8 100644 --- a/src/robot_accounts.py +++ b/src/robot_accounts.py @@ -7,14 +7,14 @@ robot_name_prefix = os.environ.get("ROBOT_NAME_PREFIX") -async def sync_robot_accounts(client, path): +async def sync_robot_accounts(client, path, logger): """Synchronize all robot accounts All robot accounts from the robot accounts file, if existent, will be updated and applied to harbor. """ - print("SYNCING ROBOT ACCOUNTS") + logger.info("Syncing robot accounts") target_robots = json.load(open(path)) # Get all system level robots @@ -55,16 +55,15 @@ async def sync_robot_accounts(client, path): # Delete all robots not defined in config file for current_robot in current_robots: if current_robot.name not in target_robot_names_with_prefix: - print( - f'- Deleting robot "{current_robot.name}" since it is not' - " defined in config files" + logger.info( + "Deleting robot as not defined", + extra={"robot": current_robot.name} ) await client.delete_robot(robot_id=current_robot.id) # Modify existing robots or create new ones for target_robot in target_robots: full_robot_name = await construct_full_robot_name(target_robot) - print(f'Full robot name: {full_robot_name}') target_robot = Robot(**target_robot) # Modify existing robot if full_robot_name in current_robot_names: @@ -73,27 +72,22 @@ async def sync_robot_accounts(client, path): ] short_robot_name = target_robot.name target_robot.name = full_robot_name - print(f'- Syncing robot "{target_robot.name}".') + logger.info("Syncing robot", extra={"robot": target_robot.name}) await client.update_robot(robot_id=robot_id, robot=target_robot) - await set_robot_secret(client, short_robot_name, robot_id) + await set_robot_secret(client, short_robot_name, robot_id, logger) # Create new robot else: - print( - "- Creating new robot" - f' "{full_robot_name}"' + logger.info( + "Creating new robot", + extra={"robot": target_robot.name} ) try: created_robot = await client.create_robot(robot=target_robot) await set_robot_secret( - client, target_robot.name, created_robot.id + client, target_robot.name, created_robot.id, logger ) - except Conflict as e: - print( - f''' => "{full_robot_name}" - Harbor Conflict Error: {e}''' - ) - except BadRequest as e: - print(f'Bad request permission: {e}') + except (Conflict, BadRequest) as e: + logger.info("Could not create robot", extra={"details": e}) async def construct_full_robot_name(target_robot: Robot) -> str: @@ -103,12 +97,12 @@ async def construct_full_robot_name(target_robot: Robot) -> str: return f'{robot_name_prefix}{target_robot["name"]}' -async def set_robot_secret(client, robot_name: str, robot_id: int): +async def set_robot_secret(client, robot_name: str, robot_id: int, logger): secret = os.environ.get( robot_name.upper().replace("-", "_") ) if secret: - print(f'Set secret for {robot_name}') + logger.info("Set robot secret", extra={"robot": robot_name}) await client.refresh_robot_secret(robot_id, secret) else: - print(f'WARN: No secret found for {robot_name}') + logger.info("No robot secret found", extra={"robot": robot_name}) diff --git a/src/utils.py b/src/utils.py index b544e6b..306580a 100644 --- a/src/utils.py +++ b/src/utils.py @@ -15,27 +15,27 @@ api_url = os.environ.get("HARBOR_API_URL") -def check_file_exists(path: str) -> bool: +def file_exists(path: str, logger) -> bool: if os.path.exists(path): return True else: - print(f"File {path} not found - skipping step") + logger.info("File not found - skipping step", extra={"path": path}) return False -async def wait_until_healthy(client) -> None: +async def wait_until_healthy(client, logger) -> None: while True: health = await client.health_check() if health.status == "healthy": - print("- Harbor is healthy") + logger.info("Harbor is healthy") break - print("- Waiting for harbor to become healthy...") + logger.info("Waiting for harbor to become healthy") sleep(5) -async def update_password(client) -> None: +async def update_password(client, logger) -> None: try: - print("Updating password") + logger.info("Updating password") old_password_client = HarborAsyncClient( url=api_url, username=admin_username, @@ -49,19 +49,16 @@ async def update_password(client) -> None: old_password=old_admin_password, new_password=new_admin_password, ) - print("- Updated admin password") + logger.info("Updated admin password") except Unauthorized: - print( - " => ERROR: Unable to change the admin password." - " Neither the old nor the new password are correct." - ) + logger.error("Unable to change the admin password") -async def sync_admin_password(client) -> None: +async def sync_admin_password(client, logger) -> None: try: await client.get_current_user() except Unauthorized: - await update_password(client) + await update_password(client, logger) def get_member_id(members: [ProjectMemberEntity], username: str) -> int | None: diff --git a/src/webhooks.py b/src/webhooks.py index cc2acb4..f9d93b6 100644 --- a/src/webhooks.py +++ b/src/webhooks.py @@ -2,23 +2,29 @@ import json -async def sync_webhooks(client, path): +async def sync_webhooks(client, path, logger): """Synchronize all webhooks All webhooks from the webhooks file, if existent, will be updated and applied to harbor. """ - print("SYNCING WEBHOOKS") + logger.info("Syncing webhooks") webhooks_config = json.load(open(path)) for webhook in webhooks_config: - await sync_webhook(client, **webhook) + await sync_webhook(client, logger, **webhook) async def sync_webhook( - client, project_name: str, policies: list[WebhookPolicy] + client, + logger, + project_name: str, + policies: list[WebhookPolicy] ): - print(f'PROJECT: "{project_name}"') + logger.info( + "Syncing webhooks for project", + extra={"project": project_name} + ) target_policies = policies current_policies = await client.get_webhook_policies( @@ -38,9 +44,9 @@ async def sync_webhook( # Delete all policies not defined in config file for current_policy in current_policies: if current_policy.name not in target_policy_names: - print( - f'- Deleting policy "{current_policy.name}" since it is not' - " defined in config files" + logger.info( + "Deleting policy", + extra={"policy": current_policy.name} ) await client.delete_webhook_policy( project_name_or_id=project_name, @@ -54,7 +60,10 @@ async def sync_webhook( policy_id = current_policy_id[ current_policy_names.index(target_policy["name"]) ] - print(f'- Syncing policy "{target_policy["name"]}"') + logger.info( + "Syncing policy", + extra={"policy": target_policy["name"]} + ) await client.update_webhook_policy( project_name_or_id=project_name, webhook_policy_id=policy_id, @@ -62,7 +71,10 @@ async def sync_webhook( ) # Create new policy else: - print(f'- Creating new policy "{target_policy["name"]}"') + logger.info( + "Creating new policy", + extra={"policy": target_policy["name"]} + ) await client.create_webhook_policy( project_name_or_id=project_name, policy=target_policy )