diff --git a/.github/workflows/build-and-publish-container-image.yaml b/.github/workflows/build-and-publish-container-image.yaml new file mode 100644 index 0000000..a9488a9 --- /dev/null +++ b/.github/workflows/build-and-publish-container-image.yaml @@ -0,0 +1,71 @@ +name: Create and publish k8s workbench container image + +on: + push: + branches: + - "**" + tags: + - "v*.*.*" + pull_request: + branches: + - "main" + + +env: + REGISTRY: ghcr.io + IMAGE_NAME: harbor-day2-operator + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: Dockerfile + - uses: ricardochaves/python-lint@v1.4.0 + with: + use-pylint: false + use-flake8: false + use-black: false + use-mypy: false + use-isort: false + - name: Log in to the container registry + uses: docker/login-action@a9794064588be971151ec5e7144cb535bcb56e36 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for container + id: meta + uses: docker/metadata-action@35e9aff4f5d665b5aa8a8f2adffaf8a1b5f49cc0 + with: + images: ${{ env.REGISTRY }}/steadforce/steadops/workbenches/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha + - name: Build container image for tests + uses: docker/build-push-action@4fad532b9fdbfb80f436784834374a1c11834153 + with: + context: . + push: false + tags: ${{ env.IMAGE_NAME }}:test + - name: Test harbor tool + run: | + docker run --rm ${{ env.IMAGE_NAME }}:test /usr/local/harbor --help + - name: Tag and push tested container image + uses: docker/build-push-action@4fad532b9fdbfb80f436784834374a1c11834153 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 68bc17f..df3cb20 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +/.project +/.pydevproject diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bbff36e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Stick to Python 3.11 until Nuitka supports Python 3.12 +FROM python:3.11-alpine@sha256:d1975f2182c9962f5daa1ad935eb092e3e32dce11d8105cb3584a31afc7b451b as base +ENV PYTHONUNBUFFERED 1 + +FROM base as builder +# we want always the latest version of fetched apk packages +# hadolint ignore=DL3018 +RUN apk add --no-cache build-base libressl-dev musl-dev libffi-dev && \ + mkdir /install +WORKDIR /install +COPY requirements.txt requirements.txt +# we want always the latest version of fetched pip packages +# hadolint ignore=DL3013 +RUN pip3 install --no-cache-dir -U pip setuptools wheel && \ + pip3 install --no-cache-dir --prefix=/install --no-warn-script-location -r ./requirements.txt + +FROM builder as native-builder +# we want always the latest version of fetched apk packages +# hadolint ignore=DL3018 +RUN apk add --no-cache ccache patchelf +COPY src/ /src/ +RUN python -m venv /venv && \ + /venv/bin/pip install --no-cache-dir -U pip nuitka setuptools wheel && \ + /venv/bin/pip install --no-cache-dir --no-warn-script-location -r ./requirements.txt && \ + /venv/bin/python -m nuitka --onefile /src/harbor.py && \ + pwd && \ + ls -lha + +FROM base as test +COPY --from=builder /install /usr/local +COPY tests/ /tests/ +WORKDIR /tests +RUN python3 -m unittest discover -v -s . + +FROM alpine:3.19@sha256:6457d53fb065d6f250e1504b9bc42d5b6c65941d57532c072d929dd0628977d0 +COPY --from=native-builder /install/harbor.bin /usr/local/harbor diff --git a/Dockerfile.requirements b/Dockerfile.requirements new file mode 100644 index 0000000..fb317ec --- /dev/null +++ b/Dockerfile.requirements @@ -0,0 +1,6 @@ +FROM local-python-base +COPY dev_requirements.txt ./dev_requirements.txt +RUN apk add --no-cache build-base librdkafka-dev +RUN python3 -m pip install -U pip setuptools wheel && \ + python3 -m pip install pip-chill && python3 -m pip install -r ./dev_requirements.txt +RUN pip-chill --no-chill > requirements.txt \ No newline at end of file diff --git a/README.md b/README.md index 85cc156..54c8a46 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,20 @@ # harbor-day2-operator -The harbor day2 operator is for automated managment of existing harbor instances using python harbor-api +The harbor day2 operator is for automated management of existing harbor instances using python harbor-api + +## Linter +We have activated linter like hadolint for dockerfiles. Please run +all the linters like documented underneath before checkin of source +code. Pull requests are only accepted when no linting errors occur. + +### hadolint + +``` + docker run --rm -i ghcr.io/hadolint/hadolint < Dockerfile +``` + +### python-lint + +``` + docker run --rm -v .:/src ricardobchaves6/python-lint-image:1.4.0 pycodestyle /src +``` + diff --git a/create_requirements_in_container.sh b/create_requirements_in_container.sh new file mode 100755 index 0000000..52e19a1 --- /dev/null +++ b/create_requirements_in_container.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +docker build . -f Dockerfile --target base -t local-python-base --no-cache +docker build . -f Dockerfile.requirements -t local-python-requirements --no-cache +id=$(docker create local-python-requirements) +docker cp $id:requirements.txt gen_requirements.txt +docker rm $id +docker image rm local-python-requirements local-python-base +cat < gen_requirements.txt > requirements.txt +rm gen_requirements.txt \ No newline at end of file diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 0000000..61ec4dd --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1 @@ +harborapi diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a36775a --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +harborapi==0.23.1 diff --git a/src/harbor.py b/src/harbor.py new file mode 100644 index 0000000..4c9285e --- /dev/null +++ b/src/harbor.py @@ -0,0 +1,441 @@ +import asyncio +from enum import Enum + +# API calls to configure harbor: +# See Harbor api: https://harbor.dev.k8s01.steadforce.com/devcenter-api-2.0 +from harborapi import HarborAsyncClient +from harborapi.exceptions import NotFound, Unauthorized, Conflict, BadRequest +from harborapi.models import ( + Robot, + Configurations, + Registry, + WebhookPolicy, + Project, + ProjectMemberEntity, +) +import argparse +import json +import os +from time import sleep + + +class ProjectRole(Enum): + ADMIN = 1 + DEVELOPER = 2 + GUEST = 3 + MAINTAINER = 4 + + +admin_username = "admin" +old_admin_password = os.environ.get("ADMIN_PASSWORD_OLD") +new_admin_password = os.environ.get("ADMIN_PASSWORD_NEW") +api_url = os.environ.get("HARBOR_API_URL") +config_folder_path = os.environ.get("CONFIG_FOLDER_PATH") +robot_name_prefix = os.environ.get("ROBOT_NAME_PREFIX") +oidc_client_secret = os.environ.get("OIDC_STATIC_CLIENT_TOKEN") +oidc_endpoint = os.environ.get("OIDC_ENDPOINT") + + +async def main() -> None: + parse_args() + + global client + client = HarborAsyncClient( + url=api_url, + username=admin_username, + secret=new_admin_password, + timeout=100, + verify=False, + ) + + # Wait for healthy harbor + print("WAITING FOR HEALTHY HARBOR") + await wait_until_healthy() + print("") + + # Update admin password + print("UPDATE ADMIN PASSWROD") + await sync_admin_password() + print("") + + # Sync harbor configuration + print("SYNCING HARBOR CONFIGURATION") + harbor_config = json.load( + open(config_folder_path + "/configurations.json") + ) + harbor_config = Configurations(**harbor_config) + harbor_config.oidc_client_secret = oidc_client_secret + harbor_config.oidc_endpoint = oidc_endpoint + await sync_harbor_config(harbor_config=harbor_config) + print("") + + # Sync registries + print("SYNCING REGISTRIES") + registries_config = json.load( + open(config_folder_path + "/registries.json") + ) + await sync_registries(target_registries=registries_config) + print("") + + # Sync projects + print("SYNCING PROJECTS") + projects_config = json.load(open(config_folder_path + "/projects.json")) + await sync_projects(target_projects=projects_config) + print("") + + # Sync project members and their roles + print("SYNCING PROJECT MEMBERS") + project_members_config = json.load( + open(config_folder_path + "/project-members.json") + ) + for project in project_members_config: + await sync_project_members(project=project) + print("") + + # Sync robot accounts + print("SYNCING ROBOT ACCOUNTS") + robot_config = json.load(open(config_folder_path + "/robots.json")) + await sync_robot_accounts(target_robots=robot_config) + print("") + + # Sync webhooks + print("SYNCING WEBHOOKS") + webhooks_config = json.load(open(config_folder_path + "/webhooks.json")) + for webhook in webhooks_config: + await sync_webhook(**webhook) + print("") + + +async def sync_harbor_config(harbor_config: Configurations): + await client.update_config(harbor_config) + + +async def sync_registries(target_registries: [Registry]): + current_registries = await client.get_registries(limit=None) + current_registry_names = [ + current_registry.name for current_registry in current_registries + ] + current_registry_id = [ + current_registry.id for current_registry in current_registries + ] + target_registry_names = [ + target_registry["name"] for target_registry in target_registries + ] + + # Delete all registries not defined in config file + for current_registry in current_registries: + if current_registry.name not in target_registry_names: + print( + f'- Deleting registry "{current_registry.name}" since it is' + " not defined in config files" + ) + await client.delete_registry(id=current_registry.id) + + # Modify existing registries or create new ones + for target_registry in target_registries: + # Modify existing registry + if target_registry["name"] in current_registry_names: + registry_id = current_registry_id[ + current_registry_names.index(target_registry["name"]) + ] + print(f'- Syncing registry "{target_registry["name"]}"') + await client.update_registry( + id=registry_id, registry=target_registry + ) + # Create new registry + else: + print(f'- Creating new registry "{target_registry["name"]}"') + await client.create_registry(registry=target_registry) + + +async def construct_full_robot_name(target_robot: Robot) -> str: + if (namespace := target_robot['permissions'][0]['namespace']) != '*': + return f'{robot_name_prefix}{namespace}+{target_robot["name"]}' + else: + return f'{robot_name_prefix}{target_robot["name"]}' + + +async def sync_robot_accounts(target_robots: [Robot]): + # Get all system level robots + current_system_robots = await client.get_robots( + query='Level=system', + limit=None, + ) + + # Get all project level robots + current_projects = await client.get_projects(limit=None) + current_project_ids = [ + current_project.project_id + for current_project in current_projects + ] + current_projects_robots = [] + for current_project_id in current_project_ids: + project_robots = await client.get_robots( + query=f'Level=project,ProjectID={current_project_id}', + limit=None + ) + current_projects_robots += project_robots + + # Combine system and project robots to get a list of all robots + current_robots = current_system_robots + current_projects_robots + current_robot_names = [ + current_robot.name for current_robot in current_robots + ] + current_robot_id = [current_robot.id for current_robot in current_robots] + + # Harbor appends a prefix and namespace if present to all + # robot account names. + # To compare against our target robot names, we have to add the prefix + target_robot_names_with_prefix = [ + await construct_full_robot_name(target_robot) + for target_robot in target_robots + ] + + # Delete all robots not defined in config file + for current_robot in current_robots: + if current_robot.name not in target_robot_names_with_prefix: + print( + f'- Deleting robot "{current_robot.name}" since it is not' + " defined in config files" + ) + await client.delete_robot(robot_id=current_robot.id) + + # Modify existing robots or create new ones + for target_robot in target_robots: + full_robot_name = await construct_full_robot_name(target_robot) + print(f'Full robot name: {full_robot_name}') + target_robot = Robot(**target_robot) + target_robot.secret = os.environ.get( + target_robot.name.upper().replace("-", "_") + ) + # Modify existing robot + if full_robot_name in current_robot_names: + robot_id = current_robot_id[ + current_robot_names.index(full_robot_name) + ] + target_robot.name = full_robot_name + print(f'- Syncing robot "{target_robot.name}".') + await client.update_robot(robot_id=robot_id, robot=target_robot) + # Create new robot + else: + print( + "- Creating new robot" + f' "{full_robot_name}"' + ) + try: + await client.create_robot(robot=target_robot) + except Conflict as e: + print( + f''' => "{full_robot_name}" + Harbor Conflict Error: {e}''' + ) + except BadRequest as e: + print(f'Bad request permission: {e}') + + +async def sync_webhook(project_name: str, policies: list[WebhookPolicy]): + print(f'PROJECT: "{project_name}"') + + target_policies = policies + current_policies = await client.get_webhook_policies( + project_name_or_id=project_name, + limit=None + ) + current_policy_names = [ + current_policy.name for current_policy in current_policies + ] + current_policy_id = [ + current_policy.id for current_policy in current_policies + ] + target_policy_names = [ + target_policy["name"] for target_policy in target_policies + ] + + # Delete all policies not defined in config file + for current_policy in current_policies: + if current_policy.name not in target_policy_names: + print( + f'- Deleting policy "{current_policy.name}" since it is not' + " defined in config files" + ) + await client.delete_webhook_policy( + project_name_or_id=project_name, + webhook_policy_id=current_policy.id, + ) + + # Modify existing policies or create new ones + for target_policy in target_policies: + # Modify existing policy + if target_policy["name"] in current_policy_names: + policy_id = current_policy_id[ + current_policy_names.index(target_policy["name"]) + ] + print(f'- Syncing policy "{target_policy["name"]}"') + await client.update_webhook_policy( + project_name_or_id=project_name, + webhook_policy_id=policy_id, + policy=target_policy, + ) + # Create new policy + else: + print(f'- Creating new policy "{target_policy["name"]}"') + await client.create_webhook_policy( + project_name_or_id=project_name, policy=target_policy + ) + + +async def sync_projects(target_projects: [Project]) -> None: + current_projects = await client.get_projects(limit=None) + current_project_names = [ + current_project.name for current_project in current_projects + ] + target_project_names = [ + target_project["project_name"] for target_project in target_projects + ] + + # Delete all projects not defined in config file + for current_project in current_projects: + if current_project.name not in target_project_names: + repositories = await client.get_repositories( + project_name=current_project.name, + limit=None + ) + if len(repositories) == 0: + print( + f'- Deleting project "{current_project.name}" since it is' + " empty and not defined in config files" + ) + await client.delete_project( + project_name_or_id=current_project.name + ) + else: + print( + f'- Deletion of project "{current_project.name}" not' + " possible since it is not empty" + ) + + # Modify existing projects or create new ones + for target_project in target_projects: + # Modify existing project + if target_project["project_name"] in current_project_names: + print(f'- Syncing project "{target_project["project_name"]}"') + await client.update_project( + project_name_or_id=current_project.name, project=target_project + ) + # Create new project + else: + print(f'- Creating new project "{target_project["project_name"]}"') + await client.create_project(project=target_project) + + +async def sync_project_members(project) -> None: + project_name = project["project_name"] + print(f'PROJECT: "{project_name}"') + + current_members = await client.get_project_members( + project_name_or_id=project_name, + limit=None + ) + target_members = [] + for project_role in ProjectRole: + target_members += [ + ProjectMemberEntity( + entity_name=username, role_id=project_role.value + ) + for username in project[project_role.name.lower()] + ] + + # Remove all non listed current project members + for current_member in current_members: + if current_member.entity_name not in [ + target_member.entity_name for target_member in target_members + ]: + print( + f'- Removing "{current_member.entity_name}" from project' + f' "{project_name}"' + ) + await client.remove_project_member( + project_name_or_id=project_name, + member_id=current_member.id, + ) + + # Update existing members and add new ones + for member in target_members: + member_id = get_member_id(current_members, member.entity_name) + # Sync existing members' project role + if member_id: + print(f'- Syncing project role of "{member.entity_name}"') + await client.update_project_member_role( + project_name_or_id=project_name, + member_id=member_id, + role=member.role_id, + ) + # Add new member + else: + print( + f'- Adding new member "{member.entity_name}" to project' + f' "{project_name}"' + ) + try: + await client.add_project_member_user( + project_name_or_id=project_name, + username_or_id=member.entity_name, + role_id=member.role_id, + ) + except NotFound: + print( + f' => ERROR: User "{member.entity_name}" not found. Make' + " sure the user has logged into harbor at least once" + ) + + +async def wait_until_healthy() -> None: + while True: + health = await client.health_check() + if health.status == "healthy": + print("- Harbor is healthy") + break + print("- Waiting for harbor to become healthy...") + sleep(5) + + +async def sync_admin_password() -> None: + try: + old_password_client = HarborAsyncClient( + url=api_url, + username=admin_username, + secret=old_admin_password, + timeout=10, + verify=False, + ) + admin = await old_password_client.get_current_user() + await old_password_client.set_user_password( + user_id=admin.user_id, + old_password=old_admin_password, + new_password=new_admin_password, + ) + print("- Updated admin password") + except Unauthorized: + print( + "- Admin password remains unchanged since it is does not match the" + " old admin password password" + ) + + +def get_member_id(members: [ProjectMemberEntity], username: str) -> int | None: + """Returns member id of username or None if username is not in members""" + for member in members: + if member.entity_name == username: + return member.id + return None + + +def parse_args(): + parser = argparse.ArgumentParser( + description="""Harbor Day 2 configurator to sync harbor configs""", + ) + args = parser.parse_args() + return args + + +asyncio.run(main())