Set up oci-prow-worker on OCI with OpenTofu

iac/oci-prow-worker/.gitignore

@@ -0,0 +1,6 @@
+# Terraform folder
+.terraform
+# Make sure to not allow checking in tfvars by mistake
+*.tfvars
+# Environment variables are often stored in this file
+.env

iac/oci-prow-worker/Makefile

@@ -0,0 +1,13 @@
+OPENTOFU_CLI ?= tofu
+
+init:
+	$(OPENTOFU_CLI) init
+
+fmt:
+	$(OPENTOFU_CLI) fmt
+
+plan:
+	$(OPENTOFU_CLI) plan
+
+apply:
+	$(OPENTOFU_CLI) apply

iac/oci-prow-worker/README.md

@@ -0,0 +1,11 @@
+# oci-prow-cluster
+
+This directory deploys the `oci-prow-cluster` OKE cluster in OCI (Oracle Cloud) via [OpenTofu](https://opentofu.org). A shared state is stored in a OCI storage bucket, please make sure to use that. Usually, this code shouldn't be executed directly but run by Prow.
+
+## Required Environment Variables
+
+The following environment variables are required before running any `make` targets:
+
+- `AWS_ACCESS_KEY_ID`: Needs to be the key ID for a [Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2) to access OCI's S3-compatible storage buckets.
+- `AWS_SECRET_ACCESS_KEY`: Needs to be the secret for a [Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2) to access OCI's S3-compatible storage buckets.
+- `AWS_ENDPOINT_URL_S3`: Needs to be `https://<object namespace>.compat.objectstorage.us-sanjose-1.oraclecloud.com`. Replace `<object namespace>` with the namespace displayed on the bucket (see OCI Console for this information).

iac/oci-prow-worker/cluster.tf

@@ -0,0 +1,41 @@
+resource "oci_containerengine_cluster" "prow" {
+  name               = "oci-prow-worker"
+  kubernetes_version = var.kubernetes_version
+
+  compartment_id = var.oci_compartment_ocid
+  vcn_id         = oci_core_vcn.prow.id
+
+  cluster_pod_network_options {
+    cni_type = "flannel"
+  }
+}
+
+resource "oci_containerengine_node_pool" "prow_worker" {
+  cluster_id     = oci_containerengine_cluster.prow.id
+  compartment_id = var.oci_compartment_ocid
+  subnet_ids     = oci_core_subnet.prow_worker_cluster[*].id
+
+  kubernetes_version = var.kubernetes_version
+  name               = "prow-worker"
+  ssh_public_key     = var.node_pool_ssh_public_key
+
+  # this matches t3.2xlarge sizings.
+  node_shape = "VM.Standard.A1.Flex"
+  node_shape_config {
+    memory_in_gbs = 32
+    ocpus         = 8
+  }
+
+  node_config_details {
+    size = var.node_pool_worker_size
+    # create placement_configs for each availability domain.
+    # There happens to be only a single one in us-sanjose-1.
+    dynamic "placement_configs" {
+      for_each = oci_core_subnet.prow_worker_cluster
+      content {
+        availability_domain = data.oci_identity_availability_domains.availability_domains.availability_domains[index(oci_core_subnet.prow_worker_cluster, placement_configs.value)].id
+        subnet_id           = placement_configs.value.id
+      }
+    }
+  }
+}

iac/oci-prow-worker/network.tf

@@ -0,0 +1,36 @@
+resource "oci_core_vcn" "prow" {
+  cidr_block     = "10.0.0.0/16"
+  compartment_id = var.oci_compartment_ocid
+  display_name   = "Prow Network"
+}
+
+resource "oci_core_internet_gateway" "prow" {
+  compartment_id = var.oci_compartment_ocid
+  display_name   = "Prow Internet Gateway"
+  vcn_id         = oci_core_vcn.prow.id
+}
+
+resource "oci_core_route_table" "prow_worker" {
+  compartment_id = var.oci_compartment_ocid
+  vcn_id         = oci_core_vcn.prow.id
+  display_name   = "Prow Worker Route Table"
+
+  route_rules {
+    destination       = "0.0.0.0/0"
+    destination_type  = "CIDR_BLOCK"
+    network_entity_id = oci_core_internet_gateway.prow.id
+  }
+}
+
+resource "oci_core_subnet" "prow_worker_cluster" {
+  count = length(data.oci_identity_availability_domains.availability_domains.availability_domains)
+
+  availability_domain = data.oci_identity_availability_domains.availability_domains.availability_domains[count.index].name
+  cidr_block          = "10.0.${20 + count.index}.0/24"
+  compartment_id      = var.oci_compartment_ocid
+  vcn_id              = oci_core_vcn.prow.id
+
+  security_list_ids = [oci_core_vcn.prow.default_security_list_id]
+  route_table_id    = oci_core_route_table.prow_worker.id
+  display_name      = "Prow Cluster Subnet ${count.index}"
+}

iac/oci-prow-worker/provider.tf

@@ -0,0 +1,9 @@
+provider "oci" {
+  tenancy_ocid = var.oci_tenant_ocid
+  region       = var.oci_region
+}
+
+data "oci_identity_availability_domains" "availability_domains" {
+  compartment_id = var.oci_tenant_ocid
+}
+

iac/oci-prow-worker/terraform.tf

@@ -0,0 +1,21 @@
+terraform {
+  required_providers {
+    oci = {
+      source  = "oracle/oci"
+      version = "5.36.0"
+    }
+  }
+
+  # make sure to set AWS_ENDPOINT_URL_S3 to 'https://<object namespace>.compat.objectstorage.us-sanjose-1.oraclecloud.com'.
+  backend "s3" {
+    bucket = "kcp-opentofu-state"
+    region = "us-sanjose-1"
+    key    = "ci-prow-worker/tf.tfstate"
+
+    skip_region_validation      = true
+    skip_credentials_validation = true
+    skip_requesting_account_id  = true
+    use_path_style              = true
+    skip_metadata_api_check     = true
+  }
+}

iac/oci-prow-worker/variables.tf

@@ -0,0 +1,36 @@
+variable "oci_tenant_ocid" {
+  type = string
+}
+
+variable "oci_compartment_ocid" {
+  type = string
+}
+
+/*
+variable "oci_user_ocid" {
+  type = string
+}
+
+variable "oci_private_key" {
+  type      = string
+  sensitive = true
+}
+*/
+
+variable "oci_region" {
+  type = string
+}
+
+variable "node_pool_ssh_public_key" {
+  type = string
+}
+
+variable "node_pool_worker_size" {
+  type    = number
+  default = 3
+}
+
+variable "kubernetes_version" {
+  type = string
+  default = "v1.29.1"
+}