[DNM] Testing manual install plan.

playbooks/update.yml

@@ -38,6 +38,7 @@
         cifmw_set_openstack_containers_openstack_final_env: "operator_env_after_update.txt"
       ansible.builtin.include_role:
         name: set_openstack_containers
+      when: cifmw_set_openstack_containers | default(true) | bool
 
 - name: Sync repos for controller to compute
   hosts: computes

reproducer.yml

@@ -74,6 +74,20 @@
   gather_facts: false
   roles:
     - role: reproducer
+  tasks:
+    - name: Read the podman.txt file
+      ansible.builtin.slurp:
+        src: /tmp/podman.txt
+      register: podman_file
+
+    - name: Decode file content
+      ansible.builtin.set_fact:
+        _podman_content: "{{ podman_file['content'] | b64decode }}"
+
+    - name: Parse the file content
+      ansible.builtin.set_fact:
+        cifmw_update_podman_username: "{{ _podman_content.split()[0] }}"
+        cifmw_update_podman_password: "{{ _podman_content.split()[1] }}"
   post_tasks:
     - name: Allow traffic from OSP VMs to OSP API (needed for shiftstack)
       become: true
@@ -94,6 +108,13 @@
             name: firewalld
             state: restarted
 
+    - name: Recreate the podman.txt file on controller-0
+      ansible.builtin.copy:
+        content: "{{ cifmw_update_podman_username }} {{ cifmw_update_podman_password }}\n"
+        dest: /tmp/podman.txt
+        mode: '0644'
+      delegate_to: controller-0
+
     - name: Run deployment if instructed to
       when:
         - cifmw_deploy_architecture | default(false) | bool

roles/ci_gen_kustomize_values/templates/common/edpm-nodeset-values/values.yaml.j2

@@ -19,6 +19,9 @@ data:
   nodeset:
     ansible:
       ansibleUser: "zuul"
+      ansibleVarsFrom:
+        - secretRef:
+            name: redhat-registry
       ansibleVars:
         edpm_fips_mode: "{{ 'enabled' if cifmw_fips_enabled|default(false)|bool else 'check' }}"
         timesync_ntp_servers:

roles/kustomize_deploy/tasks/authenticate_registry.yml

@@ -0,0 +1,71 @@
+---
+- name: Retrieve .dockerconfigjson from OpenShift secret
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    oc get secret/pull-secret -n openshift-config -o json |
+    jq -r '.data[".dockerconfigjson"]' | base64 -d > /home/zuul/authfile.txt
+
+- name: Read username and password from PODMAN_LOGIN_FILE
+  ansible.builtin.shell: >-
+    read -r username password < /tmp/podman.txt
+    && echo "${username}" "${password}"
+  register: podman_login_creds
+
+- name: Log in to Podman with registry credentials
+  ansible.builtin.command: >-
+    podman login --authfile /home/zuul/authfile.txt
+    --username "{{ podman_login_creds.stdout.split()[0] }}"
+    --password "{{ podman_login_creds.stdout.split()[1] }}"
+    brew.registry.redhat.io
+
+- name: Update OpenShift secret with the new .dockerconfigjson
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  ansible.builtin.command: >-
+    oc set data secret/pull-secret -n openshift-config
+    --from-file=.dockerconfigjson=/home/zuul/authfile.txt
+
+- name: Read username and password and create Kubernetes secret
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  ansible.builtin.shell: |
+    read -r username password < /tmp/podman.txt
+    # TODO: bug against doc: https://docs.redhat.com/en/documentation/red_hat_openstack_services_on_openshift/18.0/html-single/deploying_red_hat_openstack_services_on_openshift/index#proc_creating-the-data-plane-secrets_dataplane
+    oc create -n openstack secret generic redhat-registry \
+      --from-literal edpm_container_registry_logins="{\"registry.redhat.io\": {\"${username}\": \"${password}\"}}"
+  args:
+    executable: /bin/bash
+
+- name: Create ImageContentSourcePolicy YAML
+  ansible.builtin.copy:
+    dest: "/home/zuul/brew-registry-imageContentSourcePolicy.yaml"
+    content: |
+      apiVersion: operator.openshift.io/v1alpha1
+      kind: ImageContentSourcePolicy
+      metadata:
+        name: brew-registry
+      spec:
+        repositoryDigestMirrors:
+        - mirrors:
+          - brew.registry.redhat.io
+          source: registry.redhat.io
+        - mirrors:
+          - brew.registry.redhat.io
+          source: registry.stage.redhat.io
+        - mirrors:
+          - brew.registry.redhat.io
+          source: registry-proxy.engineering.redhat.com
+
+- name: Apply ImageContentSourcePolicy using oc
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  ansible.builtin.command: oc apply -f /home/zuul/brew-registry-imageContentSourcePolicy.yaml
+  args:
+    executable: /bin/bash
+

roles/kustomize_deploy/tasks/install_operators.yml

@@ -64,6 +64,27 @@
     dest: "{{ cifmw_kustomize_deploy_olm_dest_file }}"
     mode: "0644"
 
+# The openstack namespace must be present to be able to create the
+# secret generic redhat-registry in it.  This is copy of what is
+# generated.
+- name: Create Kubernetes namespace
+  kubernetes.core.k8s:
+    state: present
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit) }}"
+    context: "{{ cifmw_openshift_context | default(omit) }}"
+    definition:
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        labels:
+          pod-security.kubernetes.io/enforce: privileged
+          security.openshift.io/scc.podSecurityLabelSync: "false"
+        name: openstack
+
+- name: Authenticate to private registry
+  ansible.builtin.include_tasks: authenticate_registry.yml
+
 - name: OLM resources
   when: not cifmw_kustomize_deploy_generate_crs_only
   block:
@@ -107,6 +128,41 @@
         - _cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources | length == 1
         - (_cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources | first)['status']['installPlanRef'] is defined
 
+    - name: Retrieve the latest InstallPlan name sorted by creation date
+      ansible.builtin.set_fact:
+        installplan_name: >-
+          {{
+            _cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources
+            | selectattr('kind', 'equalto', 'Subscription')
+            | sort(attribute='metadata.creationTimestamp')
+            | map(attribute='status.installPlanRef.name')
+            | last
+          }}
+
+    - name: Display debug
+      ansible.builtin.debug:
+        msg: "{{ installplan_name }}"
+
+    - name: Accept the InstallPlan
+      kubernetes.core.k8s:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        api_key: "{{ cifmw_openshift_token | default(omit)}}"
+        context: "{{ cifmw_openshift_context | default(omit) }}"
+        api_version: operators.coreos.com/v1alpha1
+        kind: InstallPlan
+        name: "{{ installplan_name }}"
+        namespace: "openstack-operators"
+        state: present
+        merge_type:
+          - merge
+        definition:
+          spec:
+            approved: true
+
+    - name: Display debug
+      ansible.builtin.debug:
+        msg: "========> INSTALLPLAN '{{ installplan_name }}' HAS BEEN APPROVED."
+
     - name: Wait for the openstack operators InstallPlan to be finished
       vars:
         _install_plan: >-

roles/update/files/collect-state.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/bash
+
+BASE_DIR="${HOME}/ci-framework-data/tests/update/"
+CI_INVENTORY="${HOME}/ci-framework-data/artifacts/zuul_inventory.yml"
+OPERATOR_NAMESPACE="openstack-operators"
+NAMESPACE="openstack"
+
+function get_current_compute_state {
+    local stage="${1:-}"
+    file_pre="${BASE_DIR}${stage:+${stage}-}"
+
+    if [ -e "${CI_INVENTORY}" ]; then
+        echo "Collecting compute state ${stage:+for $stage }in ${BASE_DIR:-${PWD}}"
+
+        # Collect all running containers an all compute nodes in ~/ci-framework-data/tests/update/ by default.
+        ansible -i "${CI_INVENTORY}" -m shell -a \
+                "sudo podman ps -q --filter 'status=running' | xargs -I {} sudo podman inspect --format {% raw %} '{{.Name}} {{.Config.Image}} {{.State.StartedAt}}' {% endraw %} {}|sort" computes | \
+            awk -vfile_pre="${file_pre}" 'BEGIN {tp=strftime("%Y%m%d%H%M%S")} /^compute/ {if (s != "") {close(s)}; s = "containers-" $1 "_" tp ".txt"; next;}; s {print > file_pre s} '
+        # Collect packages list an all compute nodes in ~/ci-framework-data/tests/update/ by default.
+        ansible -i "${CI_INVENTORY}" -m shell -a \
+                "sudo dnf list installed | sort" computes | \
+            awk -vfile_pre="${file_pre}" 'BEGIN {tp=strftime("%Y%m%d%H%M%S")} /^compute/ {if (s != "") {close(s)}; s = "packages-" $1 "_" tp ".txt"; next;}; s {print > file_pre s} '
+    fi
+}
+
+function get_current_pod_state {
+    local stage="${1:-}"
+    file_pre="${BASE_DIR}${stage:+${stage}-}"
+
+    echo "Collecting pod state ${stage:+for $stage }in ${BASE_DIR:-${PWD}}"
+
+    local openstack_state_file="${file_pre}pods_os_state_$(date +%Y%m%d_%H%M%S).tsv"
+    local os_operator_state_file="${file_pre}pods_os_op_state_$(date +%Y%m%d_%H%M%S).tsv"
+    oc get pods -n "${OPERATOR_NAMESPACE}" -o json | jq -r '.items[] | select(.status.phase == "Running") | . as $pod | .status.containerStatuses[] | [$pod.metadata.name, $pod.status.startTime, .image, .state.running.startedAt ] | @tsv' > $os_operator_state_file
+
+    oc get pods -n "${NAMESPACE}" -o json | jq -r '.items[] | select(.status.phase == "Running") | . as $pod | .status.containerStatuses[] | [$pod.metadata.name, $pod.status.startTime, .image, .state.running.startedAt ] | @tsv' > $openstack_state_file
+}
+
+function get_current_state {
+    local stage="${1:-}"
+    get_current_compute_state "${stage}"
+    get_current_pod_state "${stage}"
+}
+
+declare -A STAGES
+STAGES=(
+  [1]="01_before_update"
+  [2]="02_after_openstack_operator_update"
+  [3]="03_after_ovn_controlplane_update"
+  [4]="04_after_ovn_dataplane_update"
+  [5]="05_after_controlplane_update"
+  [6]="06_after_update"
+)
+
+print_stages() {
+  echo "Available options:"
+  for key in "${!STAGES[@]}"; do
+    echo "$key) ${STAGES[$key]}"
+  done
+}
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <option>"
+  echo "Try '$0 --list' to see available options."
+  exit 1
+fi
+
+if [[ $1 == "--list" ]]; then
+  print_stages
+  exit 0
+fi
+
+# Determine the state based on the input argument
+option="$1"
+case $option in
+  1|2|3|4|5|6)
+    selected_state="${STAGES[$option]}"
+    ;;
+  "01_before_update"|"02_after_openstack_operator_update"|"03_after_ovn_controlplane_update"|"04_after_ovn_dataplane_update"|"05_after_controlplane_update"|"06_after_update")
+    selected_state="$option"
+    ;;
+  *)
+    echo "Invalid option: $option"
+    echo "Try '$0 --list' to see available options."
+    exit 1
+    ;;
+esac
+
+mkdir -p "${BASE_DIR}"
+
+# Execute the selected state logic
+echo "Executing action for state: $selected_state"
+get_current_state "$selected_state"