Ensure correct dependencies are installed for lowerbounds and cloud s…

…torage tests

+Add back patched install for botocore for S3 testing
  Using MinIO for our S3 test backend requires us to use this patch
  to avoid timeouts when uploading 0-byte files.

[noissue]

templates/github/.ci/ansible/Containerfile.j2.copy

@@ -2,27 +2,24 @@ FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }
 
 # Add source directories to container
 {% for item in plugins %}
-{% if item.source.startswith("./") or item.ci_requirements | default(false) %}
 ADD ./{{ item.name }} ./{{ item.name }}
-{% endif %}
 {% endfor %}
 
 # Install python packages
+# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
+# Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
 RUN pip3 install
+{%- if s3_test | default(false) -%}
+{{ " " }}git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
+{%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}
 {{ " " }}python-dateutil rhsm
 {%- endif -%}
 {{ " " }}{{ item.source }}
-{%- if item.name == "pulpcore" -%}
-{%- if s3_test | default(false) -%}
-[s3]
-{%- elif azure_test | default(false) -%}
-[azure]
-{%- elif gcp_test | default(false) -%}
-[google]
-{%- endif -%}
+{%- if item.lowerbounds | default(false) -%}
+{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt
 {%- endif -%}
 {%- if item.ci_requirements | default(false) -%}
 {{ " " }}-r ./{{ item.name }}/ci_requirements.txt

templates/github/.github/workflows/scripts/before_install.sh.j2

@@ -61,8 +61,7 @@ then
 fi
 
 if [[ "$TEST" = "lowerbounds" ]]; then
-  python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_requirements.txt
-  mv lowerbounds_requirements.txt requirements.txt
+  python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt
 fi
 
 if [ -f $POST_BEFORE_INSTALL ]; then

templates/github/.github/workflows/scripts/install.sh.j2

@@ -39,19 +39,44 @@ fi
 
 cd .ci/ansible/
 
+{%- set PULPCORE_PREFIX = "" if plugin_name == "pulpcore" else " pulpcore" %}
+PLUGIN_SOURCE="${PLUGIN_NAME}"
+{%- if test_s3 %}
+if [ "$TEST" = "s3" ]; then
+  PLUGIN_SOURCE="${PLUGIN_SOURCE}{{ PULPCORE_PREFIX }}[s3]"
+fi
+{%- endif %}
+{%- if test_azure %}
+if [ "$TEST" = "azure" ]; then
+  PLUGIN_SOURCE="${PLUGIN_SOURCE}{{ PULPCORE_PREFIX }}[azure]"
+fi
+{%- endif %}
+{%- if test_gcp %}
+if [ "$TEST" = "gcp" ]; then
+  PLUGIN_SOURCE="${PLUGIN_SOURCE}{{ PULPCORE_PREFIX }}[google]"
+fi
+{%- endif %}
+
 cat >> vars/main.yaml << VARSYAML
 image:
   name: pulp
   tag: "ci_build"
 plugins:
   - name: {{ plugin_name }}
-    source: "${PLUGIN_NAME}"
+    source: "${PLUGIN_SOURCE}"
 VARSYAML
 if [[ -f ../../ci_requirements.txt ]]; then
   cat >> vars/main.yaml << VARSYAML
     ci_requirements: true
 VARSYAML
 fi
+{%- if test_lowerbounds %}
+if [ "$TEST" = "lowerbounds" ]; then
+  cat >> vars/main.yaml << VARSYAML
+    lowerbounds: true
+VARSYAML
+fi
+{%- endif %}
 
 cat >> vars/main.yaml << VARSYAML
 services: