Ability to use updater script to pin dev requirements (#14644)

* Add a dev option for updater script to pin CI reqs

* Avoid removing git links for dev requirements

* Add dev to primary options

* Fix up sanitize git switch

requirements/requirements_dev.txt

@@ -10,7 +10,6 @@ black
 pytest!=7.0.0
 pytest-cov
 pytest-django
-pytest-pythonpath
 pytest-mock==1.11.1
 pytest-timeout
 pytest-xdist==1.34.0 # 2.0.0 broke zuul for some reason

requirements/updater.sh

@@ -4,7 +4,9 @@ set -ue
 requirements_in="$(readlink -f ./requirements.in)"
 requirements="$(readlink -f ./requirements.txt)"
 requirements_git="$(readlink -f ./requirements_git.txt)"
+requirements_dev="$(readlink -f ./requirements_dev.txt)"
 pip_compile="pip-compile --no-header --quiet -r --allow-unsafe"
+sanitize_git="1"
 
 _cleanup() {
   cd /
@@ -21,18 +23,22 @@ generate_requirements() {
   # FIXME: https://github.com/jazzband/pip-tools/issues/1558
   ${venv}/bin/python3 -m pip install -U 'pip<22.0' pip-tools
 
-  ${pip_compile} "${requirements_in}" "${requirements_git}" --output-file requirements.txt
+  ${pip_compile} "$1" --output-file requirements.txt
   # consider the git requirements for purposes of resolving deps
   # Then remove any git+ lines from requirements.txt
-  while IFS= read -r line; do
-    if [[ $line != \#* ]]; then  # ignore comments
-      sed -i "\!${line%#*}!d" requirements.txt
-    fi
-  done < "${requirements_git}"
+  if [[ "$sanitize_git" == "1" ]] ; then
+    while IFS= read -r line; do
+      if [[ $line != \#* ]]; then  # ignore comments
+        sed -i "\!${line%#*}!d" requirements.txt
+      fi
+    done < "${requirements_git}"
+  fi;
 }
 
 main() {
   base_dir=$(pwd)
+  dest_requirements="${requirements}"
+  input_requirements="${requirements_in} ${requirements_git}"
 
   _tmp=$(python -c "import tempfile; print(tempfile.mkdtemp(suffix='.awx-requirements', dir='/tmp'))")
 
@@ -42,6 +48,12 @@ main() {
     "run")
       NEEDS_HELP=0
     ;;
+    "dev")
+      dest_requirements="${requirements_dev}"
+      input_requirements="${requirements_dev}"
+      sanitize_git=0
+      NEEDS_HELP=0
+    ;;
     "upgrade")
       NEEDS_HELP=0
       pip_compile="${pip_compile} --upgrade"
@@ -61,12 +73,13 @@ main() {
     echo "This script generates requirements.txt from requirements.in and requirements_git.in"
     echo "It should be run from within the awx container"
     echo ""
-    echo "Usage: $0 [run|upgrade]"
+    echo "Usage: $0 [run|upgrade|dev]"
     echo ""
     echo "Commands:"
     echo "help      Print this message"
     echo "run       Run the process only upgrading pinned libraries from requirements.in"
     echo "upgrade   Upgrade all libraries to latest while respecting pinnings"
+    echo "dev       Pin the development requirements file"
     echo ""
     exit
   fi
@@ -85,10 +98,10 @@ main() {
   cp -vf requirements.txt "${_tmp}"
   cd "${_tmp}"
 
-  generate_requirements
+  generate_requirements "${input_requirements}"
 
   echo "Changing $base_dir to /awx_devel/requirements"
-  cat requirements.txt | sed "s:$base_dir:/awx_devel/requirements:" > "${requirements}"
+  cat requirements.txt | sed "s:$base_dir:/awx_devel/requirements:" > "${dest_requirements}"
 
   _cleanup
 }