Fix/single table job fixes

Docker/awshelper/Dockerfile

@@ -126,4 +126,4 @@ RUN git config --global user.email gen3 \
 RUN export GEN3_HOME="$HOME/cloud-automation" \
     && bash -c 'source "$GEN3_HOME/gen3/gen3setup.sh" && gen3 help'
 
-CMD /bin/bash
+CMD ["/bin/bash"]

Docker/python-nginx/python3.10-buster/Dockerfile

@@ -2,9 +2,9 @@
 FROM quay.io/cdis/python:3.10-buster
 
 # https://github.com/nginxinc/docker-nginx/blob/f958fbacada447737319e979db45a1da49123142/mainline/debian/Dockerfile
-ENV NGINX_VERSION   1.21.1
-ENV NJS_VERSION     0.6.1
-ENV PKG_RELEASE     1~buster
+ENV NGINX_VERSION=1.21.1
+ENV NJS_VERSION=0.6.1
+ENV PKG_RELEASE=1~buster
 
 RUN set -x \
 # create nginx user/group first, to be consistent throughout docker variants
@@ -126,7 +126,7 @@ RUN apt-get update && apt-get install -y supervisor \
 COPY supervisord.ini /etc/supervisor.d/supervisord.ini
 
 # Which uWSGI .ini file should be used, to make it customizable
-ENV UWSGI_INI /app/uwsgi.ini
+ENV UWSGI_INI=/app/uwsgi.ini
 
 # By default, disable uwsgi cheaper mode and run 2 processes.
 # If UWSGI_CHEAPER=N and UWSGI_PROCESSES=M, N is the min and M is the max
@@ -138,16 +138,16 @@ ENV UWSGI_PROCESSES=2
 # By default, allow unlimited file sizes, modify it to limit the file sizes
 # To have a maximum of 1 MB (Nginx's default) change the line to:
 # ENV NGINX_MAX_UPLOAD 1m
-ENV NGINX_MAX_UPLOAD 0
+ENV NGINX_MAX_UPLOAD=0
 
 # By default, Nginx will run a single worker process, setting it to auto
 # will create a worker for each CPU core
-ENV NGINX_WORKER_PROCESSES 1
+ENV NGINX_WORKER_PROCESSES=1
 
 # By default, Nginx listens on port 80.
 # To modify this, change LISTEN_PORT environment variable.
 # (in a Dockerfile or with an option for `docker run`)
-ENV LISTEN_PORT 80
+ENV LISTEN_PORT=80
 
 # Copy the entrypoint that will generate Nginx additional configs
 COPY entrypoint.sh /entrypoint.sh
@@ -165,4 +165,4 @@ ENTRYPOINT ["sh", "/entrypoint.sh"]
 COPY ./app /app
 WORKDIR /app
 
-CMD ["/usr/bin/supervisord"]
+CMD ["/usr/bin/supervisord"]

kube/services/jobs/indexd-single-table-migration-job.yaml

@@ -1,4 +1,5 @@
 # Setup for running this migration https://github.com/uc-cdis/indexd/blob/master/docs/migration_to_single_table_indexd.md
+# Need to apply PriorityClass before running this job. Run kubectl apply -f indexd-single-table-migration-priority-class.yaml
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -9,6 +10,7 @@ spec:
       labels:
         app: gen3job
     spec:
+      priorityClassName: indexd-single-table-migration-priority  # Assigning the priority class
       affinity:
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -46,6 +48,12 @@ spec:
           env:
             - name: START_DID
               GEN3_START_DID|-value: ""-|
+            - name: BATCH_SIZE
+              GEN3_BATCH_SIZE|-value: ""-|
+            - name: OFFSET_VALUE
+              GEN3_OFFSET_VALUE|-value: ""-|
+            - name: USE_DEFAULT_CREDS
+              GEN3_USE_DEFAULT_CREDS|-value: "false"-|
           volumeMounts:
             - name: "config-volume"
               readOnly: true
@@ -72,11 +80,28 @@ spec:
           args:
             - "-c"
             - |
-              flags="--creds-path /var/www/indexd/creds_new.json"
+              $flags=""
+
+              if [[ "$USE_DEFAULT_CREDS" == "false" ]]; then
+                echo "Using new creds"
+                flags="$flags --creds-file /var/www/indexd/creds_new.json"
+              else
+                echo "using default creds"
+                flags="$flags --creds-file /var/www/indexd/creds.json"
+              fi
+
               if [[ -n "$START_DID" ]]; then
                 flags="$flags --start-did $START_DID"
               fi
-              time poetry run python /indexd/bin/migrate_to_single_table.py $flags
+              if [[ -n "$BATCH_SIZE" ]]; then
+                flags="$flags --batch-size $BATCH_SIZE"
+              fi
+
+              if [[ -n "$OFFSET_VALUE" ]]; then
+                flags="$flags --start-offset $OFFSET_VALUE"
+              fi
+
+              time python /indexd/bin/migrate_to_single_table.py $flags
               echo "Exit code: $?"
       restartPolicy: Never
 

kube/services/jobs/indexd-single-table-migration-priority-class.yaml

@@ -0,0 +1,7 @@
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: indexd-single-table-migration-priority
+value: 100000  # High value for high priority
+globalDefault: false
+description: "Priority class for Indexd migration job"