Merge discvr-23.11 to discvr-24.3

LabPurchasing/resources/queries/labpurchasing/purchases.js

@@ -4,6 +4,31 @@ var console = require("console");
 var triggerHelper = new org.labkey.labpurchasing.LabPurchasingTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id);
 
 function beforeInsert(row, errors){
+    beforeUpsert(row, errors)
+}
+
+function beforeUpdate(row, oldRow, errors){
+    beforeUpsert(row, errors)
+}
+
+function beforeUpsert(row, errors){
+    // Validate requestor:
+    if (row.requestor) {
+        if (!isNaN(row.requestor) && !triggerHelper.isValidUserId(row.requestor)) {
+            errors.requestor = 'Unknown userId for requestor: ' + row.requestor;
+        }
+        // Try to resolve strings:
+        else if (isNaN(row.requestor)) {
+            var id = triggerHelper.resolveUserId(String(row.requestor));
+            if (!id) {
+                errors.requestor = 'Unknown userId for requestor: ' + row.requestor;
+            }
+            else {
+                row.requestor = id;
+            }
+        }
+    }
+
     // The purpose of this is to allow the user to provide a string value for
     // vendorId or vendorName, and attempt to resolve this against known vendors:
     if (!row.vendorId || isNaN(row.vendorId)) {

LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java

@@ -112,4 +112,16 @@ public void sendNotifications(List<Integer> rowIds) {
             _log.error("Unable to send purchasing email", e);
         }
     }
+
+    public boolean isValidUserId(int userId)
+    {
+        return UserManager.getUser(userId) != null;
+    }
+
+    public Integer resolveUserId(String userNameOrEmail)
+    {
+        User u = UserManager.getUserByDisplayName(userNameOrEmail);
+
+        return u == null ? null : u.getUserId();
+    }
 }

mGAP/resources/etls/prime-seq.xml

@@ -3,6 +3,13 @@
     <name>PRIMe-Seq ETLs</name>
     <description>Syncs Anonymized Data PRIMe-Seq to mGAP</description>
     <transforms>
+        <transform id="step0" type="TaskRefTransformStep">
+            <taskref ref="org.labkey.mgap.etl.ClearEtlWorkQueueTask">
+                <settings>
+
+                </settings>
+            </taskref>
+        </transform>
         <transform id="step1" type="RemoteQueryTransformStep">
             <description>Copy to local table</description>
             <source schemaName="mGAP" queryName="subjectsSource" remoteSource="PRIMESEQ">
@@ -190,6 +197,14 @@
                 </columnTransforms>
             </destination>
         </transform>
+
+        <transform id="step7" type="TaskRefTransformStep">
+            <taskref ref="org.labkey.mgap.etl.PerformQueuedEtlWorkTask">
+                <settings>
+
+                </settings>
+            </taskref>
+        </transform>
     </transforms>
     <schedule>
         <cron expression="0 30 1 * * ?"/>

mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java

@@ -1,6 +1,5 @@
 package org.labkey.mgap.columnTransforms;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
 import org.labkey.api.data.Results;
@@ -21,13 +20,12 @@
 import org.labkey.api.query.QueryService;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.PageFlowUtil;
+import org.labkey.mgap.etl.EtlQueueManager;
 import org.labkey.mgap.mGAPManager;
 
 import java.io.File;
-import java.io.IOException;
 import java.net.URI;
 import java.sql.SQLException;
-import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -107,7 +105,7 @@ protected Integer getOrCreateOutputFile(Object dataFileUrl, Object folderName, S
         {
             if (dataFileUrl == null)
             {
-                throw new IllegalArgumentException("DataFileUrl was null.");
+                throw new IllegalArgumentException("DataFileUrl was null");
             }
 
             URI uri = new URI(String.valueOf(dataFileUrl));
@@ -215,20 +213,8 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx
 
         if (doCopy)
         {
-            getStatusLogger().info("copying file locally: " + localCopy.getPath());
-            if (localCopy.exists())
-            {
-                localCopy.delete();
-            }
-
-            try
-            {
-                FileUtils.copyFile(f, localCopy);
-            }
-            catch (IOException e)
-            {
-                throw new PipelineJobException(e);
-            }
+            getStatusLogger().info("queueing file copy: " + localCopy.getPath());
+            EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), f, localCopy);
         }
 
         File index = new File(f.getPath() + ".tbi");
@@ -243,15 +229,8 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx
 
             if (!indexLocal.exists())
             {
-                getStatusLogger().info("copying index locally: " + indexLocal.getPath());
-                try
-                {
-                    FileUtils.copyFile(index, indexLocal);
-                }
-                catch (IOException e)
-                {
-                    throw new PipelineJobException(e);
-                }
+                getStatusLogger().info("queueing copy of index: " + indexLocal.getPath());
+                EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), index, indexLocal);
             }
         }
 

mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java

@@ -4,6 +4,7 @@
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
+import org.labkey.mgap.etl.EtlQueueManager;
 
 import java.io.File;
 import java.io.IOException;
@@ -26,12 +27,7 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
         // NOTE: lucene is a special case since the DB tracks one file, but we need this whole folder:
         File sourceDir = f.getParentFile();
         File targetDir = new File(subdir, "LuceneIndex");
-
-        // NOTE: rsync should no-op if there are no source changes
-        getStatusLogger().info("Copying lucene index dir to: " + targetDir.getPath());
-        new SimpleScriptWrapper(getStatusLogger()).execute(Arrays.asList(
-                "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), targetDir.getPath()
-        ));
+        EtlQueueManager.get().queueRsyncCopy(getContainerUser().getContainer(), sourceDir, targetDir);
 
         return new File(targetDir, sourceDir.getName() + "/" + f.getName());
     }

mGAP/src/org/labkey/mgap/etl/ClearEtlWorkQueueTask.java

@@ -0,0 +1,43 @@
+package org.labkey.mgap.etl;
+
+import org.apache.xmlbeans.XmlException;
+import org.jetbrains.annotations.NotNull;
+import org.labkey.api.di.TaskRefTask;
+import org.labkey.api.pipeline.PipelineJob;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.pipeline.RecordedActionSet;
+import org.labkey.api.writer.ContainerUser;
+
+import java.util.List;
+import java.util.Map;
+
+public class ClearEtlWorkQueueTask  implements TaskRefTask
+{
+    private ContainerUser _containerUser = null;
+
+    @Override
+    public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException
+    {
+        EtlQueueManager.get().clearQueue(_containerUser.getContainer(), job.getLogger());
+
+        return new RecordedActionSet();
+    }
+
+    @Override
+    public List<String> getRequiredSettings()
+    {
+        return null;
+    }
+
+    @Override
+    public void setSettings(Map<String, String> settings) throws XmlException
+    {
+
+    }
+
+    @Override
+    public void setContainerUser(ContainerUser containerUser)
+    {
+        _containerUser = containerUser;
+    }
+}

mGAP/src/org/labkey/mgap/etl/EtlQueueManager.java

@@ -0,0 +1,122 @@
+package org.labkey.mgap.etl;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.logging.log4j.Logger;
+import org.labkey.api.data.Container;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class EtlQueueManager
+{
+    private static EtlQueueManager _instance = new EtlQueueManager();
+
+    private Map<Container, List<Pair<File, File>>> _pendingFileCopy = new HashMap<>();
+
+    private Map<Container, List<Pair<File, File>>> _pendingRsyncCopy = new HashMap<>();
+
+    private EtlQueueManager()
+    {
+
+    }
+
+    public static EtlQueueManager get()
+    {
+        return _instance;
+    }
+
+    public void clearQueue(Container container, Logger log)
+    {
+        if (_pendingFileCopy.containsKey(container) && !_pendingFileCopy.get(container).isEmpty())
+        {
+            log.error("The file copy queue was not empty!");
+        }
+
+        if (_pendingRsyncCopy.containsKey(container) && !_pendingRsyncCopy.get(container).isEmpty())
+        {
+            log.error("The rsync copy queue was not empty!");
+        }
+
+        _pendingFileCopy.clear();
+        _pendingRsyncCopy.clear();
+    }
+
+    public void performQueuedWork(Container container, Logger log)
+    {
+        List<Pair<File, File>> queue = _pendingFileCopy.get(container);
+        if (queue != null && !queue.isEmpty())
+        {
+            queue.forEach(x -> copyFile(x.getLeft(), x.getRight(), log));
+        }
+        _pendingFileCopy.clear();
+
+        List<Pair<File, File>> rsyncQueue = _pendingRsyncCopy.get(container);
+        if (rsyncQueue != null && !rsyncQueue.isEmpty())
+        {
+            rsyncQueue.forEach(x -> doRsyncCopy(x.getLeft(), x.getRight(), log));
+        }
+        _pendingRsyncCopy.clear();
+    }
+
+    public void queueFileCopy(Container c, File source, File destination)
+    {
+        if (!_pendingFileCopy.containsKey(c))
+        {
+            _pendingFileCopy.put(c, new ArrayList<>());
+        }
+
+        _pendingFileCopy.get(c).add(Pair.of(source, destination));
+    }
+
+    public void queueRsyncCopy(Container c, File source, File destination)
+    {
+        if (!_pendingRsyncCopy.containsKey(c))
+        {
+            _pendingRsyncCopy.put(c, new ArrayList<>());
+        }
+
+        _pendingRsyncCopy.get(c).add(Pair.of(source, destination));
+    }
+
+    private void doRsyncCopy(File sourceDir, File destination, Logger log)
+    {
+        // NOTE: rsync should no-op if there are no source changes
+        log.info("Performing rsync from: " + sourceDir.getPath() + " to " + destination.getPath());
+        try
+        {
+            new SimpleScriptWrapper(log).execute(Arrays.asList(
+                    "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), destination.getPath()
+            ));
+        }
+        catch (PipelineJobException e)
+        {
+            log.error("Error running rsync", e);
+        }
+    }
+
+    private void copyFile(File source, File destination, Logger log)
+    {
+        if (destination.exists())
+        {
+            destination.delete();
+        }
+
+        try
+        {
+            log.info("Copying file: " + source.getPath());
+            FileUtils.copyFile(source, destination);
+        }
+        catch (IOException e)
+        {
+            log.error("Error copying file", e);
+        }
+    }
+}