From ade16d7d32e7646c7225edb768bed9eec154a56a Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 12 Feb 2024 11:22:07 -0800 Subject: [PATCH 1/3] Support extracting from embedded build for remote pipeline scenarios --- .../labkey/embedded/EmbeddedExtractor.java | 149 ++++++++++++++++++ .../src/org/labkey/embedded/LabKeyServer.java | 8 + .../LabKeyTomcatServletWebServerFactory.java | 145 ++--------------- 3 files changed, 166 insertions(+), 136 deletions(-) create mode 100644 server/embedded/src/org/labkey/embedded/EmbeddedExtractor.java diff --git a/server/embedded/src/org/labkey/embedded/EmbeddedExtractor.java b/server/embedded/src/org/labkey/embedded/EmbeddedExtractor.java new file mode 100644 index 0000000000..2918f582b2 --- /dev/null +++ b/server/embedded/src/org/labkey/embedded/EmbeddedExtractor.java @@ -0,0 +1,149 @@ +package org.labkey.embedded; + +import org.labkey.bootstrap.ConfigException; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.jar.JarFile; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class EmbeddedExtractor +{ + private static final int BUFFER_SIZE = 1024 * 64; + + private void extractExecutableJar(File jarFilePath, File destDirectory, boolean remotePipeline) + { + try + { + try (JarFile jar = new JarFile(jarFilePath)) + { + boolean foundDistributionZip = false; + var entries = jar.entries(); + while (entries.hasMoreElements()) + { + var entry = entries.nextElement(); + var entryName = entry.getName(); + + if ("labkey/distribution.zip".equals(entryName)) + { + foundDistributionZip = true; + try (var distInputStream = jar.getInputStream(entry)) + { + extractZip(distInputStream, destDirectory); + } + } + if (remotePipeline) + { + if (entry.getName().contains("labkeyBootstrap") && entry.getName().toLowerCase().endsWith(".jar")) + { + try (var in = jar.getInputStream(entry)) + { + extractFile(in, new File(destDirectory, "labkeyBootstrap.jar")); + } + } + if (entry.getName().contains("tomcat-servlet-api") && entry.getName().toLowerCase().endsWith(".jar")) + { + File pipelineLib = new File(destDirectory, "pipeline-lib"); + if (!pipelineLib.exists()) + { + if (!pipelineLib.mkdirs()) + { + throw new ConfigException("Failed to create directory " + pipelineLib + " Please check file system permissions"); + } + } + try (var in = jar.getInputStream(entry)) + { + extractFile(in, new File(pipelineLib, "servletApi.jar")); + } + } + } + } + +// if (!foundDistributionZip) +// { +// throw new ConfigException("Unable to find distribution zip required to run LabKey Server."); +// } + } + } + catch (IOException | ConfigException e) + { + throw new RuntimeException(e); + } + } + + public void extractExecutableJarFromDir(File currentDir, File destDir, boolean remotePipeline) throws ConfigException + { + File[] files = currentDir.listFiles(file -> { + String name = file.getName().toLowerCase(); + return name.endsWith(".jar") && !name.contains("labkeybootstrap"); + }); + + if (files == null) + { + throw new ConfigException("Executable jar not found."); + } + + // only 1 jar should be there + if (files.length == 1) + { + extractExecutableJar(files[0], destDir, remotePipeline); + } + else + { + throw new ConfigException("Multiple jars found - " + Arrays.asList(files) + ". Must provide only one jar."); + } + } + + private void extractZip(InputStream zipInputStream, File destDir) throws IOException + { + //noinspection SSBasedInspection + if (!destDir.exists() && !destDir.mkdirs()) + { + throw new IOException("Failed to create directory " + destDir + " - please check file system permissions"); + } + try (ZipInputStream zipIn = new ZipInputStream(zipInputStream)) + { + ZipEntry entry = zipIn.getNextEntry(); + // iterates over entries in the zip file + while (entry != null) + { + File filePath = new File(destDir, entry.getName()); + if (!entry.isDirectory()) + { + // if the entry is a file, extracts it + extractFile(zipIn, filePath); + } + else + { + // if the entry is a directory, make the directory + //noinspection SSBasedInspection + if (!filePath.exists() && !filePath.mkdirs()) + { + throw new IOException("Failed to create directory " + filePath + " - please check file system permissions"); + } + } + zipIn.closeEntry(); + entry = zipIn.getNextEntry(); + } + } + } + + private static void extractFile(InputStream zipIn, File filePath) throws IOException + { + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) + { + byte[] bytesIn = new byte[BUFFER_SIZE]; + int read; + while ((read = zipIn.read(bytesIn)) != -1) + { + bos.write(bytesIn, 0, read); + } + } + } + +} diff --git a/server/embedded/src/org/labkey/embedded/LabKeyServer.java b/server/embedded/src/org/labkey/embedded/LabKeyServer.java index 08662804ab..7d5b6ba4a1 100644 --- a/server/embedded/src/org/labkey/embedded/LabKeyServer.java +++ b/server/embedded/src/org/labkey/embedded/LabKeyServer.java @@ -14,6 +14,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.validation.annotation.Validated; +import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -34,6 +35,13 @@ public class LabKeyServer public static void main(String[] args) { + if (args.length > 0 && args[0].equalsIgnoreCase("pipeline")) + { + File currentDir = new File("").getAbsoluteFile(); + new EmbeddedExtractor().extractExecutableJarFromDir(currentDir, currentDir, true); + return; + } + // Issue 40038: Ride-or-die Mode - default to shutting down by default in embedded deployment scenario if (System.getProperty(TERMINATE_ON_STARTUP_FAILURE) == null) { diff --git a/server/embedded/src/org/labkey/embedded/LabKeyTomcatServletWebServerFactory.java b/server/embedded/src/org/labkey/embedded/LabKeyTomcatServletWebServerFactory.java index e4521e8990..55b86f2384 100644 --- a/server/embedded/src/org/labkey/embedded/LabKeyTomcatServletWebServerFactory.java +++ b/server/embedded/src/org/labkey/embedded/LabKeyTomcatServletWebServerFactory.java @@ -14,19 +14,10 @@ import org.springframework.boot.web.servlet.ServletContextInitializer; import javax.sql.DataSource; -import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.jar.JarFile; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import static org.labkey.embedded.LabKeyServer.SERVER_GUID_PARAMETER_NAME; @@ -35,8 +26,6 @@ class LabKeyTomcatServletWebServerFactory extends TomcatServletWebServerFactory private static final Log LOG = LogFactory.getLog(LabKeyTomcatServletWebServerFactory.class); private final LabKeyServer _server; - private static final int BUFFER_SIZE = 4096; - public LabKeyTomcatServletWebServerFactory(LabKeyServer server) { _server = server; @@ -69,26 +58,25 @@ protected TomcatWebServer getTomcatWebServer(Tomcat tomcat) // for development, point to the local deploy/labkeyWebapp directory in configs/application.properties boolean webAppLocationPresent = contextProperties.getWebAppLocation() != null; - var webAppLocation = ""; + File webAppLocation; try { if (!webAppLocationPresent) { - final var currentPath = new File("").getAbsolutePath(); - var destDirectory = currentPath + "/server"; - webAppLocation = destDirectory + "/labkeywebapp"; - boolean extracted = new File(webAppLocation).exists(); - String jarFilePath = getExecutableJar(currentPath); + final var currentPath = new File(""); + var destDirectory = new File(currentPath, "server"); + webAppLocation = new File(destDirectory, "labkeywebapp"); - if (!extracted) + if (!webAppLocation.exists()) { - extractExecutableJar(destDirectory, jarFilePath); + EmbeddedExtractor extractor = new EmbeddedExtractor(); + extractor.extractExecutableJarFromDir(currentPath, destDirectory, false); } } else { - webAppLocation = contextProperties.getWebAppLocation(); + webAppLocation = new File(contextProperties.getWebAppLocation()); } // Turn off the default web.xml behavior so that we don't stomp over customized values @@ -103,7 +91,7 @@ protected TomcatWebServer getTomcatWebServer(Tomcat tomcat) // Spring Boot's webapp is being deployed to the root. We have to deploy elsewhere in this initial // call, but can immediately swap it with the desired place - StandardContext context = (StandardContext) tomcat.addWebapp("/labkey", webAppLocation); + StandardContext context = (StandardContext) tomcat.addWebapp("/labkey", webAppLocation.getAbsolutePath()); // set the root path to the context explicitly context.setPath(contextProperties.getContextPath()); @@ -368,119 +356,4 @@ private ContextResource getMailResource() return mailResource; } - - private void extractExecutableJar(String destDirectory, String jarFilePath) - { - try - { - try (JarFile jar = new JarFile(jarFilePath)) - { - boolean foundDistributionZip = false; - var entries = jar.entries(); - while (entries.hasMoreElements()) - { - var entry = entries.nextElement(); - var entryName = entry.getName(); - - if ("labkey/distribution.zip".equals(entryName)) - { - foundDistributionZip = true; - try (var distInputStream = jar.getInputStream(entry)) - { - extractZip(distInputStream, destDirectory); - } - } - } - - if (!foundDistributionZip) - { - throw new ConfigException("Unable to find distribution zip required to run LabKey server."); - } - } - } - catch (IOException | ConfigException e) - { - throw new RuntimeException(e); - } - } - - private static String getExecutableJar(String currentPath) throws ConfigException - { - File currentDir = new File(currentPath); - List jarsPresent = new ArrayList<>(); - - File[] files = currentDir.listFiles(); - if (files != null) - { - for (File file : files) - { - if (file.getName().toLowerCase().endsWith(".jar")) - { - jarsPresent.add(file.getName()); - } - } - } - - if (jarsPresent.isEmpty()) - { - throw new ConfigException("Executable jar not found."); - } - - // only 1 jar should be there - if (jarsPresent.size() == 1) - { - return jarsPresent.get(0); - } - - throw new ConfigException("Multiple jars found - " + jarsPresent + ". Must provide only one jar."); - } - - private void extractZip(InputStream zipInputStream, String destDirectory) throws IOException - { - File destDir = new File(destDirectory); - //noinspection SSBasedInspection - if (!destDir.exists() && !destDir.mkdirs()) - { - throw new IOException("Failed to create directory " + destDir + " - please check file system permissions"); - } - try (ZipInputStream zipIn = new ZipInputStream(zipInputStream)) - { - ZipEntry entry = zipIn.getNextEntry(); - // iterates over entries in the zip file - while (entry != null) - { - String filePath = destDirectory + File.separator + entry.getName(); - if (!entry.isDirectory()) - { - // if the entry is a file, extracts it - extractFile(zipIn, filePath); - } - else - { - // if the entry is a directory, make the directory - File dir = new File(filePath); - //noinspection SSBasedInspection - if (!dir.exists() && !dir.mkdirs()) - { - throw new IOException("Failed to create directory " + dir + " - please check file system permissions"); - } - } - zipIn.closeEntry(); - entry = zipIn.getNextEntry(); - } - } - } - - private static void extractFile(ZipInputStream zipIn, String filePath) throws IOException - { - try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) - { - byte[] bytesIn = new byte[BUFFER_SIZE]; - int read; - while ((read = zipIn.read(bytesIn)) != -1) - { - bos.write(bytesIn, 0, read); - } - } - } } From 550fa9484ee0aaddf3b948c90f954e9122e42a41 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Tue, 13 Feb 2024 18:28:09 -0800 Subject: [PATCH 2/3] Change command-line arg --- server/embedded/src/org/labkey/embedded/LabKeyServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/embedded/src/org/labkey/embedded/LabKeyServer.java b/server/embedded/src/org/labkey/embedded/LabKeyServer.java index 7d5b6ba4a1..e79bddd7df 100644 --- a/server/embedded/src/org/labkey/embedded/LabKeyServer.java +++ b/server/embedded/src/org/labkey/embedded/LabKeyServer.java @@ -35,7 +35,7 @@ public class LabKeyServer public static void main(String[] args) { - if (args.length > 0 && args[0].equalsIgnoreCase("pipeline")) + if (args.length > 0 && args[0].equalsIgnoreCase("-extract")) { File currentDir = new File("").getAbsoluteFile(); new EmbeddedExtractor().extractExecutableJarFromDir(currentDir, currentDir, true); From 2321ce35d56587c666e016bc276cc1466a3f5d4c Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 19 Feb 2024 15:44:27 -0800 Subject: [PATCH 3/3] Fail when we don't find distribution.zip --- .../src/org/labkey/embedded/EmbeddedExtractor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/embedded/src/org/labkey/embedded/EmbeddedExtractor.java b/server/embedded/src/org/labkey/embedded/EmbeddedExtractor.java index 2918f582b2..316fef227a 100644 --- a/server/embedded/src/org/labkey/embedded/EmbeddedExtractor.java +++ b/server/embedded/src/org/labkey/embedded/EmbeddedExtractor.java @@ -64,10 +64,10 @@ private void extractExecutableJar(File jarFilePath, File destDirectory, boolean } } -// if (!foundDistributionZip) -// { -// throw new ConfigException("Unable to find distribution zip required to run LabKey Server."); -// } + if (!foundDistributionZip) + { + throw new ConfigException("Unable to find distribution zip required to run LabKey Server."); + } } } catch (IOException | ConfigException e)