From 79720b9a96dec5cec97ce78c9d3a430bd65a097d Mon Sep 17 00:00:00 2001 From: Trey Chadick Date: Tue, 4 Feb 2025 13:27:42 -0800 Subject: [PATCH] Stress test framework improvements (#2239) --- src/org/labkey/test/WebTestHelper.java | 18 ++++++- .../components/ui/grids/GridFilterModal.java | 6 +++ .../components/ui/search/SampleFinder.java | 2 + .../labkey/test/stress/ActivityRecorder.java | 49 +++++++++++++++++++ src/org/labkey/test/stress/HarConverter.java | 4 +- .../test/stress/RecentRequestsCollector.java | 29 +++++++++++ src/org/labkey/test/stress/Simulation.java | 33 +++++++++---- src/org/labkey/test/util/APIAssayHelper.java | 28 +++++++++++ .../util/perf/JsonPerfScenarioHelper.java | 49 ++++++++++++++++--- 9 files changed, 196 insertions(+), 22 deletions(-) create mode 100644 src/org/labkey/test/stress/ActivityRecorder.java create mode 100644 src/org/labkey/test/stress/RecentRequestsCollector.java diff --git a/src/org/labkey/test/WebTestHelper.java b/src/org/labkey/test/WebTestHelper.java index ba27a62b07..c9b42319e3 100644 --- a/src/org/labkey/test/WebTestHelper.java +++ b/src/org/labkey/test/WebTestHelper.java @@ -42,12 +42,14 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.json.JSONObject; +import org.junit.Assert; import org.labkey.remoteapi.CommandException; import org.labkey.remoteapi.CommandResponse; import org.labkey.remoteapi.Connection; import org.labkey.remoteapi.SimplePostCommand; import org.labkey.remoteapi.query.DeleteRowsCommand; import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SaveRowsResponse; import org.labkey.remoteapi.query.SelectRowsCommand; import org.labkey.serverapi.reader.Readers; import org.labkey.test.util.InstallCert; @@ -82,10 +84,12 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Random; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -113,6 +117,7 @@ public class WebTestHelper private static final Map> savedCookies = new HashMap<>(); private static final Map savedSessionKeys = new HashMap<>(); private static final Map savedApiKeys = new HashMap<>(); + private static final Set deletedApiKeys = new HashSet<>(); static { TestProperties.load(); } @@ -195,8 +200,10 @@ public static void deleteApiKey(Connection connection, String apiKey) { DeleteRowsCommand deleteRowsCommand = new DeleteRowsCommand("core", "apiKeys"); deleteRowsCommand.setRows(rows); - deleteRowsCommand.execute(connection, null); + SaveRowsResponse response = deleteRowsCommand.execute(connection, null); savedApiKeys.remove(apiKey); + deletedApiKeys.add(apiKey); + Assert.assertEquals("Wrong number of rows affected by apiKey deletion", 1, response.getRowsAffected()); } else { @@ -210,7 +217,14 @@ public static void deleteApiKey(Connection connection, String apiKey) } else { - TestLogger.warn("Refusing to delete an API key not created by this test"); + if (deletedApiKeys.contains(apiKey)) + { + TestLogger.warn("API key already deleted"); + } + else + { + TestLogger.warn("Refusing to delete an API key not created by 'WebTestHelper.createApiKey'"); + } } } diff --git a/src/org/labkey/test/components/ui/grids/GridFilterModal.java b/src/org/labkey/test/components/ui/grids/GridFilterModal.java index d90d2f09de..67d119c9f4 100644 --- a/src/org/labkey/test/components/ui/grids/GridFilterModal.java +++ b/src/org/labkey/test/components/ui/grids/GridFilterModal.java @@ -89,6 +89,12 @@ public FilterExpressionPanel selectExpressionTab() return elementCache().filterExpressionPanel(); } + public GridFilterModal setFilter(FilterExpressionPanel.Expression expression) + { + selectExpressionTab().setFilter(expression); + return this; + } + /** * Select the facet tab for the current field. Will throw NoSuchElementException if tab isn't present. * @return panel for configuring faceted filter diff --git a/src/org/labkey/test/components/ui/search/SampleFinder.java b/src/org/labkey/test/components/ui/search/SampleFinder.java index 2a803d413f..61d8adfeca 100644 --- a/src/org/labkey/test/components/ui/search/SampleFinder.java +++ b/src/org/labkey/test/components/ui/search/SampleFinder.java @@ -27,6 +27,8 @@ */ public class SampleFinder extends WebDriverComponent { + public static final String ALL_SAMPLE_TYPES = "All Sample Types"; + private final WebElement _el; private final WebDriver _driver; diff --git a/src/org/labkey/test/stress/ActivityRecorder.java b/src/org/labkey/test/stress/ActivityRecorder.java new file mode 100644 index 0000000000..7d407637ef --- /dev/null +++ b/src/org/labkey/test/stress/ActivityRecorder.java @@ -0,0 +1,49 @@ +package org.labkey.test.stress; + +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.miniprofiler.RequestInfo; +import org.labkey.test.util.LogMethod; +import org.labkey.test.util.LoggedParam; +import org.labkey.test.util.TestLogger; + +import java.io.IOException; +import java.util.List; +import java.util.function.Supplier; + +public class ActivityRecorder +{ + private final RecentRequestsCollector recentRequestsCollector; + + public ActivityRecorder(Connection connection) throws IOException, CommandException + { + recentRequestsCollector = new RecentRequestsCollector(connection); + } + + @LogMethod + public R recordActivity(@LoggedParam String description, Supplier activity) throws IOException, CommandException + { + R result = activity.get(); + + List recentRequests = recentRequestsCollector.getRecentRequests(); + TestLogger.log("%s triggered %s requests".formatted(description, recentRequests.size())); + + return result; + } + + public void recordActivity(String description, Runnable activity) throws IOException, CommandException + { + recordActivity(description, () -> { + activity.run(); + return null; + }); + } + + public List skipRecentRequests() throws IOException, CommandException + { + List recentRequests = recentRequestsCollector.getRecentRequests(); + TestLogger.log("Skipping %s requests".formatted(recentRequests.size())); + + return recentRequests; + } +} diff --git a/src/org/labkey/test/stress/HarConverter.java b/src/org/labkey/test/stress/HarConverter.java index f12dbb92de..6acb6cd313 100644 --- a/src/org/labkey/test/stress/HarConverter.java +++ b/src/org/labkey/test/stress/HarConverter.java @@ -73,7 +73,7 @@ public class HarConverter { private static final Logger LOG = LogManager.getLogger(HarConverter.class); - private static final Set excludedActions = Set.of(new ControllerActionId("login", "whoami")); + public static final Set EXCLUDED_ACTIONS = Set.of(new ControllerActionId("login", "whoami")); private final String inputParam; @@ -213,7 +213,7 @@ private boolean shouldIncludeHarEntry(JSONObject entry) try { ControllerActionId actionId = new ControllerActionId(url); - if (excludedActions.contains(actionId) || StringUtils.isBlank(actionId.getAction()) || "app".equals(actionId.getAction())) + if (EXCLUDED_ACTIONS.contains(actionId) || StringUtils.isBlank(actionId.getAction()) || "app".equals(actionId.getAction())) { LOG.info("Skipping request: " + url); return false; diff --git a/src/org/labkey/test/stress/RecentRequestsCollector.java b/src/org/labkey/test/stress/RecentRequestsCollector.java new file mode 100644 index 0000000000..81a431a098 --- /dev/null +++ b/src/org/labkey/test/stress/RecentRequestsCollector.java @@ -0,0 +1,29 @@ +package org.labkey.test.stress; + +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.miniprofiler.RecentRequestsCommand; +import org.labkey.remoteapi.miniprofiler.RequestInfo; +import org.labkey.test.util.Crawler; + +import java.io.IOException; +import java.util.List; + +public class RecentRequestsCollector +{ + private final Connection _connection; + private long lastRequestId = 0L; + + public RecentRequestsCollector(Connection connection) throws IOException, CommandException + { + _connection = connection; + getRecentRequests(); // Prime 'lastRequestId' + } + + public List getRecentRequests() throws IOException, CommandException + { + List requestInfos = new RecentRequestsCommand(lastRequestId).execute(_connection, null).getRequestInfos(); + lastRequestId = requestInfos.get(requestInfos.size() - 1).getId(); + return requestInfos.stream().filter(requestInfo -> HarConverter.EXCLUDED_ACTIONS.contains(new Crawler.ControllerActionId(requestInfo.getUrl()))).toList(); + } +} diff --git a/src/org/labkey/test/stress/Simulation.java b/src/org/labkey/test/stress/Simulation.java index f18672e71d..c1907d51ae 100644 --- a/src/org/labkey/test/stress/Simulation.java +++ b/src/org/labkey/test/stress/Simulation.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -35,7 +36,6 @@ import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; /** * Simulation: A series of activities that represents some user workflow (e.g. loading the dashboard, navigating to the sample finder, and performing a search). For simplicity, API simulations in the initial proof of concept will consist of a single activity. @@ -230,7 +230,10 @@ private void makeRequest(Activity.RequestParams requestParams) throws Interrupte * {@link #_connectionFactory} - used to generate an API connection for the simulation to use * *
  • - * {@link #activityDefinitions} - {@link Activity} list that defines the simulation. These are deserialized from {@link ApiTestsDocument} XML files. + * {@link #activityFiles} - {@link ApiTestsDocument} XML files that define the simulation. Used to generate {@link Activity} definitions. + *
    • + *
  • + * {@link #replacements} - String replacements to inject into activity definitions. e.g. 'CONTAINER' or 'USERID' *
    • *
  • * {@link #maxActivityThreads} - the size of thread pool to use for requests @@ -248,7 +251,8 @@ public static class Definition { private final Supplier _connectionFactory; - private List activityDefinitions = Collections.emptyList(); + private List activityFiles = new ArrayList<>(); + private Map replacements = Collections.emptyMap(); private int maxActivityThreads = 6; // This seems to be the number of parallel requests browsers handle private int delayBetweenActivities = 5_000; private boolean runOnce = false; @@ -287,15 +291,13 @@ public Definition setDelayBetweenActivities(int delayBetweenActivities) public Definition setActivityFiles(File... activityFiles) { - return setActivityFilesWithReplacements( - Arrays.stream(activityFiles).collect(Collectors.toMap(f -> f, f -> Collections.emptyMap()))); + this.activityFiles = Arrays.asList(activityFiles); + return this; } - public Definition setActivityFilesWithReplacements(Map> activityFilesWithReplacements) + public Definition setReplacements(Map replacements) { - activityDefinitions = activityFilesWithReplacements.entrySet().stream() - .map(entry -> new Activity(entry.getKey().getName(), Definition.parseTests(entry.getKey(), entry.getValue()))) - .toList(); + this.replacements = new HashMap<>(replacements); return this; } @@ -305,6 +307,17 @@ public Definition setRunOnce(boolean runOnce) return this; } + private List buildActivityDefinitions() + { + if (activityFiles.isEmpty()) + { + throw new IllegalArgumentException("No activity files specified"); + } + return activityFiles.stream() + .map(file -> new Activity(file.getName(), Definition.parseTests(file, replacements))) + .toList(); + } + /** * Start the simulation according to this definition * @param resultCollectorFactory The simulation will submit results to the supplied {@link ResultCollector} @@ -316,7 +329,7 @@ public Simulation startSimulation(Function Connection connection = _connectionFactory.get(); // Prime connection before starting simulation to ensure credentials are good new WhoAmICommand().execute(connection, null); - return new Simulation<>(connection, activityDefinitions, delayBetweenActivities, maxActivityThreads, resultCollectorFactory.apply(connection), runOnce); + return new Simulation<>(connection, buildActivityDefinitions(), delayBetweenActivities, maxActivityThreads, resultCollectorFactory.apply(connection), runOnce); } public Simulation startSimulation() throws IOException, CommandException diff --git a/src/org/labkey/test/util/APIAssayHelper.java b/src/org/labkey/test/util/APIAssayHelper.java index 3c305c4046..432c7a68d1 100644 --- a/src/org/labkey/test/util/APIAssayHelper.java +++ b/src/org/labkey/test/util/APIAssayHelper.java @@ -251,6 +251,34 @@ public static List getListOfAssayNames(String containerPath) throws IOEx return resultData; } + /** + * For a given container get rowIds for all assay protocols. + * + * @param containerPath Container path. + * @param connection remoteApi connection + * @return Map of assay rowIds by protocol name and assay type + * @throws IOException Can be thrown by the SelectRowsCommand. + * @throws CommandException Can be thrown by the SelectRowsCommand. + */ + public static Map getProtocolIds(String containerPath, Connection connection) throws IOException, CommandException + { + SelectRowsCommand cmd = new SelectRowsCommand("assay", "AssayList"); + cmd.setColumns(Arrays.asList("Name", "Type", "RowId")); + + Map resultData = new HashMap<>(); + + SelectRowsResponse response = cmd.execute(connection, containerPath); + for(Row row : response.getRowset()) + { + String type = (String) row.getValue("Type"); + String name = (String) row.getValue("Name"); + Integer rowId = (Integer) row.getValue("RowId"); + resultData.put(type + "." + name, rowId); + } + + return resultData; + } + public void saveBatch(String assayName, String runName, Map runProperties, List> resultRows, String projectName) throws IOException, CommandException { int assayId = getIdFromAssayName(assayName, projectName); diff --git a/src/org/labkey/test/util/perf/JsonPerfScenarioHelper.java b/src/org/labkey/test/util/perf/JsonPerfScenarioHelper.java index 425228a2ae..b38308ee91 100644 --- a/src/org/labkey/test/util/perf/JsonPerfScenarioHelper.java +++ b/src/org/labkey/test/util/perf/JsonPerfScenarioHelper.java @@ -1,7 +1,13 @@ package org.labkey.test.util.perf; +import org.labkey.remoteapi.Command; import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.CommandResponse; import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.PostCommand; +import org.labkey.remoteapi.ResponseObject; +import org.labkey.remoteapi.assay.ImportRunCommand; +import org.labkey.remoteapi.assay.ImportRunResponse; import org.labkey.remoteapi.miniprofiler.RequestInfo; import org.labkey.remoteapi.query.ImportDataCommand; import org.labkey.remoteapi.query.ImportDataResponse; @@ -9,6 +15,7 @@ import org.labkey.test.params.perf.PerfScenario; import org.labkey.test.stress.AbstractScenario; import org.labkey.test.stress.RequestInfoTsvWriter; +import org.labkey.test.util.APIAssayHelper; import org.labkey.test.util.LogMethod; import org.labkey.test.util.TestDateUtils; import org.labkey.test.util.TestLogger; @@ -35,14 +42,16 @@ public class JsonPerfScenarioHelper private final String containerPath; private final Connection connection; private final Function perfDataFileSupplier; + private final Map assayIds; private Function resultHandler = Function.identity(); private int importThreads = 3; - public JsonPerfScenarioHelper(String containerPath, Connection connection, File perfDataDir) + public JsonPerfScenarioHelper(String containerPath, Connection connection, File perfDataDir) throws IOException, CommandException { this.containerPath = containerPath; this.connection = connection; this.perfDataFileSupplier = perfDataDir == null ? TestFileUtils::getSampleData : name -> new File(perfDataDir, name); + this.assayIds = APIAssayHelper.getProtocolIds(containerPath, connection); } public JsonPerfScenarioHelper setResultHandler(Function resultHandler) @@ -72,7 +81,10 @@ public Map runPerfScenarios(List scenarios) throws expectedDuration += perfScenario.getAverage(); } importExecutor.shutdown(); - importExecutor.awaitTermination(expectedDuration, TimeUnit.MILLISECONDS); + if (!importExecutor.awaitTermination(expectedDuration, TimeUnit.MILLISECONDS)) + { + TestLogger.warn("Import timed out"); + } Map results = new LinkedHashMap<>(); for (Map.Entry> entry : futures.entrySet()) { @@ -92,18 +104,39 @@ private Result startImport(PerfScenario perfScenario) String schemaName = SCHEMA_NAMES.get(perfScenario.getType()); if (schemaName == null) { - return null; // unsupported type + throw new IllegalArgumentException("Unsupported scenario type '%s' in '%s'".formatted(perfScenario.getType(), perfScenario.getName())); + } + + PostCommand command; + if (schemaName.startsWith("assay.")) + { + String key = schemaName.split("\\.", 2)[1] + "." + perfScenario.getTypeName(); + Integer assayId = assayIds.get(key); + if (assayId == null) + { + throw new IllegalArgumentException("'%s' not defined in '%s'. Found %s".formatted(key, containerPath, assayIds.keySet())); + } + command = new ImportRunCommand(assayId, perfDataFileSupplier.apply(perfScenario.getFileName())); } - ImportDataCommand command = new ImportDataCommand(schemaName, perfScenario.getTypeName()); - command.setFile(perfDataFileSupplier.apply(perfScenario.getFileName())); + else + { + ImportDataCommand importDataCommand = new ImportDataCommand(schemaName, perfScenario.getTypeName()); + importDataCommand.setFile(perfDataFileSupplier.apply(perfScenario.getFileName())); + importDataCommand.setInsertOption(ImportDataCommand.InsertOption.MERGE); + + command = importDataCommand; + } + command.setTimeout(Math.max(connection.getTimeout(), perfScenario.getAverage() * importThreads) * 2); - command.setInsertOption(ImportDataCommand.InsertOption.MERGE); Timer timer = new Timer(); String msgSuffix = ""; try { - ImportDataResponse response = command.execute(connection, containerPath); - msgSuffix = "Imported %d rows".formatted(response.getRowCount()); + CommandResponse response = command.execute(connection, containerPath); + if (response instanceof ImportDataResponse idr) + { + msgSuffix = "Imported %d rows".formatted(idr.getRowCount()); + } return new Result(perfScenario, response.getStatusCode(), timer); } catch (IOException e)