From 48c43ea256e3919fd682696e4fc64b6e74c198f1 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 5 Apr 2024 13:06:42 -0400 Subject: [PATCH] Make RFS runnable as Container (#550) This changes introduces a Dockerfile for RFS that is linked to the gradle build process such that a user can make changes to RFS code or dependencies and simply execute ./gradlew buildDockerImages from the RFS directory and create a Docker image with the latest changes. A user could alternatively execute ./gradlew composeUp and create a docker environment with source/target/RFS containers using latest changes. Note: The gradle link is in place for docker compose but more work is needed here to limit manual steps a user needs to take to test RFS in this environment. Along with this a few command line arguments were incorporated into RFS Signed-off-by: Tanner Lewis --- RFS/.gitignore | 5 ++ RFS/README.md | 48 ++++++++++++++++- RFS/build.gradle | 51 +++++++++++++++++-- RFS/buildSrc/build.gradle | 27 ++++++++++ RFS/buildSrc/settings.gradle | 7 +++ RFS/docker/Dockerfile | 8 +++ RFS/docker/docker-compose.yml | 38 ++++++++++++++ .../java/com/rfs/ReindexFromSnapshot.java | 30 +++++++++-- .../GlobalMetadataCreator_OS_2_11.java | 8 +-- 9 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 RFS/.gitignore create mode 100644 RFS/buildSrc/build.gradle create mode 100644 RFS/buildSrc/settings.gradle create mode 100644 RFS/docker/Dockerfile create mode 100644 RFS/docker/docker-compose.yml diff --git a/RFS/.gitignore b/RFS/.gitignore new file mode 100644 index 000000000..1b6985c00 --- /dev/null +++ b/RFS/.gitignore @@ -0,0 +1,5 @@ +# Ignore Gradle project-specific cache directory +.gradle + +# Ignore Gradle build output directory +build diff --git a/RFS/README.md b/RFS/README.md index 726ad8a2e..8c5e9fc3f 100644 --- a/RFS/README.md +++ b/RFS/README.md @@ -1,5 +1,20 @@ # reindex-from-snapshot +### Table of Contents (Generated) +- [reindex-from-snapshot](#reindex-from-snapshot) + - [How to run](#how-to-run) + - [Using a local snapshot directory](#using-a-local-snapshot-directory) + - [Using an existing S3 snapshot](#using-an-existing-s3-snapshot) + - [Using a source cluster](#using-a-source-cluster) + - [Using Docker](#using-docker) + - [Handling auth](#handling-auth) + - [How to set up an ES 6.8 Source Cluster w/ an attached debugger](#how-to-set-up-an-es-68-source-cluster-w-an-attached-debugger) + - [How to set up an ES 7.10 Source Cluster running in Docker](#how-to-set-up-an-es-710-source-cluster-running-in-docker) + - [Providing AWS permissions for S3 snapshot creation](#providing-aws-permissions-for-s3-snapshot-creation) + - [Setting up the Cluster w/ some sample docs](#setting-up-the-cluster-w-some-sample-docs) + - [How to set up an OS 2.11 Target Cluster](#how-to-set-up-an-os-211-target-cluster) + + ## How to run RFS provides a number of different options for running it. We'll look at some of them below. @@ -35,7 +50,7 @@ gradle build gradle run --args='-n global_state_snapshot --s3-local-dir /tmp/s3_files --s3-repo-uri $S3_REPO_URI --s3-region $S3_REGION -l /tmp/lucene_files --target-host $TARGET_HOST --target-username $TARGET_USERNAME --target-password $TARGET_PASSWORD -s es_6_8 -t os_2_11 --movement-type everything' ``` -### Using an source cluster +### Using a source cluster In this scenario, you have a source cluster, and don't yet have a snapshot. RFS will need to first make a snapshot of your source cluster, send it to S3, and then begin reindexing. In this scenario, you'll supply the source cluster-related args (`--source-host`, `--source-username`, `--source-password`) and the S3-related args (`--s3-local-dir`, `--s3-repo-uri`, `--s3-region`), but not the `--snapshot-dir` one. @@ -56,6 +71,37 @@ gradle build gradle run --args='-n global_state_snapshot --source-host $SOURCE_HOST --source-username $SOURCE_USERNAME --source-password $SOURCE_PASSWORD --s3-local-dir /tmp/s3_files --s3-repo-uri $S3_REPO_URI --s3-region $S3_REGION -l /tmp/lucene_files --target-host $TARGET_HOST --target-username $TARGET_USERNAME --target-password $TARGET_PASSWORD -s es_6_8 -t os_2_11 --movement-type everything' ``` +### Using Docker +RFS has support for packaging its java application as a Docker image by using the Dockerfile located in the `RFS/docker` directory. This support is directly built into Gradle as well so that a user can perform the below action, and generate a fresh Docker image (`migrations/reindex_from_snapshot:latest`) with the latest local code changes available. +```shell +./gradlew buildDockerImages +``` +Also built into this Docker/Gradle support is the ability to spin up a testing RFS environment using Docker compose. This compose file can be seen [here](./docker/docker-compose.yml) and includes the RFS container, a source cluster container, and a target cluster container. + +This environment can be spun up with the Gradle command +```shell +./gradlew composeUp +``` +And deleted with the Gradle command +```shell +./gradlew composeDown +``` + +After the Docker compose containers are created the elasticsearch/opensearch source and target clusters can be interacted with like normal. For RFS testing, a user should load templates/indices/documents into the source cluster and take a snapshot before kicking off RFS. +```shell +# To check indices on the source cluster +curl https://localhost:19200/_cat/indices?v --insecure -u admin:admin + +# To check indices on the target cluster +curl https://localhost:29200/_cat/indices?v --insecure -u admin:admin +``` + +Once the user is ready, they can kick off the RFS migration by running the RFS java command with the proper arguments on the RFS container like below: +```shell +docker exec -it rfs-compose-reindex-from-snapshot-1 sh -c "/rfs-app/runJavaWithClasspath.sh com.rfs.ReindexFromSnapshot --snapshot-name --snapshot-dir --lucene-dir '/lucene' --target-host https://opensearchtarget:9200 --target-username admin --target-password admin --source-version es_7_10 --target-version os_2_11" +``` + + ### Handling auth RFS currently supports both basic auth (username/password) and no auth for both the source and target clusters. To use the no-auth approach, just neglect the username/password arguments. diff --git a/RFS/build.gradle b/RFS/build.gradle index a59a964af..b61ec74d6 100644 --- a/RFS/build.gradle +++ b/RFS/build.gradle @@ -1,10 +1,14 @@ plugins { id 'application' id 'java' + id "com.avast.gradle.docker-compose" version "0.17.4" + id 'com.bmuschko.docker-remote-api' } -sourceCompatibility = '11' -targetCompatibility = '11' +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage + +java.sourceCompatibility = JavaVersion.VERSION_11 +java.targetCompatibility = JavaVersion.VERSION_11 repositories { mavenCentral() @@ -29,7 +33,44 @@ application { mainClassName = 'com.rfs.ReindexFromSnapshot' } -task demoPrintOutSnapshot(type: JavaExec) { +// Cleanup additional docker build directory +clean.doFirst { + delete project.file("./docker/build") +} + +task demoPrintOutSnapshot (type: JavaExec) { classpath = sourceSets.main.runtimeClasspath - main = 'com.rfs.DemoPrintOutSnapshot' -} \ No newline at end of file + mainClass = 'com.rfs.DemoPrintOutSnapshot' +} + +task copyDockerRuntimeJars (type: Copy) { + description = 'Copy runtime JARs and app jar to docker build directory' + + // Define the destination directory + def buildDir = project.file("./docker/build/runtimeJars") + into buildDir + + // Add all the required runtime JARs to be copied + from configurations.runtimeClasspath + from tasks.named('jar') + include '*.jar' +} + +// ./gradlew composeUp +// ./gradlew composeDown +dockerCompose { + useComposeFiles = ['docker/docker-compose.yml'] + projectName = 'rfs-compose' +} + +// ./gradlew buildDockerImages +task buildDockerImages (type: DockerBuildImage) { + dependsOn copyDockerRuntimeJars + def dockerImageName = "reindex_from_snapshot" + inputDir = project.file("./docker") + images.add("migrations/${dockerImageName}:${version}") + images.add("migrations/${dockerImageName}:latest") +} + +tasks.getByName('composeUp') + .dependsOn(tasks.getByName('buildDockerImages')) \ No newline at end of file diff --git a/RFS/buildSrc/build.gradle b/RFS/buildSrc/build.gradle new file mode 100644 index 000000000..44eb3eee1 --- /dev/null +++ b/RFS/buildSrc/build.gradle @@ -0,0 +1,27 @@ +/* + * This file was generated by the Gradle 'init' task. + */ + +plugins { + // Support convention plugins written in Groovy. Convention plugins are build scripts in 'src/main' that automatically become available as plugins in the main build. + id 'groovy-gradle-plugin' + id 'org.owasp.dependencycheck' version '8.2.1' +} + +repositories { + // Use the plugin portal to apply community plugins in convention plugins. + gradlePluginPortal() + mavenCentral() +} + +dependencies { + implementation 'com.bmuschko:gradle-docker-plugin:9.3.1' +} + +tasks.withType(Tar){ + duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} + +tasks.withType(Zip){ + duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} diff --git a/RFS/buildSrc/settings.gradle b/RFS/buildSrc/settings.gradle new file mode 100644 index 000000000..3f67e420d --- /dev/null +++ b/RFS/buildSrc/settings.gradle @@ -0,0 +1,7 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This settings file is used to specify which projects to include in your build-logic build. + */ + +rootProject.name = 'buildSrc' diff --git a/RFS/docker/Dockerfile b/RFS/docker/Dockerfile new file mode 100644 index 000000000..8e01e5b15 --- /dev/null +++ b/RFS/docker/Dockerfile @@ -0,0 +1,8 @@ +# Using same base image as other Java containers in this repo +FROM openjdk:11-jre +# Requires Gradle to genearte runtime jars initially +COPY ./build/runtimeJars /rfs-app/jars +WORKDIR /rfs-app +RUN printf "#!/bin/sh\njava -XX:MaxRAMPercentage=80.0 -cp /rfs-app/jars/*:. \"\$@\" " > /rfs-app/runJavaWithClasspath.sh +RUN chmod +x /rfs-app/runJavaWithClasspath.sh +CMD ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/RFS/docker/docker-compose.yml b/RFS/docker/docker-compose.yml new file mode 100644 index 000000000..155d553c4 --- /dev/null +++ b/RFS/docker/docker-compose.yml @@ -0,0 +1,38 @@ +version: '3.7' +services: + + # TODO: Sync gradle for RFS and C&R + # Temporarily this requires the user to first build the elasticsearch image from the TrafficCapture gradle project + # directory using a command like ./gradlew buildDockerImages + elasticsearchsource: + image: 'migrations/elasticsearch_searchguard:latest' + networks: + - migrations + environment: + - discovery.type=single-node + ports: + - '19200:9200' + + # TODO: Add example command users can run to kickoff RFS after data is loaded on source + reindex-from-snapshot: + image: 'migrations/reindex_from_snapshot:latest' + depends_on: + elasticsearchsource: + condition: service_started + opensearchtarget: + condition: service_started + networks: + - migrations + + opensearchtarget: + image: 'opensearchproject/opensearch:2.11.1' + environment: + - discovery.type=single-node + networks: + - migrations + ports: + - "29200:9200" + +networks: + migrations: + driver: bridge diff --git a/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java b/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java index 4c456a84d..ecdb76b79 100644 --- a/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java +++ b/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java @@ -3,7 +3,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -68,18 +70,27 @@ public static class Args { @Parameter(names = {"--movement-type"}, description = "What you want to move - everything, metadata, or data. Default: 'everything'", required = false, converter = MovementType.ArgsConverter.class) public MovementType movementType = MovementType.EVERYTHING; + @Parameter(names = {"--template-whitelist"}, description = "List of template names to migrate. Note: For ES 6.8 this refers to legacy templates and for ES 7.10 this is index templates (e.g. 'posts_index_template1, posts_index_template2')", required = false) + public List templateWhitelist; + + @Parameter(names = {"--component-template-whitelist"}, description = "List of component template names to migrate (e.g. 'posts_template1, posts_template2')", required = false) + public List componentTemplateWhitelist; + + @Parameter(names = {"--enable-persistent-run"}, description = "If enabled, the java process will continue in an idle mode after the migration is completed. Default: false", arity=0, required = false) + public boolean enablePersistentRun; + @Parameter(names = {"--log-level"}, description = "What log level you want. Default: 'info'", required = false, converter = Logging.ArgsConverter.class) public Level logLevel = Level.INFO; } - public static void main(String[] args) { + public static void main(String[] args) throws InterruptedException { // Grab out args Args arguments = new Args(); JCommander.newBuilder() .addObject(arguments) .build() .parse(args); - + String snapshotName = arguments.snapshotName; Path snapshotDirPath = (arguments.snapshotDirPath != null) ? Paths.get(arguments.snapshotDirPath) : null; Path s3LocalDirPath = (arguments.s3LocalDirPath != null) ? Paths.get(arguments.s3LocalDirPath) : null; @@ -94,6 +105,8 @@ public static void main(String[] args) { String targetPass = arguments.targetPass; ClusterVersion sourceVersion = arguments.sourceVersion; ClusterVersion targetVersion = arguments.targetVersion; + List templateWhitelist = arguments.templateWhitelist; + List componentTemplateWhitelist = arguments.componentTemplateWhitelist; MovementType movementType = arguments.movementType; Level logLevel = arguments.logLevel; @@ -103,8 +116,6 @@ public static void main(String[] args) { ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass); // Should probably be passed in as an arguments - String[] templateWhitelist = {"posts_index_template"}; - String[] componentTemplateWhitelist = {"posts_template"}; int awarenessAttributeDimensionality = 3; // https://opensearch.org/docs/2.11/api-reference/cluster-api/cluster-awareness/ // Sanity checks @@ -235,7 +246,7 @@ public static void main(String[] args) { if (sourceVersion == ClusterVersion.ES_6_8) { ObjectNode root = globalMetadata.toObjectNode(); ObjectNode transformedRoot = transformer.transformGlobalMetadata(root); - GlobalMetadataCreator_OS_2_11.create(transformedRoot, targetConnection, new String[0], templateWhitelist); + GlobalMetadataCreator_OS_2_11.create(transformedRoot, targetConnection, Collections.emptyList(), templateWhitelist); } else if (sourceVersion == ClusterVersion.ES_7_10) { ObjectNode root = globalMetadata.toObjectNode(); ObjectNode transformedRoot = transformer.transformGlobalMetadata(root); @@ -338,5 +349,14 @@ public static void main(String[] args) { } catch (Exception e) { e.printStackTrace(); } + + // Optional temporary persistent runtime flag to continue Java process after steps have completed. This should get + // replaced as this app develops and becomes aware of determining work to be completed + if (arguments.enablePersistentRun) { + while (true) { + logger.info("Process is in idle mode, to retry migration please restart this app."); + Thread.sleep(TimeUnit.MINUTES.toMillis(5)); + } + } } } diff --git a/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataCreator_OS_2_11.java b/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataCreator_OS_2_11.java index 16921eb4f..9216a4f60 100644 --- a/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataCreator_OS_2_11.java +++ b/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataCreator_OS_2_11.java @@ -14,7 +14,7 @@ public class GlobalMetadataCreator_OS_2_11 { private static final Logger logger = LogManager.getLogger(GlobalMetadataCreator_OS_2_11.class); - public static void create(ObjectNode root, ConnectionDetails connectionDetails, String[] componentTemplateWhitelist, String[] indexTemplateWhitelist) throws Exception { + public static void create(ObjectNode root, ConnectionDetails connectionDetails, List componentTemplateWhitelist, List indexTemplateWhitelist) throws Exception { logger.info("Setting Global Metadata"); GlobalMetadataData_OS_2_11 globalMetadata = new GlobalMetadataData_OS_2_11(root); @@ -23,7 +23,7 @@ public static void create(ObjectNode root, ConnectionDetails connectionDetails, createIndexTemplates(globalMetadata, connectionDetails, indexTemplateWhitelist); } - public static void createTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, String[] indexTemplateWhitelist) throws Exception { + public static void createTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, List indexTemplateWhitelist) throws Exception { logger.info("Setting Legacy Templates"); ObjectNode templates = globalMetadata.getTemplates(); @@ -59,7 +59,7 @@ public static void createTemplates(GlobalMetadataData_OS_2_11 globalMetadata, Co } } - public static void createComponentTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, String[] indexTemplateWhitelist) throws Exception { + public static void createComponentTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, List indexTemplateWhitelist) throws Exception { logger.info("Setting Component Templates"); ObjectNode templates = globalMetadata.getComponentTemplates(); @@ -95,7 +95,7 @@ public static void createComponentTemplates(GlobalMetadataData_OS_2_11 globalMet } } - public static void createIndexTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, String[] indexTemplateWhitelist) throws Exception { + public static void createIndexTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, List indexTemplateWhitelist) throws Exception { logger.info("Setting Index Templates"); ObjectNode templates = globalMetadata.getIndexTemplates();