Set initial admin password

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

.github/actions/start-opensearch/action.yml

@@ -34,7 +34,12 @@ runs:
 
         if [[ -d "$OPENSEARCH_HOME/plugins/opensearch-security" ]]; then
           if [[ "$SECURED" == "true" ]]; then
-            bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s
+            DEMO_CONFIG_SCRIPT="$OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh"
+            if grep --quiet -i org.opensearch.security.tools.democonfig.Installer "$DEMO_CONFIG_SCRIPT"; then
+              OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin bash "$DEMO_CONFIG_SCRIPT" -y -i -s -t
+            else
+              bash "$DEMO_CONFIG_SCRIPT" -y -i -s
+            fi
             sed -i.bak -e 's/plugins.security.audit.type:.*/plugins.security.audit.type: log4j/' $OPENSEARCH_HOME/config/opensearch.yml
             cp ./client/.ci/opensearch/*.pem $OPENSEARCH_HOME/config/
             url="https://localhost:9200"
@@ -64,7 +69,7 @@ runs:
 
         for attempt in {1..20}; do
           sleep 5
-          if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD $url; then
+          if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:admin $url; then
             echo '=====> ready'
             exit 0
           fi
@@ -74,4 +79,3 @@ runs:
       env:
         SECURED: ${{ inputs.secured }}
         RUNNER_OS: ${{ runner.os }}
-        OPENSEARCH_INITIAL_ADMIN_PASSWORD: myStrongPassword123!

.github/workflows/integration.yml

@@ -59,7 +59,6 @@ jobs:
           path: client/build/output/*
 
   integration-opensearch-unreleased:
-#    if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268)
     name: Integration OpenSearch Unreleased
     runs-on: ubuntu-latest
     strategy:

abstractions/src/OpenSearch.OpenSearch.Ephemeral/README.md

@@ -22,8 +22,6 @@ on disk (LocalAppData).
 The easiest way to get started is by simply passing the version you want to be bootstrapped to `EphemeralCluster`.
 `Start` starts the `OpenSearchNode`'s and waits for them to be started. The default overload waits `2 minutes`.
 
-NOTE: The ephemeral cluster running with security enabled will require an initial admin password to be set. You can set it as an environment variable `OPENSEARCH_INITIAL_ADMIN_PASSWORD`.
-
 ```csharp
 using (var cluster = new EphemeralCluster("1.0.0"))
 {

abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs

@@ -172,26 +172,34 @@ protected static void WriteFileIfNotExist(string fileLocation, string contents)
 
 		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
 			string binary, string description, params string[] arguments) =>
-			ExecuteBinaryInternal(config, writer, binary, description, null, arguments);
+			ExecuteBinaryInternal(config, writer, binary, description, null, null, arguments);
 
 		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
 			string binary, string description, StartedHandler startedHandler, params string[] arguments) =>
-			ExecuteBinaryInternal(config, writer, binary, description, startedHandler, arguments);
+			ExecuteBinaryInternal(config, writer, binary, description, startedHandler, null, arguments);
+
+		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
+			string binary, string description, IDictionary<string, string> environmentVariables,
+			params string[] arguments) =>
+			ExecuteBinaryInternal(config, writer, binary, description, null, environmentVariables, arguments);
 
 		private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
-			string binary, string description, StartedHandler startedHandler, params string[] arguments)
+			string binary, string description, StartedHandler startedHandler, IDictionary<string, string> environmentVariables, params string[] arguments)
 		{
 			var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}";
 			writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}");
 
+			var environment = environmentVariables != null
+				? new Dictionary<string, string>(environmentVariables)
+				: new Dictionary<string, string>();
+
+			environment.Add(config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath);
+			environment.Add("OPENSEARCH_HOME", config.FileSystem.OpenSearchHome);
+
 			var timeout = TimeSpan.FromSeconds(420);
 			var processStartArguments = new StartArguments(binary, arguments)
 			{
-				Environment = new Dictionary<string, string>
-				{
-					{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
-					{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome},
-				}
+				Environment = environment
 			};
 
 			var result = startedHandler != null

abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs

@@ -26,10 +26,11 @@
 *  under the License.
 */
 
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using OpenSearch.OpenSearch.Managed.ConsoleWriters;
-using OpenSearch.Stack.ArtifactsApi;
+using SemanticVersioning;
 
 namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks
 {
@@ -38,26 +39,55 @@ public class InitialConfiguration : ClusterComposeTask
 		public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluster)
 		{
 			var fs = cluster.FileSystem;
-			var configFile = Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml");
 
-			if (File.Exists(configFile) && File.ReadLines(configFile).Any(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#")))
-			{
-				cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} opensearch.yml already exists, skipping initial configuration");
-				return;
-			}
+			var installConfigDir = Path.Combine(fs.OpenSearchHome, "config");
+			var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml");
+			var configFile = Path.Combine(fs.ConfigPath, "opensearch.yml");
+			var configSecurity = Path.Combine(fs.ConfigPath, "opensearch-security");
+
+			var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0);
+
+			var installSecurityConfig = Path.Combine(installConfigDir, "opensearch-security");
 
-			var securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh";
+			if (isNewDemoScript && Directory.Exists(configSecurity)) Directory.Move(configSecurity, installSecurityConfig);
+
+			const string securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh";
 			var securityInstallDemoConfig = Path.Combine(fs.OpenSearchHome, securityInstallDemoConfigSubPath);
 
 			cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]");
 
-			ExecuteBinary(
-				cluster.ClusterConfiguration,
-				cluster.Writer,
-				"/bin/bash",
-				"install security plugin demo configuration",
-				securityInstallDemoConfig,
-				"-y", "-i", "-s");
+			var alreadyInstalled = File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"));
+
+			if (!alreadyInstalled)
+			{
+				var env = new Dictionary<string, string>();
+				var args = new List<string> { securityInstallDemoConfig, "-y", "-i" };
+
+				if (isNewDemoScript)
+				{
+					env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin");
+					args.Add("-t");
+				}
+
+				ExecuteBinary(
+					cluster.ClusterConfiguration,
+					cluster.Writer,
+					"/bin/bash",
+					"install security plugin demo configuration",
+					env,
+					args.ToArray());
+			}
+
+			Directory.CreateDirectory(fs.ConfigPath);
+
+			if (isNewDemoScript)
+			{
+				Directory.CreateDirectory(configSecurity);
+				CopyFolder(installSecurityConfig, configSecurity);
+			}
+
+			foreach (var f in new[]{"opensearch.yml", "esnode.pem", "esnode-key.pem", "root-ca.pem"})
+				File.Copy(Path.Combine(installConfigDir, f), Path.Combine(fs.ConfigPath, f), true);
 
 			if (cluster.ClusterConfiguration.EnableSsl) return;
 

samples/Samples/Utils/OpenSearchClientOptions.cs

@@ -18,7 +18,6 @@ public static IValueDescriptor<IOpenSearchClient> AddOpenSearchClientOptions(thi
 	{
 		Option<Uri> host = new("--host", () => new Uri("https://localhost:9200"), "The OpenSearch host to connect to");
 		Option<string> username = new("--username", () => "admin", "The username to use for authentication");
-		// TODO: This line needs to be updated to replace 'admin' with custom password
 		Option<string> password = new("--password", () => "admin", "The password to use for authentication");
 
 		Action<Option> add = global ? command.AddGlobalOption : command.AddOption;

tests/Tests.YamlRunner/Models.fs

@@ -61,9 +61,9 @@ let (|IsDoCatch|_|) (s:string) =
     | "conflict" -> Some Conflict 
     | "unavailable" -> Some Unavailable
     | "param" -> Some UnknownParameter
-    | "request" -> Some OtherBadResponse 
-    | s -> Some <| CatchRegex (s.Trim('/'))
-    
+    | "request" -> Some OtherBadResponse
+    | s -> Some <| CatchRegex (Regex.Replace(s.Trim('/'), @"(?<!\\)\\_", "_"))
+
 type NodeSelector =
     | NodeVersionSelector of string
     | NodeAttributeSelector of string * string

tests/Tests/Search/SearchTemplate/RenderSearchTemplate/RenderSearchTemplateApiTests.cs

@@ -40,6 +40,7 @@
 
 namespace Tests.Search.SearchTemplate.RenderSearchTemplate
 {
+	[SkipVersion("2.10.*,2.11.*", "Broken by security plugin https://github.com/opensearch-project/security/issues/3672")]
 	public class RenderSearchTemplateApiTests
 		: ApiIntegrationTestBase<ReadOnlyCluster, RenderSearchTemplateResponse, IRenderSearchTemplateRequest, RenderSearchTemplateDescriptor,
 			RenderSearchTemplateRequest>