Updates references to security demo configuration (#480)

* Updates github action reference to secruity demo configuration Signed-off-by: Darshit Chanpura <dchanp@amazon.com> * Updates abstractions README to reflect changes with demo configuration Signed-off-by: Darshit Chanpura <dchanp@amazon.com> * Updates references to admin password Signed-off-by: Darshit Chanpura <dchanp@amazon.com> * Wider test range Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Set initial admin password Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Fix disabling SSL Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Improve demo config condition Signed-off-by: Thomas Farr <tsfarr@amazon.com> * Use random password on >=2.12 Signed-off-by: Thomas Farr <tsfarr@amazon.com> --------- Signed-off-by: Darshit Chanpura <dchanp@amazon.com> Signed-off-by: Thomas Farr <tsfarr@amazon.com> Co-authored-by: Thomas Farr <tsfarr@amazon.com> (cherry picked from commit dd2e674)
opensearch-project · Jan 18, 2024 · 5d49723 · 5d49723
1 parent d888669
commit 5d49723
Show file tree

Hide file tree

Showing 10 changed files with 74 additions and 35 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -17,17 +17,14 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - 2.9.0
+          - 2.11.1
+          - 2.10.0
           - 2.8.0
-          - 2.7.0
           - 2.6.0
-          - 2.5.0
           - 2.4.1
-          - 2.3.0
           - 2.2.1
-          - 2.1.0
           - 2.0.1
-          - 1.3.11
+          - 1.3.14
           - 1.2.4
           - 1.1.0
 
@@ -62,7 +59,6 @@ jobs:
           path: client/build/output/*
 
   integration-opensearch-unreleased:
-    if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268)
     name: Integration OpenSearch Unreleased
     runs-on: ubuntu-latest
     strategy:

diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs
@@ -34,6 +34,7 @@
 using System.Security.Cryptography;
 using System.Text;
 using OpenSearch.OpenSearch.Managed;
+using OpenSearch.OpenSearch.Managed.Configuration;
 using OpenSearch.Stack.ArtifactsApi;
 
 namespace OpenSearch.OpenSearch.Ephemeral
@@ -59,6 +60,13 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo
 
 		protected EphemeralClusterComposer<TConfiguration> Composer { get; }
 
+		protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port)
+		{
+			base.ModifyNodeConfiguration(nodeConfiguration, port);
+
+			if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true");
+		}
+
 		public virtual ICollection<Uri> NodesUris(string hostName = null)
 		{
 			hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any()

diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs
@@ -172,22 +172,35 @@ protected static void WriteFileIfNotExist(string fileLocation, string contents)
 
 		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
 			string binary, string description, params string[] arguments) =>
-			ExecuteBinaryInternal(config, writer, binary, description, arguments);
+			ExecuteBinaryInternal(config, writer, binary, description, null, arguments);
+
+		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
+			string binary, string description, IDictionary<string, string> environmentVariables,
+			params string[] arguments) =>
+			ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments);
 
 		private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
-			string binary, string description, params string[] arguments)
+			string binary, string description, IDictionary<string, string> environmentVariables, params string[] arguments)
 		{
 			var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}";
 			writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}");
 
+			var environment = new Dictionary<string, string>
+			{
+				{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
+				{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome}
+			};
+
+			if (environmentVariables != null)
+			{
+				foreach (var kvp in environmentVariables)
+					environment[kvp.Key] = kvp.Value;
+			}
+
 			var timeout = TimeSpan.FromSeconds(420);
 			var processStartArguments = new StartArguments(binary, arguments)
 			{
-				Environment = new Dictionary<string, string>
-				{
-					{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
-					{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome},
-				}
+				Environment = environment
 			};
 
 			var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter());

diff --git a/...tions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs b/...tions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs
@@ -26,10 +26,12 @@
 *  under the License.
 */
 
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using OpenSearch.OpenSearch.Managed.ConsoleWriters;
-using OpenSearch.Stack.ArtifactsApi;
+using OpenSearch.Stack.ArtifactsApi.Products;
+using SemanticVersioning;
 
 namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks
 {
@@ -38,30 +40,39 @@ public class InitialConfiguration : ClusterComposeTask
 		public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluster)
 		{
 			var fs = cluster.FileSystem;
-			var configFile = Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml");
 
-			if (File.Exists(configFile) && File.ReadLines(configFile).Any(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#")))
-			{
-				cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} opensearch.yml already exists, skipping initial configuration");
+			var installConfigDir = Path.Combine(fs.OpenSearchHome, "config");
+			var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml");
+			var pluginSecurity = Path.Combine(fs.OpenSearchHome, "plugins/opensearch-security");
+
+			if (!Directory.Exists(pluginSecurity))
 				return;
-			}
 
-			var securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh";
-			var securityInstallDemoConfig = Path.Combine(fs.OpenSearchHome, securityInstallDemoConfigSubPath);
+			var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0);
+
+			const string securityInstallDemoConfigSubPath = "tools/install_demo_configuration.sh";
+			var securityInstallDemoConfig = Path.Combine(pluginSecurity, securityInstallDemoConfigSubPath);
 
 			cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]");
 
+			if (File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"))) return;
+
+			var env = new Dictionary<string, string>();
+			var args = new List<string> { securityInstallDemoConfig, "-y", "-i" };
+
+			if (isNewDemoScript)
+			{
+				env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin");
+				args.Add("-t");
+			}
+
 			ExecuteBinary(
 				cluster.ClusterConfiguration,
 				cluster.Writer,
 				"/bin/bash",
 				"install security plugin demo configuration",
-				securityInstallDemoConfig,
-				"-y", "-i", "-s");
-
-			if (cluster.ClusterConfiguration.EnableSsl) return;
-
-			File.AppendAllText(configFile, "plugins.security.disabled: true");
+				env,
+				args.ToArray());
 		}
 	}
 }
diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs
@@ -27,6 +27,7 @@
 */
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
@@ -93,12 +94,21 @@ public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluste
 				cluster.Writer?.WriteDiagnostic(
 					$"{{{nameof(InstallPlugins)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}");
 
-				if (!Directory.Exists(fs.ConfigPath)) Directory.CreateDirectory(fs.ConfigPath);
+				var homeConfigPath = Path.Combine(fs.OpenSearchHome, "config");
+
+				if (!Directory.Exists(homeConfigPath)) Directory.CreateDirectory(homeConfigPath);
+
+				var env = new Dictionary<string, string>
+				{
+					{ fs.ConfigEnvironmentVariableName, homeConfigPath }
+				};
+
 				ExecuteBinary(
 					cluster.ClusterConfiguration,
 					cluster.Writer,
 					fs.PluginBinary,
 					$"install opensearch plugin: {plugin.SubProductName}",
+					env,
 					"install", "--batch", GetPluginLocation(plugin, v));
 
 				CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin);

diff --git a/guides/document-lifecycle.md b/guides/document-lifecycle.md
@@ -7,7 +7,7 @@ Assuming you have OpenSearch running locally on port 9200, you can create a clie
 var node = new Uri("https://localhost:9200");
 var config = new ConnectionSettings(node)
     .ServerCertificateValidationCallback(CertificateValidations.AllowAll)
-    .BasicAuthentication("admin", "admin")
+    .BasicAuthentication("admin", <admin-password>)
     .DisableDirectStreaming();
 var client = new OpenSearchClient(config);
 

diff --git a/guides/index-template.md b/guides/index-template.md
@@ -12,7 +12,7 @@ using OpenSearch.Net;
 var node = new Uri("https://localhost:9200");
 var config = new ConnectionSettings(node)
     .ServerCertificateValidationCallback(CertificateValidations.AllowAll)
-    .BasicAuthentication("admin", "admin");
+    .BasicAuthentication("admin", <admin-password>);
 
 var client = new OpenSearchClient(config);;
 ```

diff --git a/guides/search.md b/guides/search.md
@@ -12,7 +12,7 @@ var node = new Uri("https://localhost:9200");
 var config = new ConnectionSettings(node)
     .ThrowExceptions()
     .ServerCertificateValidationCallback(CertificateValidations.AllowAll)
-    .BasicAuthentication("admin", "admin");
+    .BasicAuthentication("admin", <admin-password>);
 var client = new OpenSearchClient(config);
 
 class Movie

diff --git a/tests/Tests.YamlRunner/Models.fs b/tests/Tests.YamlRunner/Models.fs
@@ -61,9 +61,9 @@ let (|IsDoCatch|_|) (s:string) =
     | "conflict" -> Some Conflict 
     | "unavailable" -> Some Unavailable
     | "param" -> Some UnknownParameter
-    | "request" -> Some OtherBadResponse 
-    | s -> Some <| CatchRegex (s.Trim('/'))
-    
+    | "request" -> Some OtherBadResponse
+    | s -> Some <| CatchRegex (Regex.Replace(s.Trim('/'), @"(?<!\\)\\_", "_"))
+
 type NodeSelector =
     | NodeVersionSelector of string
     | NodeAttributeSelector of string * string

diff --git a/tests/Tests/Search/SearchTemplate/RenderSearchTemplate/RenderSearchTemplateApiTests.cs b/tests/Tests/Search/SearchTemplate/RenderSearchTemplate/RenderSearchTemplateApiTests.cs
@@ -40,6 +40,7 @@
 
 namespace Tests.Search.SearchTemplate.RenderSearchTemplate
 {
+	[SkipVersion("2.10.*,2.11.*", "Broken by security plugin https://github.com/opensearch-project/security/issues/3672")]
 	public class RenderSearchTemplateApiTests
 		: ApiIntegrationTestBase<ReadOnlyCluster, RenderSearchTemplateResponse, IRenderSearchTemplateRequest, RenderSearchTemplateDescriptor,
 			RenderSearchTemplateRequest>