Merge branch 'elastic:main' into master

config/serverless.yml

@@ -48,6 +48,8 @@ xpack.index_management.enableIndexStats: false
 xpack.index_management.editableIndexSettings: limited
 # Disable Storage size column in the Data streams table from Index Management UI
 xpack.index_management.enableDataStreamsStorageColumn: false
+# Disable _source field in the Mappings editor's advanced options form from Index Management UI
+xpack.index_management.enableMappingsSourceFieldSection: false
 # Disable toggle for enabling data retention in DSL form from Index Management UI
 xpack.index_management.enableTogglingDataRetention: false
 
@@ -138,7 +140,7 @@ xpack.actions.queued.max: 10000
 # Disables ESQL in advanced settings (hides it from the UI)
 uiSettings:
   overrides:
-    discover:enableESQL: true
+    enableESQL: true
 
 # Task Manager
 xpack.task_manager.allow_reading_invalid_state: false

docs/discover/try-esql.asciidoc

@@ -11,7 +11,7 @@ In this tutorial we'll use the {kib} sample web logs in Discover and Lens to exp
 [[prerequisite]]
 === Prerequisite 
 
-To be able to select **Try {esql}** from the Data views menu the `discover:enableESQL` setting must be enabled from **Stack Management > Advanced Settings**. It is enabled by default. 
+To be able to select **Try {esql}** from the Data views menu the `enableESQL` setting must be enabled from **Stack Management > Advanced Settings**. It is enabled by default. 
 
 [float]
 [[tutorial-try-esql]]

docs/management/advanced-options.asciidoc

@@ -209,6 +209,9 @@ The default selection in the time filter.
 The maximum height that a cell occupies in a table. Set to 0 to disable
 truncation.
 
+[[enableESQL]]`enableESQL`::
+This setting enables ES|QL in Kibana.
+
 [float]
 [[presentation-labs]]
 ==== Presentation Labs
@@ -290,9 +293,6 @@ in the current data view is used.
 The columns that appear by default on the *Discover* page. The default is
 `_source`.
 
-[[discover:enableESQL]]`discover:enableESQL`::
-experimental[] Allows ES|QL queries for search.
-
 [[discover-max-doc-fields-displayed]]`discover:maxDocFieldsDisplayed`::
 Specifies the maximum number of fields to show in the document column of the *Discover* table.
 

docs/setup/settings.asciidoc

@@ -14,8 +14,11 @@ KBN_PATH_CONF=/home/kibana/config ./bin/kibana
 --
 
 The default host and port settings configure {kib} to run on `localhost:5601`. To change this behavior and allow remote users to connect, you'll need to update your `kibana.yml` file. You can also enable SSL and set a
-variety of other options. Finally, environment variables can be injected into
-configuration using `${MY_ENV_VAR}` syntax.
+variety of other options. 
+
+Environment variables can be injected into configuration using `${MY_ENV_VAR}` syntax. By default, configuration validation
+will fail if an environment variable used in the config file is not present when Kibana starts. This behavior can be changed by using a default value
+for the environment variable, using the `${MY_ENV_VAR:defaultValue}` syntax.
 
 `console.ui.enabled`::
 Toggling this causes the server to regenerate assets on the next startup,

docs/user/alerting/rule-types/es-query.asciidoc

@@ -12,7 +12,8 @@ The {es} query rule type runs a user-configured query, compares the number of
 matches to a configured threshold, and schedules actions to run when the 
 threshold condition is met.
 
-In *{stack-manage-app}* > *{rules-ui}*, click *Create rule*, fill in the name and optional tags, then select *{es} query*.
+In *{stack-manage-app}* > *{rules-ui}*, click *Create rule*.
+Select the *{es} query* rule type then fill in the name and optional tags.
 An {es} query rule can be defined using {es} Query Domain Specific Language (DSL), {es} Query Language (ES|QL), {kib} Query Language (KQL), or Lucene.
 
 [float]

docs/user/alerting/rule-types/geo-rule-types.asciidoc

@@ -10,7 +10,8 @@
 
 The tracking containment rule alerts when an entity is contained or no longer contained within a boundary. 
 
-In *{stack-manage-app}* > *{rules-ui}*, click *Create rule*, fill in the name and optional tags, then select *Tracking containment*.
+In *{stack-manage-app}* > *{rules-ui}*, click *Create rule*.
+Select the *Tracking containment* rule type then fill in the name and optional tags.
 
 [float]
 === Define the conditions

docs/user/alerting/rule-types/index-threshold.asciidoc

@@ -10,7 +10,8 @@
 
 The index threshold rule type runs an {es} query. It aggregates field values from documents, compares them to threshold values, and schedules actions to run when the thresholds are met.
 
-In *{stack-manage-app}* > *{rules-ui}*, click *Create rule*, fill in the name and optional tags, then select *Index threshold*.
+In *{stack-manage-app}* > *{rules-ui}*, click *Create rule*.
+Select the *Index threshold* rule type then fill in the name and optional tags.
 
 [float]
 === Define the conditions
@@ -107,15 +108,11 @@ You can also specify <<rule-action-variables,variables common to all rules>>.
 
 In this example, you will use the {kib} <<add-sample-data,sample weblog data set>> to set up and tune the conditions on an index threshold rule. For this example, you want to detect when any of the top four sites serve more than 420,000 bytes over a 24 hour period.
 
-.  Open the main menu, then click *{stack-manage-app} > {rules-ui}*.
+.  Go to *{stack-manage-app} > {rules-ui}* and click *Create rule*.
 
-.  Create a new rule.
+.  Select the **Index threshold** rule type.
 
-.. Provide a rule name and select the **Index threshold** rule type.
-+
-[role="screenshot"]
-image::user/alerting/images/rule-types-index-threshold-select.png[Choosing an index threshold rule type]
-// NOTE: This is an autogenerated screenshot. Do not edit it directly.
+.. Provide a rule name.
 
 .. Select an index. Click *Index*, and set *Indices to query* to `kibana_sample_data_logs`. Set the *Time field* to `@timestamp`.
 +

examples/response_stream/server/plugin.ts

@@ -27,7 +27,7 @@ export class ResponseStreamPlugin implements Plugin {
   public setup(core: CoreSetup, plugins: ResponseStreamSetupPlugins) {
     const router = core.http.createRouter<DataRequestHandlerContext>();
 
-    core.getStartServices().then(([_, depsStart]) => {
+    void core.getStartServices().then(([_, depsStart]) => {
       defineReducerStreamRoute(router, this.logger);
       defineSimpleStringStreamRoute(router, this.logger);
     });

examples/response_stream/server/routes/reducer_stream.ts

@@ -85,44 +85,45 @@ export const defineReducerStreamRoute = (router: IRouter, logger: Logger) => {
         let progress = 0;
 
         async function pushStreamUpdate() {
-          setTimeout(() => {
-            try {
-              progress++;
-
-              if (progress > 100 || shouldStop) {
-                end();
-                return;
-              }
-
-              push(updateProgressAction(progress));
-
-              const randomEntity = entities[Math.floor(Math.random() * entities.length)];
-              const randomAction = actions[Math.floor(Math.random() * actions.length)];
-
-              if (randomAction === 'add') {
-                const randomCommits = Math.floor(Math.random() * 100);
-                push(addToEntityAction(randomEntity, randomCommits));
-              } else if (randomAction === 'delete') {
-                push(deleteEntityAction(randomEntity));
-              } else if (randomAction === 'throw-error') {
-                // Throw an error. It should not crash Kibana!
-                // It should be caught and logged to the Kibana server console.
-                throw new Error('There was a (simulated) server side error!');
-              } else if (randomAction === 'emit-error') {
-                // Emit an error as a stream action.
-                push(errorAction('(Simulated) error pushed to the stream'));
-                return;
-              }
-
-              pushStreamUpdate();
-            } catch (e) {
-              logger.error(e);
+          await new Promise((resolve) =>
+            setTimeout(resolve, Math.floor(Math.random() * maxTimeoutMs))
+          );
+          try {
+            progress++;
+
+            if (progress > 100 || shouldStop) {
+              end();
+              return;
             }
-          }, Math.floor(Math.random() * maxTimeoutMs));
+
+            push(updateProgressAction(progress));
+
+            const randomEntity = entities[Math.floor(Math.random() * entities.length)];
+            const randomAction = actions[Math.floor(Math.random() * actions.length)];
+
+            if (randomAction === 'add') {
+              const randomCommits = Math.floor(Math.random() * 100);
+              push(addToEntityAction(randomEntity, randomCommits));
+            } else if (randomAction === 'delete') {
+              push(deleteEntityAction(randomEntity));
+            } else if (randomAction === 'throw-error') {
+              // Throw an error. It should not crash Kibana!
+              // It should be caught and logged to the Kibana server console.
+              throw new Error('There was a (simulated) server side error!');
+            } else if (randomAction === 'emit-error') {
+              // Emit an error as a stream action.
+              push(errorAction('(Simulated) error pushed to the stream'));
+              return;
+            }
+
+            void pushStreamUpdate();
+          } catch (e) {
+            logger.error(e);
+          }
         }
 
         // do not call this using `await` so it will run asynchronously while we return the stream already.
-        pushStreamUpdate();
+        void pushStreamUpdate();
 
         return response.ok(responseWithHeaders);
       }

examples/response_stream/server/routes/single_string_stream.ts

@@ -67,7 +67,7 @@ export const defineSimpleStringStreamRoute = (router: IRouter, logger: Logger) =
               await timeout(Math.floor(Math.random() * maxTimeoutMs));
 
               if (!shouldStop) {
-                pushStreamUpdate();
+                void pushStreamUpdate();
               }
             } else {
               end();
@@ -78,7 +78,7 @@ export const defineSimpleStringStreamRoute = (router: IRouter, logger: Logger) =
         }
 
         // do not call this using `await` so it will run asynchronously while we return the stream already.
-        pushStreamUpdate();
+        void pushStreamUpdate();
 
         return response.ok(responseWithHeaders);
       }

examples/search_examples/server/plugin.ts

@@ -48,7 +48,7 @@ export class SearchExamplesPlugin
     this.logger.debug('search_examples: Setup');
     const router = core.http.createRouter<DataRequestHandlerContext>();
 
-    core.getStartServices().then(([_, depsStart]) => {
+    void core.getStartServices().then(([_, depsStart]) => {
       const myStrategy = mySearchStrategyProvider(depsStart.data);
       const fibonacciStrategy = fibonacciStrategyProvider();
       deps.data.search.registerSearchStrategy('myStrategy', myStrategy);

package.json

@@ -990,7 +990,7 @@
     "deepmerge": "^4.2.2",
     "del": "^6.1.0",
     "diff": "^5.1.0",
-    "elastic-apm-node": "^4.5.2",
+    "elastic-apm-node": "^4.5.3",
     "email-addresses": "^5.0.0",
     "eventsource-parser": "^1.1.1",
     "execa": "^5.1.1",
@@ -1053,7 +1053,7 @@
     "lz-string": "^1.4.4",
     "mapbox-gl-draw-rectangle-mode": "1.0.4",
     "maplibre-gl": "3.1.0",
-    "markdown-it": "^12.3.2",
+    "markdown-it": "^14.1.0",
     "mdast-util-to-hast": "10.2.0",
     "memoize-one": "^6.0.0",
     "mime": "^2.4.4",

packages/core/http/core-http-server/src/versioning/types.ts

@@ -32,7 +32,7 @@ export type VersionedRouteConfig<Method extends RouteMethod> = Omit<
   RouteConfig<unknown, unknown, unknown, Method>,
   'validate' | 'options'
 > & {
-  options?: Omit<RouteConfigOptions<Method>, 'access'>;
+  options?: Omit<RouteConfigOptions<Method>, 'access' | 'description'>;
   /** See {@link RouteConfigOptions<RouteMethod>['access']} */
   access: Exclude<RouteConfigOptions<Method>['access'], undefined>;
   /**

packages/kbn-config/src/__fixtures__/en_var_with_defaults.yml

@@ -0,0 +1,4 @@
+foo: 'pre-${KBN_ENV_VAR1}-mid-${KBN_ENV_VAR2:default2}-post'
+nested_list:
+  - id: 'a'
+    values: ['${KBN_ENV_VAR1:default1}', '${KBN_ENV_VAR2:default2}']

packages/kbn-config/src/raw/read_config.test.ts

@@ -130,3 +130,26 @@ test('supports unsplittable key syntax on nested list', () => {
     }
   `);
 });
+
+test('supports var:default syntax', () => {
+  process.env.KBN_ENV_VAR1 = 'val1';
+
+  const config = getConfigFromFiles([fixtureFile('/en_var_with_defaults.yml')]);
+
+  delete process.env.KBN_ENV_VAR1;
+
+  expect(config).toMatchInlineSnapshot(`
+    Object {
+      "foo": "pre-val1-mid-default2-post",
+      "nested_list": Array [
+        Object {
+          "id": "a",
+          "values": Array [
+            "val1",
+            "default2",
+          ],
+        },
+      ],
+    }
+  `);
+});

packages/kbn-config/src/raw/read_config.ts

@@ -11,21 +11,10 @@ import { safeLoad } from 'js-yaml';
 import { set } from '@kbn/safer-lodash-set';
 import { isPlainObject } from 'lodash';
 import { ensureValidObjectPath } from '@kbn/std';
-import { splitKey, getUnsplittableKey } from './utils';
+import { splitKey, getUnsplittableKey, replaceEnvVarRefs } from './utils';
 
 const readYaml = (path: string) => safeLoad(readFileSync(path, 'utf8'));
 
-function replaceEnvVarRefs(val: string) {
-  return val.replace(/\$\{(\w+)\}/g, (match, envVarName) => {
-    const envVarValue = process.env[envVarName];
-    if (envVarValue !== undefined) {
-      return envVarValue;
-    }
-
-    throw new Error(`Unknown environment variable referenced in config : ${envVarName}`);
-  });
-}
-
 interface YamlEntry {
   path: string[];
   value: any;

packages/kbn-config/src/raw/utils.test.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { splitKey, getUnsplittableKey } from './utils';
+import { splitKey, getUnsplittableKey, replaceEnvVarRefs } from './utils';
 
 describe('splitKey', () => {
   it('correctly splits on the dot delimiter', () => {
@@ -33,3 +33,39 @@ describe('getUnsplittableKey', () => {
     expect(getUnsplittableKey('foo.bar')).toEqual(undefined);
   });
 });
+
+describe('replaceEnvVarRefs', () => {
+  it('throws an error if the variable is not defined', () => {
+    expect(() => replaceEnvVarRefs('${VAR_1}', {})).toThrowErrorMatchingInlineSnapshot(
+      `"Unknown environment variable referenced in config : VAR_1"`
+    );
+  });
+  it('replaces the environment variable with its value', () => {
+    expect(replaceEnvVarRefs('${VAR_1}', { VAR_1: 'foo' })).toEqual('foo');
+  });
+  it('replaces the environment variable within a longer string', () => {
+    expect(replaceEnvVarRefs('hello ${VAR_1} bar', { VAR_1: 'foo' })).toEqual('hello foo bar');
+  });
+  it('replaces multiple occurrences of the same variable', () => {
+    expect(replaceEnvVarRefs('${VAR_1}-${VAR_1}', { VAR_1: 'foo' })).toEqual('foo-foo');
+  });
+  it('replaces multiple occurrences of different variables', () => {
+    expect(replaceEnvVarRefs('${VAR_1}-${VAR_2}', { VAR_1: 'foo', VAR_2: 'bar' })).toEqual(
+      'foo-bar'
+    );
+  });
+  it('uses the default value if specified and the var is not defined', () => {
+    expect(replaceEnvVarRefs('${VAR:default}', {})).toEqual('default');
+  });
+  it('uses the value from the var if specified even with a default value', () => {
+    expect(replaceEnvVarRefs('${VAR:default}', { VAR: 'value' })).toEqual('value');
+  });
+  it('supports defining a default value for multiple variables', () => {
+    expect(replaceEnvVarRefs('${VAR1:var}:${VAR2:var2}', {})).toEqual('var:var2');
+  });
+  it('only use default value for variables that are not set', () => {
+    expect(replaceEnvVarRefs('${VAR1:default1}:${VAR2:default2}', { VAR2: 'var2' })).toEqual(
+      'default1:var2'
+    );
+  });
+});

packages/kbn-config/src/raw/utils.ts

@@ -19,3 +19,25 @@ export const getUnsplittableKey = (rawKey: string): string | undefined => {
   }
   return undefined;
 };
+
+export function replaceEnvVarRefs(
+  val: string,
+  env: {
+    [key: string]: string | undefined;
+  } = process.env
+) {
+  return val.replace(/\$\{(\w+)(:(\w+))?\}/g, (match, ...groups) => {
+    const envVarName = groups[0];
+    const defaultValue = groups[2];
+
+    const envVarValue = env[envVarName];
+    if (envVarValue !== undefined) {
+      return envVarValue;
+    }
+    if (defaultValue !== undefined) {
+      return defaultValue;
+    }
+
+    throw new Error(`Unknown environment variable referenced in config : ${envVarName}`);
+  });
+}

packages/kbn-discover-utils/index.ts

@@ -13,7 +13,6 @@ export {
   DEFAULT_COLUMNS_SETTING,
   DOC_HIDE_TIME_COLUMN_SETTING,
   DOC_TABLE_LEGACY,
-  ENABLE_ESQL,
   FIELDS_LIMIT_SETTING,
   HIDE_ANNOUNCEMENTS,
   MAX_DOC_FIELDS_DISPLAYED,

packages/kbn-discover-utils/src/constants.ts

@@ -12,7 +12,6 @@ export const CONTEXT_TIE_BREAKER_FIELDS_SETTING = 'context:tieBreakerFields';
 export const DEFAULT_COLUMNS_SETTING = 'defaultColumns';
 export const DOC_HIDE_TIME_COLUMN_SETTING = 'doc_table:hideTimeColumn';
 export const DOC_TABLE_LEGACY = 'doc_table:legacy';
-export const ENABLE_ESQL = 'discover:enableESQL';
 export const FIELDS_LIMIT_SETTING = 'fields:popularLimit';
 export const HIDE_ANNOUNCEMENTS = 'hideAnnouncements';
 export const MAX_DOC_FIELDS_DISPLAYED = 'discover:maxDocFieldsDisplayed';