From a380f25fffc0f30fe96cab1757230b9be4c86f82 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 12 Feb 2025 11:10:21 -0500 Subject: [PATCH 01/18] Update PluginStats with new field in response optional_extended_plugins Signed-off-by: Craig Perkins --- spec/schemas/_common.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index de722eebe..cf988cbd2 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1516,6 +1516,10 @@ components: type: ['null', string] opensearch_version: $ref: '#/components/schemas/VersionString' + optional_extended_plugins: + type: array + items: + type: string required: - classname - description @@ -1525,6 +1529,7 @@ components: - name - opensearch_version - version + - optional_extended_plugins NodeStatistics: type: object properties: From faf5b330c8126ba9bdaab98e37f2ad590e2e5ed3 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 12 Feb 2025 11:13:52 -0500 Subject: [PATCH 02/18] Fix lint error and add to CHANGELOG Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + spec/schemas/_common.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 278732640..ac1199e17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Changed naming of `snapshot._common`'s `Status`, `ShardsStats`, `ShardsStatsStage`, `ShardsStatsSummary` and `ShardsStatsSummaryItem` schemas to be prefixed with `Snapshot` ([#730](https://github.com/opensearch-project/opensearch-api-specification/pull/730)) - Changed `ml.get_memory` and `ml.get_message` to split out `get_all` variants ([#796](https://github.com/opensearch-project/opensearch-api-specification/pull/796)) - Changed `ml.get_tools` to have two different operation groups `ml.get_all_tools` and `ml.get_tool` ([#799](https://github.com/opensearch-project/opensearch-api-specification/pull/799)) +- Update `PluginStats` with new field in response `optional_extended_plugins` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) ## [0.1.0] - 2024-10-25 diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index cf988cbd2..bddc76358 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1528,8 +1528,8 @@ components: - java_version - name - opensearch_version - - version - optional_extended_plugins + - version NodeStatistics: type: object properties: From 0758d80500602a7b1ff0f566fbe53a0b161e3d34 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 12 Feb 2025 20:01:15 -0500 Subject: [PATCH 03/18] Update test-spec Signed-off-by: Craig Perkins --- .github/workflows/test-spec.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index dad66e2f1..e705a199d 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -28,38 +28,38 @@ jobs: admin_password: admin - version: 2.0.0 admin_password: admin - - version: 2.18.0 - - version: 2.18.0 + - version: 2.19.0 + - version: 2.19.0 tests: plugins/index_state_management - - version: 2.18.0 + - version: 2.19.0 tests: plugins/ml - - version: 2.18.0 + - version: 2.19.0 tests: routing - - version: 2.18.0 + - version: 2.19.0 tests: snapshot - - version: 2.18.0 + - version: 2.19.0 tests: remote_store - - version: 2.18.0 + - version: 2.19.0 tests: dangling url: http://localhost:9200 - - version: 2.18.0 + - version: 2.19.0 tests: plugins/replication url: http://localhost:9200 - - version: 2.18.0 + - version: 2.19.0 tests: plugins/streaming - - version: 2.18.0 + - version: 2.19.0 tests: plugins/notifications - - version: 2.18.0 + - version: 2.19.0 tests: plugins/query_insights - - version: 2.18.0 + - version: 2.19.0 tests: plugins/workload-management - - version: 2.18.0 + - version: 2.19.0 tests: plugins/analysis - - version: 2.18.0 + - version: 2.19.0 tests: plugins/security cert: tests/plugins/security/.kirk.pem key: tests/plugins/security/.kirk-key.pem - - version: 2.19.0 + - version: 2.20.0 hub: opensearchstaging ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' - version: 3.0.0 From 25567150c16391a62a979ec8b1ea5e725c55ccfe Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 14 Feb 2025 10:28:41 -0500 Subject: [PATCH 04/18] Add test for /_cluster/stats/{metric}/nodes/{node_id} Signed-off-by: Craig Perkins --- .../analysis/indices/cluster/stats.yaml | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 tests/plugins/analysis/indices/cluster/stats.yaml diff --git a/tests/plugins/analysis/indices/cluster/stats.yaml b/tests/plugins/analysis/indices/cluster/stats.yaml new file mode 100644 index 000000000..8786a36f8 --- /dev/null +++ b/tests/plugins/analysis/indices/cluster/stats.yaml @@ -0,0 +1,290 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test cluster stats. +chapters: + - synopsis: Returns statistics about a cluster. + path: /_cluster/stats/{metric}/nodes/{node_id} + parameters: + metric: + - plugins + node_id: _cluster_manager + method: GET + response: + status: 200 + payload: + nodes: + plugins: + - name: opensearch-cross-cluster-replication + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Cross Cluster Replication Plugin + classname: org.opensearch.replication.ReplicationPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-custom-codecs + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch plugin that implements custom compression codecs + classname: org.opensearch.index.codec.customcodecs.CustomCodecPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-reports-scheduler + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Scheduler for Dashboards Reports Plugin + classname: org.opensearch.reportsscheduler.ReportsSchedulerPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: analysis-phonenumber + version: "2.19.0" + opensearch_version: "2.19.0" + java_version: "11" + description: "Adds an analyzer for phone numbers to OpenSearch." + classname: org.opensearch.analysis.phone.PhoneNumberAnalysisPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-alerting + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Amazon OpenSearch alerting plugin + classname: org.opensearch.alerting.AlertingPlugin + custom_foldername: "" + extended_plugins: + - lang-painless + has_native_controller: false + optional_extended_plugins: [] + - name: analysis-icu + version: "2.19.0" + opensearch_version: "2.19.0" + java_version: "11" + description: "The ICU Analysis plugin integrates the Lucene ICU module into OpenSearch, adding ICU-related analysis components." + classname: org.opensearch.plugin.analysis.icu.AnalysisICUPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-anomaly-detection + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch anomaly detector plugin + classname: org.opensearch.timeseries.TimeSeriesAnalyticsPlugin + custom_foldername: "" + extended_plugins: + - lang-painless + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-geospatial + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Geospatial plugin to host geospatial features + classname: org.opensearch.geospatial.plugin.GeospatialPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-system-templates + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch plugin that provides basic application based configuration templates. + classname: org.opensearch.system.applicationtemplates.ApplicationBasedConfigurationSystemTemplatesPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-performance-analyzer + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Performance Analyzer Plugin + classname: org.opensearch.performanceanalyzer.PerformanceAnalyzerPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-ml + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: machine learning plugin for opensearch + classname: org.opensearch.ml.plugin.MachineLearningPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-asynchronous-search + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Provides support for asynchronous search + classname: org.opensearch.search.asynchronous.plugin.AsynchronousSearchPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-skills + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Skills + classname: org.opensearch.agent.ToolPlugin + custom_foldername: "" + extended_plugins: + - opensearch-ml + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-sql + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch SQL + classname: org.opensearch.sql.plugin.SQLPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-notifications + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Notifications Plugin + classname: org.opensearch.notifications.NotificationPlugin + custom_foldername: "" + extended_plugins: + - opensearch-notifications-core + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-ltr + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Learning to Rank Query w/ RankLib Models + classname: com.o19s.es.ltr.LtrQueryParserPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-neural-search + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: A plugin that adds dense neural retrieval into the OpenSearch ecosytem + classname: org.opensearch.neuralsearch.plugin.NeuralSearch + custom_foldername: "" + extended_plugins: + - opensearch-knn + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-security + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Provide access control related features for OpenSearch + classname: org.opensearch.security.OpenSearchSecurityPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-flow-framework + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch plugin that enables builders to innovate AI apps on OpenSearch + classname: org.opensearch.flowframework.FlowFrameworkPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-index-management + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Index Management Plugin + classname: org.opensearch.indexmanagement.IndexManagementPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: query-insights + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Query Insights plugin + classname: org.opensearch.plugin.insights.QueryInsightsPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-job-scheduler + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Job Scheduler plugin + classname: org.opensearch.jobscheduler.JobSchedulerPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-notifications-core + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Notifications Core Plugin + classname: org.opensearch.notifications.core.NotificationCorePlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-knn + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch k-NN plugin + classname: org.opensearch.knn.plugin.KNNPlugin + custom_foldername: "" + extended_plugins: + - lang-painless + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-observability + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Plugin for OpenSearch Dashboards Observability + classname: org.opensearch.observability.ObservabilityPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-security-analytics + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Security Analytics plugin + classname: org.opensearch.securityanalytics.SecurityAnalyticsPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + - opensearch-alerting + has_native_controller: false + optional_extended_plugins: [] From e9c9eb13552ba878703c3f90ee0a03656c2eb7c7 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sat, 22 Feb 2025 19:58:05 -0500 Subject: [PATCH 05/18] Add mechanism to assert that response contains given a list of target values Signed-off-by: Craig Perkins --- json_schemas/test_story.schema.yaml | 4 + .../analysis/indices/cluster/stats.yaml | 280 +----------------- tools/src/tester/ChapterEvaluator.ts | 3 +- tools/src/tester/ResponsePayloadEvaluator.ts | 38 ++- .../tester/SupplementalChapterEvaluator.ts | 3 +- tools/src/tester/types/story.types.ts | 1 + .../tester/ResponsePayloadEvaluator.test.ts | 10 +- 7 files changed, 46 insertions(+), 293 deletions(-) diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index a0bc7ba23..21cc3111c 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -196,6 +196,10 @@ definitions: content_type: type: string default: application/json + contains: + type: array + items: + type: string payload: $ref: '#/definitions/Payload' required: [status] diff --git a/tests/plugins/analysis/indices/cluster/stats.yaml b/tests/plugins/analysis/indices/cluster/stats.yaml index 8786a36f8..3111a3d22 100644 --- a/tests/plugins/analysis/indices/cluster/stats.yaml +++ b/tests/plugins/analysis/indices/cluster/stats.yaml @@ -11,280 +11,6 @@ chapters: method: GET response: status: 200 - payload: - nodes: - plugins: - - name: opensearch-cross-cluster-replication - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Cross Cluster Replication Plugin - classname: org.opensearch.replication.ReplicationPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-custom-codecs - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch plugin that implements custom compression codecs - classname: org.opensearch.index.codec.customcodecs.CustomCodecPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-reports-scheduler - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Scheduler for Dashboards Reports Plugin - classname: org.opensearch.reportsscheduler.ReportsSchedulerPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: analysis-phonenumber - version: "2.19.0" - opensearch_version: "2.19.0" - java_version: "11" - description: "Adds an analyzer for phone numbers to OpenSearch." - classname: org.opensearch.analysis.phone.PhoneNumberAnalysisPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-alerting - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Amazon OpenSearch alerting plugin - classname: org.opensearch.alerting.AlertingPlugin - custom_foldername: "" - extended_plugins: - - lang-painless - has_native_controller: false - optional_extended_plugins: [] - - name: analysis-icu - version: "2.19.0" - opensearch_version: "2.19.0" - java_version: "11" - description: "The ICU Analysis plugin integrates the Lucene ICU module into OpenSearch, adding ICU-related analysis components." - classname: org.opensearch.plugin.analysis.icu.AnalysisICUPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-anomaly-detection - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch anomaly detector plugin - classname: org.opensearch.timeseries.TimeSeriesAnalyticsPlugin - custom_foldername: "" - extended_plugins: - - lang-painless - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-geospatial - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Geospatial plugin to host geospatial features - classname: org.opensearch.geospatial.plugin.GeospatialPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-system-templates - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch plugin that provides basic application based configuration templates. - classname: org.opensearch.system.applicationtemplates.ApplicationBasedConfigurationSystemTemplatesPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-performance-analyzer - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Performance Analyzer Plugin - classname: org.opensearch.performanceanalyzer.PerformanceAnalyzerPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-ml - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: machine learning plugin for opensearch - classname: org.opensearch.ml.plugin.MachineLearningPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-asynchronous-search - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Provides support for asynchronous search - classname: org.opensearch.search.asynchronous.plugin.AsynchronousSearchPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-skills - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Skills - classname: org.opensearch.agent.ToolPlugin - custom_foldername: "" - extended_plugins: - - opensearch-ml - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-sql - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch SQL - classname: org.opensearch.sql.plugin.SQLPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-notifications - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Notifications Plugin - classname: org.opensearch.notifications.NotificationPlugin - custom_foldername: "" - extended_plugins: - - opensearch-notifications-core - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-ltr - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Learning to Rank Query w/ RankLib Models - classname: com.o19s.es.ltr.LtrQueryParserPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-neural-search - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: A plugin that adds dense neural retrieval into the OpenSearch ecosytem - classname: org.opensearch.neuralsearch.plugin.NeuralSearch - custom_foldername: "" - extended_plugins: - - opensearch-knn - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-security - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Provide access control related features for OpenSearch - classname: org.opensearch.security.OpenSearchSecurityPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-flow-framework - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch plugin that enables builders to innovate AI apps on OpenSearch - classname: org.opensearch.flowframework.FlowFrameworkPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-index-management - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Index Management Plugin - classname: org.opensearch.indexmanagement.IndexManagementPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: query-insights - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Query Insights plugin - classname: org.opensearch.plugin.insights.QueryInsightsPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-job-scheduler - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Job Scheduler plugin - classname: org.opensearch.jobscheduler.JobSchedulerPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-notifications-core - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Notifications Core Plugin - classname: org.opensearch.notifications.core.NotificationCorePlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-knn - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch k-NN plugin - classname: org.opensearch.knn.plugin.KNNPlugin - custom_foldername: "" - extended_plugins: - - lang-painless - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-observability - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Plugin for OpenSearch Dashboards Observability - classname: org.opensearch.observability.ObservabilityPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-security-analytics - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Security Analytics plugin - classname: org.opensearch.securityanalytics.SecurityAnalyticsPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - - opensearch-alerting - has_native_controller: false - optional_extended_plugins: [] + contains: + - analysis-phonenumber + - optional_extended_plugins \ No newline at end of file diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index e31047581..38749ca0c 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -70,7 +70,8 @@ export default class ChapterEvaluator { const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED } const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } } const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload - const payload_body_evaluation = status.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED } + const response_contains: string[] | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.contains) : chapter.response?.contains + const payload_body_evaluation = status.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload, response_contains) : { result: Result.SKIPPED } if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`) diff --git a/tools/src/tester/ResponsePayloadEvaluator.ts b/tools/src/tester/ResponsePayloadEvaluator.ts index a3ca8ad7c..9662ec1a0 100644 --- a/tools/src/tester/ResponsePayloadEvaluator.ts +++ b/tools/src/tester/ResponsePayloadEvaluator.ts @@ -21,19 +21,35 @@ export default class ResponsePayloadEvaluator { this.logger = logger } - evaluate(response: ActualResponse, expected_payload?: Payload): Evaluation { - if (expected_payload == null) return { result: Result.PASSED } + evaluate(response: ActualResponse, expected_payload?: Payload, response_contains?: string[]): Evaluation { const payload = response.payload this.logger.info(`${to_json(payload)}`) - const delta = atomizeChangeset(diff(expected_payload, payload)) - const messages: string[] = _.compact(delta.map((value, _index, _array) => { - switch (value.type) { - case Operation.UPDATE: - return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'` - case Operation.REMOVE: - return `missing ${value.path.replace('$.', '')}='${value.value}'` - } - })) + + let messages: string[] = []; + + if (response_contains && response_contains.length > 0) { + const payloadStr = JSON.stringify(payload); + const missingValues = response_contains.filter(value => !payloadStr.includes(value)); + + if (missingValues.length > 0) { + messages.push(`Response payload is missing required values: ${missingValues.join(', ')}`); + } + } + + if (!!expected_payload) { + const delta = atomizeChangeset(diff(expected_payload, payload)) + const diffMessages: string[] = _.compact(delta.map((value, _index, _array) => { + switch (value.type) { + case Operation.UPDATE: + return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'` + case Operation.REMOVE: + return `missing ${value.path.replace('$.', '')}='${value.value}'` + } + })) + + messages = [...messages, ...diffMessages]; + } + return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED } } } \ No newline at end of file diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts index fd8d4ce2b..e4f666b2c 100644 --- a/tools/src/tester/SupplementalChapterEvaluator.ts +++ b/tools/src/tester/SupplementalChapterEvaluator.ts @@ -53,7 +53,8 @@ export default class SupplementalChapterEvaluator { const status = chapter.status ?? [200, 201] const overall = status.includes(response.status) ? { result: Result.PASSED } : { result: Result.ERROR, message: response.message, error: response.error as Error } const response_payload: Payload | undefined = overall.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload - const payload_body_evaluation = overall.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED } + const response_contains: string[] | undefined = overall.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.contains) : chapter.response?.contains + const payload_body_evaluation = overall.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload, response_contains) : { result: Result.SKIPPED } const result: Result = overall_result(_.compact([overall, payload_body_evaluation, output_values_evaluation.evaluation])) var evaluation_result: EvaluationWithOutput = { evaluation: { result } } diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index c0c40cbbc..e6184bc36 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -192,6 +192,7 @@ export interface ExpectedResponse { status: number; content_type?: string; payload?: Payload; + contains?: string[]; } /** * This interface was referenced by `Story`'s JSON-Schema diff --git a/tools/tests/tester/ResponsePayloadEvaluator.test.ts b/tools/tests/tester/ResponsePayloadEvaluator.test.ts index d8d904337..05fcccec3 100644 --- a/tools/tests/tester/ResponsePayloadEvaluator.test.ts +++ b/tools/tests/tester/ResponsePayloadEvaluator.test.ts @@ -25,15 +25,19 @@ describe('ResponsePayloadEvaluator', () => { describe('evaluate', () => { test('succeeds without an expected payload', () => { - expect(evaluator.evaluate(create_response({}), undefined)).toEqual({ result: Result.PASSED }) + expect(evaluator.evaluate(create_response({}), undefined, undefined)).toEqual({ result: Result.PASSED }) }) test('fails with a non-matching payload', () => { - expect(evaluator.evaluate(create_response({}), { x: 1 })).toEqual({ result: Result.FAILED, message: "missing x='1'" }) + expect(evaluator.evaluate(create_response({}), { x: 1 }, undefined)).toEqual({ result: Result.FAILED, message: "missing x='1'" }) }) test('succeeds with a matching payload', () => { - expect(evaluator.evaluate(create_response({ x: 1 }), { x: 1 })).toEqual({ result: Result.PASSED }) + expect(evaluator.evaluate(create_response({ x: 1 }), { x: 1 }, undefined)).toEqual({ result: Result.PASSED }) + }) + + test('succeeds with a matching payload using contains', () => { + expect(evaluator.evaluate(create_response({ x: 1 }), undefined, ["x"])).toEqual({ result: Result.PASSED }) }) }) }) From e555bc293b9f356cfdd2ac5871b52911118f9718 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sat, 22 Feb 2025 20:02:20 -0500 Subject: [PATCH 06/18] Add additional unit test Signed-off-by: Craig Perkins --- tools/tests/tester/ResponsePayloadEvaluator.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/tests/tester/ResponsePayloadEvaluator.test.ts b/tools/tests/tester/ResponsePayloadEvaluator.test.ts index 05fcccec3..ecbc3eef6 100644 --- a/tools/tests/tester/ResponsePayloadEvaluator.test.ts +++ b/tools/tests/tester/ResponsePayloadEvaluator.test.ts @@ -39,5 +39,9 @@ describe('ResponsePayloadEvaluator', () => { test('succeeds with a matching payload using contains', () => { expect(evaluator.evaluate(create_response({ x: 1 }), undefined, ["x"])).toEqual({ result: Result.PASSED }) }) + + test('fails with a non-matching payload using container', () => { + expect(evaluator.evaluate(create_response({ x: 1 }), undefined, ["y"])).toEqual({ result: Result.FAILED, message: "Response payload is missing required values: y" }) + }) }) }) From 1c434d5a6e9a410f9d11501c2ffdfcb841ad8ff6 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sat, 22 Feb 2025 21:57:11 -0500 Subject: [PATCH 07/18] Revert "Add additional unit test" This reverts commit e555bc293b9f356cfdd2ac5871b52911118f9718. Signed-off-by: Craig Perkins --- tools/tests/tester/ResponsePayloadEvaluator.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/tests/tester/ResponsePayloadEvaluator.test.ts b/tools/tests/tester/ResponsePayloadEvaluator.test.ts index ecbc3eef6..05fcccec3 100644 --- a/tools/tests/tester/ResponsePayloadEvaluator.test.ts +++ b/tools/tests/tester/ResponsePayloadEvaluator.test.ts @@ -39,9 +39,5 @@ describe('ResponsePayloadEvaluator', () => { test('succeeds with a matching payload using contains', () => { expect(evaluator.evaluate(create_response({ x: 1 }), undefined, ["x"])).toEqual({ result: Result.PASSED }) }) - - test('fails with a non-matching payload using container', () => { - expect(evaluator.evaluate(create_response({ x: 1 }), undefined, ["y"])).toEqual({ result: Result.FAILED, message: "Response payload is missing required values: y" }) - }) }) }) From 5c3aed62a39d7f9908cf952a5792e5352bb04301 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sat, 22 Feb 2025 21:57:21 -0500 Subject: [PATCH 08/18] Revert "Add mechanism to assert that response contains given a list of target values" This reverts commit e9c9eb13552ba878703c3f90ee0a03656c2eb7c7. Signed-off-by: Craig Perkins --- json_schemas/test_story.schema.yaml | 4 - .../analysis/indices/cluster/stats.yaml | 280 +++++++++++++++++- tools/src/tester/ChapterEvaluator.ts | 3 +- tools/src/tester/ResponsePayloadEvaluator.ts | 38 +-- .../tester/SupplementalChapterEvaluator.ts | 3 +- tools/src/tester/types/story.types.ts | 1 - .../tester/ResponsePayloadEvaluator.test.ts | 10 +- 7 files changed, 293 insertions(+), 46 deletions(-) diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 21cc3111c..a0bc7ba23 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -196,10 +196,6 @@ definitions: content_type: type: string default: application/json - contains: - type: array - items: - type: string payload: $ref: '#/definitions/Payload' required: [status] diff --git a/tests/plugins/analysis/indices/cluster/stats.yaml b/tests/plugins/analysis/indices/cluster/stats.yaml index 3111a3d22..8786a36f8 100644 --- a/tests/plugins/analysis/indices/cluster/stats.yaml +++ b/tests/plugins/analysis/indices/cluster/stats.yaml @@ -11,6 +11,280 @@ chapters: method: GET response: status: 200 - contains: - - analysis-phonenumber - - optional_extended_plugins \ No newline at end of file + payload: + nodes: + plugins: + - name: opensearch-cross-cluster-replication + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Cross Cluster Replication Plugin + classname: org.opensearch.replication.ReplicationPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-custom-codecs + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch plugin that implements custom compression codecs + classname: org.opensearch.index.codec.customcodecs.CustomCodecPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-reports-scheduler + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Scheduler for Dashboards Reports Plugin + classname: org.opensearch.reportsscheduler.ReportsSchedulerPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: analysis-phonenumber + version: "2.19.0" + opensearch_version: "2.19.0" + java_version: "11" + description: "Adds an analyzer for phone numbers to OpenSearch." + classname: org.opensearch.analysis.phone.PhoneNumberAnalysisPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-alerting + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Amazon OpenSearch alerting plugin + classname: org.opensearch.alerting.AlertingPlugin + custom_foldername: "" + extended_plugins: + - lang-painless + has_native_controller: false + optional_extended_plugins: [] + - name: analysis-icu + version: "2.19.0" + opensearch_version: "2.19.0" + java_version: "11" + description: "The ICU Analysis plugin integrates the Lucene ICU module into OpenSearch, adding ICU-related analysis components." + classname: org.opensearch.plugin.analysis.icu.AnalysisICUPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-anomaly-detection + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch anomaly detector plugin + classname: org.opensearch.timeseries.TimeSeriesAnalyticsPlugin + custom_foldername: "" + extended_plugins: + - lang-painless + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-geospatial + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Geospatial plugin to host geospatial features + classname: org.opensearch.geospatial.plugin.GeospatialPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-system-templates + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch plugin that provides basic application based configuration templates. + classname: org.opensearch.system.applicationtemplates.ApplicationBasedConfigurationSystemTemplatesPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-performance-analyzer + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Performance Analyzer Plugin + classname: org.opensearch.performanceanalyzer.PerformanceAnalyzerPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-ml + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: machine learning plugin for opensearch + classname: org.opensearch.ml.plugin.MachineLearningPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-asynchronous-search + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Provides support for asynchronous search + classname: org.opensearch.search.asynchronous.plugin.AsynchronousSearchPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-skills + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Skills + classname: org.opensearch.agent.ToolPlugin + custom_foldername: "" + extended_plugins: + - opensearch-ml + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-sql + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch SQL + classname: org.opensearch.sql.plugin.SQLPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-notifications + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Notifications Plugin + classname: org.opensearch.notifications.NotificationPlugin + custom_foldername: "" + extended_plugins: + - opensearch-notifications-core + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-ltr + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Learning to Rank Query w/ RankLib Models + classname: com.o19s.es.ltr.LtrQueryParserPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-neural-search + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: A plugin that adds dense neural retrieval into the OpenSearch ecosytem + classname: org.opensearch.neuralsearch.plugin.NeuralSearch + custom_foldername: "" + extended_plugins: + - opensearch-knn + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-security + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: Provide access control related features for OpenSearch + classname: org.opensearch.security.OpenSearchSecurityPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-flow-framework + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch plugin that enables builders to innovate AI apps on OpenSearch + classname: org.opensearch.flowframework.FlowFrameworkPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-index-management + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Index Management Plugin + classname: org.opensearch.indexmanagement.IndexManagementPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + has_native_controller: false + optional_extended_plugins: [] + - name: query-insights + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Query Insights plugin + classname: org.opensearch.plugin.insights.QueryInsightsPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-job-scheduler + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Job Scheduler plugin + classname: org.opensearch.jobscheduler.JobSchedulerPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-notifications-core + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Notifications Core Plugin + classname: org.opensearch.notifications.core.NotificationCorePlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-knn + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch k-NN plugin + classname: org.opensearch.knn.plugin.KNNPlugin + custom_foldername: "" + extended_plugins: + - lang-painless + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-observability + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Plugin for OpenSearch Dashboards Observability + classname: org.opensearch.observability.ObservabilityPlugin + custom_foldername: "" + extended_plugins: [] + has_native_controller: false + optional_extended_plugins: [] + - name: opensearch-security-analytics + version: "2.19.0.0" + opensearch_version: "2.19.0" + java_version: "11" + description: OpenSearch Security Analytics plugin + classname: org.opensearch.securityanalytics.SecurityAnalyticsPlugin + custom_foldername: "" + extended_plugins: + - opensearch-job-scheduler + - opensearch-alerting + has_native_controller: false + optional_extended_plugins: [] diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 38749ca0c..e31047581 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -70,8 +70,7 @@ export default class ChapterEvaluator { const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED } const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } } const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload - const response_contains: string[] | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.contains) : chapter.response?.contains - const payload_body_evaluation = status.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload, response_contains) : { result: Result.SKIPPED } + const payload_body_evaluation = status.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED } if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`) diff --git a/tools/src/tester/ResponsePayloadEvaluator.ts b/tools/src/tester/ResponsePayloadEvaluator.ts index 9662ec1a0..a3ca8ad7c 100644 --- a/tools/src/tester/ResponsePayloadEvaluator.ts +++ b/tools/src/tester/ResponsePayloadEvaluator.ts @@ -21,35 +21,19 @@ export default class ResponsePayloadEvaluator { this.logger = logger } - evaluate(response: ActualResponse, expected_payload?: Payload, response_contains?: string[]): Evaluation { + evaluate(response: ActualResponse, expected_payload?: Payload): Evaluation { + if (expected_payload == null) return { result: Result.PASSED } const payload = response.payload this.logger.info(`${to_json(payload)}`) - - let messages: string[] = []; - - if (response_contains && response_contains.length > 0) { - const payloadStr = JSON.stringify(payload); - const missingValues = response_contains.filter(value => !payloadStr.includes(value)); - - if (missingValues.length > 0) { - messages.push(`Response payload is missing required values: ${missingValues.join(', ')}`); - } - } - - if (!!expected_payload) { - const delta = atomizeChangeset(diff(expected_payload, payload)) - const diffMessages: string[] = _.compact(delta.map((value, _index, _array) => { - switch (value.type) { - case Operation.UPDATE: - return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'` - case Operation.REMOVE: - return `missing ${value.path.replace('$.', '')}='${value.value}'` - } - })) - - messages = [...messages, ...diffMessages]; - } - + const delta = atomizeChangeset(diff(expected_payload, payload)) + const messages: string[] = _.compact(delta.map((value, _index, _array) => { + switch (value.type) { + case Operation.UPDATE: + return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'` + case Operation.REMOVE: + return `missing ${value.path.replace('$.', '')}='${value.value}'` + } + })) return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED } } } \ No newline at end of file diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts index e4f666b2c..fd8d4ce2b 100644 --- a/tools/src/tester/SupplementalChapterEvaluator.ts +++ b/tools/src/tester/SupplementalChapterEvaluator.ts @@ -53,8 +53,7 @@ export default class SupplementalChapterEvaluator { const status = chapter.status ?? [200, 201] const overall = status.includes(response.status) ? { result: Result.PASSED } : { result: Result.ERROR, message: response.message, error: response.error as Error } const response_payload: Payload | undefined = overall.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload - const response_contains: string[] | undefined = overall.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.contains) : chapter.response?.contains - const payload_body_evaluation = overall.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload, response_contains) : { result: Result.SKIPPED } + const payload_body_evaluation = overall.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED } const result: Result = overall_result(_.compact([overall, payload_body_evaluation, output_values_evaluation.evaluation])) var evaluation_result: EvaluationWithOutput = { evaluation: { result } } diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index e6184bc36..c0c40cbbc 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -192,7 +192,6 @@ export interface ExpectedResponse { status: number; content_type?: string; payload?: Payload; - contains?: string[]; } /** * This interface was referenced by `Story`'s JSON-Schema diff --git a/tools/tests/tester/ResponsePayloadEvaluator.test.ts b/tools/tests/tester/ResponsePayloadEvaluator.test.ts index 05fcccec3..d8d904337 100644 --- a/tools/tests/tester/ResponsePayloadEvaluator.test.ts +++ b/tools/tests/tester/ResponsePayloadEvaluator.test.ts @@ -25,19 +25,15 @@ describe('ResponsePayloadEvaluator', () => { describe('evaluate', () => { test('succeeds without an expected payload', () => { - expect(evaluator.evaluate(create_response({}), undefined, undefined)).toEqual({ result: Result.PASSED }) + expect(evaluator.evaluate(create_response({}), undefined)).toEqual({ result: Result.PASSED }) }) test('fails with a non-matching payload', () => { - expect(evaluator.evaluate(create_response({}), { x: 1 }, undefined)).toEqual({ result: Result.FAILED, message: "missing x='1'" }) + expect(evaluator.evaluate(create_response({}), { x: 1 })).toEqual({ result: Result.FAILED, message: "missing x='1'" }) }) test('succeeds with a matching payload', () => { - expect(evaluator.evaluate(create_response({ x: 1 }), { x: 1 }, undefined)).toEqual({ result: Result.PASSED }) - }) - - test('succeeds with a matching payload using contains', () => { - expect(evaluator.evaluate(create_response({ x: 1 }), undefined, ["x"])).toEqual({ result: Result.PASSED }) + expect(evaluator.evaluate(create_response({ x: 1 }), { x: 1 })).toEqual({ result: Result.PASSED }) }) }) }) From d9a8299bd539943b14fc41a44c9a2f6dfcff4451 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sat, 22 Feb 2025 22:19:25 -0500 Subject: [PATCH 09/18] Remove test Signed-off-by: Craig Perkins --- .../analysis/indices/cluster/stats.yaml | 290 ------------------ 1 file changed, 290 deletions(-) delete mode 100644 tests/plugins/analysis/indices/cluster/stats.yaml diff --git a/tests/plugins/analysis/indices/cluster/stats.yaml b/tests/plugins/analysis/indices/cluster/stats.yaml deleted file mode 100644 index 8786a36f8..000000000 --- a/tests/plugins/analysis/indices/cluster/stats.yaml +++ /dev/null @@ -1,290 +0,0 @@ -$schema: ../../../../../json_schemas/test_story.schema.yaml - -description: Test cluster stats. -chapters: - - synopsis: Returns statistics about a cluster. - path: /_cluster/stats/{metric}/nodes/{node_id} - parameters: - metric: - - plugins - node_id: _cluster_manager - method: GET - response: - status: 200 - payload: - nodes: - plugins: - - name: opensearch-cross-cluster-replication - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Cross Cluster Replication Plugin - classname: org.opensearch.replication.ReplicationPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-custom-codecs - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch plugin that implements custom compression codecs - classname: org.opensearch.index.codec.customcodecs.CustomCodecPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-reports-scheduler - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Scheduler for Dashboards Reports Plugin - classname: org.opensearch.reportsscheduler.ReportsSchedulerPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: analysis-phonenumber - version: "2.19.0" - opensearch_version: "2.19.0" - java_version: "11" - description: "Adds an analyzer for phone numbers to OpenSearch." - classname: org.opensearch.analysis.phone.PhoneNumberAnalysisPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-alerting - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Amazon OpenSearch alerting plugin - classname: org.opensearch.alerting.AlertingPlugin - custom_foldername: "" - extended_plugins: - - lang-painless - has_native_controller: false - optional_extended_plugins: [] - - name: analysis-icu - version: "2.19.0" - opensearch_version: "2.19.0" - java_version: "11" - description: "The ICU Analysis plugin integrates the Lucene ICU module into OpenSearch, adding ICU-related analysis components." - classname: org.opensearch.plugin.analysis.icu.AnalysisICUPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-anomaly-detection - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch anomaly detector plugin - classname: org.opensearch.timeseries.TimeSeriesAnalyticsPlugin - custom_foldername: "" - extended_plugins: - - lang-painless - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-geospatial - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Geospatial plugin to host geospatial features - classname: org.opensearch.geospatial.plugin.GeospatialPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-system-templates - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch plugin that provides basic application based configuration templates. - classname: org.opensearch.system.applicationtemplates.ApplicationBasedConfigurationSystemTemplatesPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-performance-analyzer - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Performance Analyzer Plugin - classname: org.opensearch.performanceanalyzer.PerformanceAnalyzerPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-ml - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: machine learning plugin for opensearch - classname: org.opensearch.ml.plugin.MachineLearningPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-asynchronous-search - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Provides support for asynchronous search - classname: org.opensearch.search.asynchronous.plugin.AsynchronousSearchPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-skills - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Skills - classname: org.opensearch.agent.ToolPlugin - custom_foldername: "" - extended_plugins: - - opensearch-ml - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-sql - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch SQL - classname: org.opensearch.sql.plugin.SQLPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-notifications - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Notifications Plugin - classname: org.opensearch.notifications.NotificationPlugin - custom_foldername: "" - extended_plugins: - - opensearch-notifications-core - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-ltr - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Learning to Rank Query w/ RankLib Models - classname: com.o19s.es.ltr.LtrQueryParserPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-neural-search - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: A plugin that adds dense neural retrieval into the OpenSearch ecosytem - classname: org.opensearch.neuralsearch.plugin.NeuralSearch - custom_foldername: "" - extended_plugins: - - opensearch-knn - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-security - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: Provide access control related features for OpenSearch - classname: org.opensearch.security.OpenSearchSecurityPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-flow-framework - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch plugin that enables builders to innovate AI apps on OpenSearch - classname: org.opensearch.flowframework.FlowFrameworkPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-index-management - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Index Management Plugin - classname: org.opensearch.indexmanagement.IndexManagementPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - has_native_controller: false - optional_extended_plugins: [] - - name: query-insights - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Query Insights plugin - classname: org.opensearch.plugin.insights.QueryInsightsPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-job-scheduler - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Job Scheduler plugin - classname: org.opensearch.jobscheduler.JobSchedulerPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-notifications-core - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Notifications Core Plugin - classname: org.opensearch.notifications.core.NotificationCorePlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-knn - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch k-NN plugin - classname: org.opensearch.knn.plugin.KNNPlugin - custom_foldername: "" - extended_plugins: - - lang-painless - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-observability - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Plugin for OpenSearch Dashboards Observability - classname: org.opensearch.observability.ObservabilityPlugin - custom_foldername: "" - extended_plugins: [] - has_native_controller: false - optional_extended_plugins: [] - - name: opensearch-security-analytics - version: "2.19.0.0" - opensearch_version: "2.19.0" - java_version: "11" - description: OpenSearch Security Analytics plugin - classname: org.opensearch.securityanalytics.SecurityAnalyticsPlugin - custom_foldername: "" - extended_plugins: - - opensearch-job-scheduler - - opensearch-alerting - has_native_controller: false - optional_extended_plugins: [] From 513fc0b2ddacade8aaaf3d98bf17c23789e81db3 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sun, 23 Feb 2025 21:29:19 -0500 Subject: [PATCH 10/18] Remove from required and update sha Signed-off-by: Craig Perkins --- .github/workflows/test-spec.yml | 2 +- spec/schemas/_common.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index e705a199d..2a1c4ea80 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -61,7 +61,7 @@ jobs: key: tests/plugins/security/.kirk-key.pem - version: 2.20.0 hub: opensearchstaging - ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' + ref: '@sha256:037a2eb5a4f48b9a9a29950be5e5fae4ebd82d41de1066e00bb8d8b0ce7871b1' - version: 3.0.0 hub: opensearchstaging ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de' diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index bddc76358..f25bd0294 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1528,7 +1528,6 @@ components: - java_version - name - opensearch_version - - optional_extended_plugins - version NodeStatistics: type: object From f419abc3e4f4d621ea97e7eda017cc5574f02c62 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 27 Feb 2025 13:39:53 +1300 Subject: [PATCH 11/18] Update FlowFrameworkDeleteResponse to use WriteResponseBase Signed-off-by: Thomas Farr --- CHANGELOG.md | 3 ++- spec/schemas/_common.yaml | 1 + spec/schemas/flow_framework.common.yaml | 27 ++++--------------------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ec29801c..bea8b40fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `POST /_plugins/_ml/_execute/{algorithm_name}` ([#811](https://github.com/opensearch-project/opensearch-api-specification/pull/811)) - Added search suggester types ([#817](https://github.com/opensearch-project/opensearch-api-specification/pull/817)) - Added `SimplePatternTokenizer` and `SimplePatternSplitTokenizer` ([#820](https://github.com/opensearch-project/opensearch-api-specification/pull/820)) +- Added `optional_extended_plugins` field to `PluginStats` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) @@ -120,7 +121,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Changed naming of `snapshot._common`'s `Status`, `ShardsStats`, `ShardsStatsStage`, `ShardsStatsSummary` and `ShardsStatsSummaryItem` schemas to be prefixed with `Snapshot` ([#730](https://github.com/opensearch-project/opensearch-api-specification/pull/730)) - Changed `ml.get_memory` and `ml.get_message` to split out `get_all` variants ([#796](https://github.com/opensearch-project/opensearch-api-specification/pull/796)) - Changed `ml.get_tools` to have two different operation groups `ml.get_all_tools` and `ml.get_tool` ([#799](https://github.com/opensearch-project/opensearch-api-specification/pull/799)) -- Update `PluginStats` with new field in response `optional_extended_plugins` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) +- Changed `FlowFrameworkDeleteResponse` to utilize `WriteResponseBase` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) ## [0.1.0] - 2024-10-25 diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index f25bd0294..698f02dd5 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1520,6 +1520,7 @@ components: type: array items: type: string + x-version-added: '2.19' required: - classname - description diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index d31caa059..a478e0261 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -77,29 +77,10 @@ components: $ref: '#/components/schemas/version' minProperties: 1 FlowFrameworkDeleteResponse: - type: object - properties: - _index: - type: string - description: The name of the index where the document was stored. - _id: - type: string - description: The unique identifier of the deleted document. - _version: - type: integer - description: The version of the document after the deletion. - result: - type: string - description: The result of the deletion operation. - enum: [deleted, not_found] - _shards: - $ref: '#/components/schemas/shards' - _seq_no: - type: integer - description: The sequence number assigned to the document after the deletion. - _primary_term: - type: integer - description: The primary term assigned to the document after the deletion. + allOf: + - $ref: '_common.yaml#/components/schemas/WriteResponseBase' + - type: object + properties: {} FlowFrameworkGetResponse: type: object properties: From 6f7acf84293278b5e5577b515123c480c20ce4b9 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 27 Feb 2025 13:40:19 +1300 Subject: [PATCH 12/18] Fix k-NN model training tests to work with small training sample set Signed-off-by: Thomas Farr --- tests/default/knn/models.yaml | 13 +++++++++++++ tests/default/knn/models/search.yaml | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/default/knn/models.yaml b/tests/default/knn/models.yaml index 3589fc162..4cfa893a5 100644 --- a/tests/default/knn/models.yaml +++ b/tests/default/knn/models.yaml @@ -27,6 +27,10 @@ prologues: - {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2} - {index: {_index: movies, _id: '2'}} - {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1} + - {index: {_index: movies, _id: '3'}} + - {recommendation_vector: [3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5], duration: 4.8} + - {index: {_index: movies, _id: '4'}} + - {recommendation_vector: [4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5], duration: 18.6} status: [200] - method: POST path: /_plugins/_knn/models/model-1/_train @@ -41,6 +45,15 @@ prologues: mode: on_disk compression_level: 32x spaceType: l2 + method: + name: ivf + engine: faiss + parameters: + nlist: 4 + encoder: + name: pq + parameters: + code_size: 2 status: [200] epilogues: - path: /movies diff --git a/tests/default/knn/models/search.yaml b/tests/default/knn/models/search.yaml index 52711b0f5..d608b878e 100644 --- a/tests/default/knn/models/search.yaml +++ b/tests/default/knn/models/search.yaml @@ -27,6 +27,10 @@ prologues: - {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2} - {index: {_index: movies, _id: '2'}} - {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1} + - {index: {_index: movies, _id: '3'}} + - {recommendation_vector: [3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5], duration: 4.8} + - {index: {_index: movies, _id: '4'}} + - {recommendation_vector: [4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5], duration: 18.6} status: [200] - method: POST path: /_plugins/_knn/models/model-1/_train @@ -41,6 +45,15 @@ prologues: mode: on_disk compression_level: 32x spaceType: l2 + method: + name: ivf + engine: faiss + parameters: + nlist: 4 + encoder: + name: pq + parameters: + code_size: 2 status: [200] epilogues: - path: /movies From 076e65758d43b12da381d8e2c4e2c4661723d58a Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 27 Feb 2025 13:58:12 +1300 Subject: [PATCH 13/18] Add `id` and `group_by` to `TopQuery` Signed-off-by: Thomas Farr --- CHANGELOG.md | 1 + spec/schemas/insights._common.yaml | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bea8b40fa..4c093c399 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added search suggester types ([#817](https://github.com/opensearch-project/opensearch-api-specification/pull/817)) - Added `SimplePatternTokenizer` and `SimplePatternSplitTokenizer` ([#820](https://github.com/opensearch-project/opensearch-api-specification/pull/820)) - Added `optional_extended_plugins` field to `PluginStats` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) +- Added `id` & `group_by` fields to `TopQuery` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/spec/schemas/insights._common.yaml b/spec/schemas/insights._common.yaml index 26907f160..b30ceb977 100644 --- a/spec/schemas/insights._common.yaml +++ b/spec/schemas/insights._common.yaml @@ -19,6 +19,10 @@ components: TopQuery: type: object properties: + id: + type: string + group_by: + $ref: '#/components/schemas/GroupingType' timestamp: type: integer description: The timestamp of the query execution. @@ -56,6 +60,13 @@ components: measurements: type: object $ref: '#/components/schemas/Measurements' + GroupingType: + type: string + enum: + - NONE + - SIMILARITY + - none + - similarity TaskResourceUsages: type: object properties: From 065fe261eb61529030cb089e7df6de8aa5f2791b Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 27 Feb 2025 14:02:16 +1300 Subject: [PATCH 14/18] Even smaller k-NN model training Signed-off-by: Thomas Farr --- tests/default/knn/models.yaml | 8 ++------ tests/default/knn/models/search.yaml | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/default/knn/models.yaml b/tests/default/knn/models.yaml index 4cfa893a5..17409c4d8 100644 --- a/tests/default/knn/models.yaml +++ b/tests/default/knn/models.yaml @@ -27,10 +27,6 @@ prologues: - {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2} - {index: {_index: movies, _id: '2'}} - {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1} - - {index: {_index: movies, _id: '3'}} - - {recommendation_vector: [3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5], duration: 4.8} - - {index: {_index: movies, _id: '4'}} - - {recommendation_vector: [4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5], duration: 18.6} status: [200] - method: POST path: /_plugins/_knn/models/model-1/_train @@ -49,11 +45,11 @@ prologues: name: ivf engine: faiss parameters: - nlist: 4 + nlist: 2 encoder: name: pq parameters: - code_size: 2 + code_size: 1 status: [200] epilogues: - path: /movies diff --git a/tests/default/knn/models/search.yaml b/tests/default/knn/models/search.yaml index d608b878e..d26d0cf0e 100644 --- a/tests/default/knn/models/search.yaml +++ b/tests/default/knn/models/search.yaml @@ -27,10 +27,6 @@ prologues: - {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2} - {index: {_index: movies, _id: '2'}} - {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1} - - {index: {_index: movies, _id: '3'}} - - {recommendation_vector: [3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5], duration: 4.8} - - {index: {_index: movies, _id: '4'}} - - {recommendation_vector: [4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5], duration: 18.6} status: [200] - method: POST path: /_plugins/_knn/models/model-1/_train @@ -49,11 +45,11 @@ prologues: name: ivf engine: faiss parameters: - nlist: 4 + nlist: 2 encoder: name: pq parameters: - code_size: 2 + code_size: 1 status: [200] epilogues: - path: /movies From 490f99437aae664bb795bf27e7d2222d96509ed5 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 27 Feb 2025 14:46:21 +1300 Subject: [PATCH 15/18] Fix k-NN tests Signed-off-by: Thomas Farr --- CHANGELOG.md | 1 + spec/namespaces/knn.yaml | 6 ++--- spec/schemas/knn._common.yaml | 35 ++++++++++++---------------- tests/default/knn/models/search.yaml | 10 ++++++++ tests/default/knn/models/train.yaml | 32 +++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c093c399..36f27fae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed GeoDistanceSort schema ([#805](https://github.com/opensearch-project/opensearch-api-specification/pull/805)) - Fixed Aggregation schemas ([#801](https://github.com/opensearch-project/opensearch-api-specification/pull/801)) - Fixed FilterQueryRequestProcessor to use correct query type ([#821](https://github.com/opensearch-project/opensearch-api-specification/pull/821)) +- Fixed `knn.train_model`'s request body `method` field to accept an object rather than a string ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) ### Changed - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683)) diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index c547a7894..5fc8cf768 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -390,7 +390,7 @@ components: in: query description: The default operator for query string query (AND or OR). schema: - $ref: '../schemas/knn._common.yaml#/components/schemas/DefaultOperator' + $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/Operator' knn.search_models::query.df: name: df in: query @@ -518,7 +518,7 @@ components: in: query description: Search operation type. schema: - $ref: '../schemas/knn._common.yaml#/components/schemas/SearchType' + $ref: '../schemas/_common.yaml#/components/schemas/SearchType' knn.search_models::query.seq_no_primary_term: name: seq_no_primary_term in: query @@ -580,7 +580,7 @@ components: in: query description: Specify suggest mode. schema: - $ref: '../schemas/knn._common.yaml#/components/schemas/SuggestMode' + $ref: '../schemas/_common.yaml#/components/schemas/SuggestMode' knn.search_models::query.suggest_size: name: suggest_size in: query diff --git a/spec/schemas/knn._common.yaml b/spec/schemas/knn._common.yaml index 3efddc192..d0a6140be 100644 --- a/spec/schemas/knn._common.yaml +++ b/spec/schemas/knn._common.yaml @@ -6,25 +6,6 @@ info: paths: {} components: schemas: - DefaultOperator: - type: string - description: The default operator for query string query (AND or OR). - enum: - - AND - - OR - SearchType: - type: string - description: Search operation type. - enum: - - dfs_query_then_fetch - - query_then_fetch - SuggestMode: - type: string - description: Specify suggest mode. - enum: - - always - - missing - - popular Stats: type: object properties: @@ -143,6 +124,20 @@ components: $ref: '_common.yaml#/components/schemas/ByteCount' current_size_in_bytes: $ref: '_common.yaml#/components/schemas/ByteCount' + KnnMethod: + type: object + properties: + name: + type: string + space_type: + type: string + engine: + type: string + parameters: + type: object + additionalProperties: true + required: + - name TrainedModel: type: object properties: @@ -166,7 +161,7 @@ components: compression_level: type: string method: - type: string + $ref: '#/components/schemas/KnnMethod' spaceType: type: string required: diff --git a/tests/default/knn/models/search.yaml b/tests/default/knn/models/search.yaml index d26d0cf0e..a67bac7e6 100644 --- a/tests/default/knn/models/search.yaml +++ b/tests/default/knn/models/search.yaml @@ -51,6 +51,16 @@ prologues: parameters: code_size: 1 status: [200] + - method: GET + path: /_plugins/_knn/models/model-1 + retry: + count: 3 + response: + status: 200 + payload: + model_id: model-1 + state: created + status: [200] epilogues: - path: /movies method: DELETE diff --git a/tests/default/knn/models/train.yaml b/tests/default/knn/models/train.yaml index fbcfb19d5..91fe9e787 100644 --- a/tests/default/knn/models/train.yaml +++ b/tests/default/knn/models/train.yaml @@ -70,6 +70,15 @@ chapters: mode: on_disk compression_level: 32x spaceType: l2 + method: + name: ivf + engine: faiss + parameters: + nlist: 9 + encoder: + name: pq + parameters: + code_size: 3 response: status: 200 output: @@ -104,5 +113,28 @@ chapters: mode: on_disk compression_level: 32x spaceType: l2 + method: + name: ivf + engine: faiss + parameters: + nlist: 9 + encoder: + name: pq + parameters: + code_size: 3 response: status: 200 + - synopsis: Wait for the model to get trained (2). + warnings: + multiple-paths-detected: false + method: GET + path: /_plugins/_knn/models/{model_id} + parameters: + model_id: model-1 + retry: + count: 3 + response: + status: 200 + payload: + model_id: model-1 + state: created \ No newline at end of file From 50f81b7e387667b9a766d44d5ed9f2a82fb145ce Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 27 Feb 2025 16:54:06 +1300 Subject: [PATCH 16/18] Fix flow framework test Signed-off-by: Thomas Farr --- spec/schemas/flow_framework.common.yaml | 27 ++++++++++++------- .../flow_framework/workflow/state/search.yaml | 9 ++++--- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index a478e0261..b6da56940 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -129,14 +129,6 @@ components: skipped: type: integer description: The number of shards where the deletion was skipped. - query: - type: object - description: The search query to match workflows. Use `match_all` to retrieve all workflows, or `match` to search by specific fields like `use_case`. - properties: - match: - type: object - match_all: - type: object total: type: object properties: @@ -205,6 +197,23 @@ components: type: string user: $ref: '#/components/schemas/user' + provision_start_time: + $ref: '_common.yaml#/components/schemas/DateTime' + provision_end_time: + $ref: '_common.yaml#/components/schemas/DateTime' + resources_created: + $ref: '#/components/schemas/ResourcesCreated' + ResourcesCreated: + type: object + properties: + workflow_step_name: + type: string + workflow_step_id: + type: string + resource_type: + type: string + resource_id: + type: string user: type: object properties: @@ -348,7 +357,7 @@ components: type: object properties: query: - $ref: '#/components/schemas/query' + $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' WorkflowSearchResponse: type: object properties: diff --git a/tests/default/flow_framework/workflow/state/search.yaml b/tests/default/flow_framework/workflow/state/search.yaml index 67138eb1e..5f5aa0f76 100644 --- a/tests/default/flow_framework/workflow/state/search.yaml +++ b/tests/default/flow_framework/workflow/state/search.yaml @@ -18,13 +18,15 @@ prologues: output: workflow_id: payload.workflow_id chapters: - - synopsis: Search for a workflow. + - synopsis: Search for a workflow state. path: /_plugins/_flow_framework/workflow/state/_search method: GET request: payload: query: - match_all: {} + term: + workflow_id: + value: ${workflow.workflow_id} response: status: 200 payload: @@ -32,5 +34,4 @@ chapters: hits: - _index: .plugins-flow-framework-state _source: - state: NOT_STARTED - + workflow_id: ${workflow.workflow_id} \ No newline at end of file From b7b4974b2e1713e01866c36c27295c0121d435ee Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Fri, 28 Feb 2025 10:13:32 +1300 Subject: [PATCH 17/18] Split k-NN plugin tests out of default Signed-off-by: Thomas Farr --- .github/workflows/test-spec.yml | 2 ++ tests/plugins/knn/docker-compose.yml | 12 ++++++++++++ tests/{default => plugins/knn}/knn/models.yaml | 2 +- .../{default => plugins/knn}/knn/models/search.yaml | 2 +- tests/{default => plugins/knn}/knn/models/train.yaml | 2 +- tests/{default => plugins/knn}/knn/stats.yaml | 2 +- tests/{default => plugins/knn}/knn/warmup.yaml | 2 +- 7 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 tests/plugins/knn/docker-compose.yml rename tests/{default => plugins/knn}/knn/models.yaml (96%) rename tests/{default => plugins/knn}/knn/models/search.yaml (97%) rename tests/{default => plugins/knn}/knn/models/train.yaml (98%) rename tests/{default => plugins/knn}/knn/stats.yaml (92%) rename tests/{default => plugins/knn}/knn/warmup.yaml (91%) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 2a1c4ea80..98098bcf9 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -55,6 +55,8 @@ jobs: tests: plugins/workload-management - version: 2.19.0 tests: plugins/analysis + - version: 2.19.0 + tests: plugins/knn - version: 2.19.0 tests: plugins/security cert: tests/plugins/security/.kirk.pem diff --git a/tests/plugins/knn/docker-compose.yml b/tests/plugins/knn/docker-compose.yml new file mode 100644 index 000000000..04c267ec5 --- /dev/null +++ b/tests/plugins/knn/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3' + +services: + opensearch-cluster: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node diff --git a/tests/default/knn/models.yaml b/tests/plugins/knn/knn/models.yaml similarity index 96% rename from tests/default/knn/models.yaml rename to tests/plugins/knn/knn/models.yaml index 17409c4d8..d94fcf797 100644 --- a/tests/default/knn/models.yaml +++ b/tests/plugins/knn/knn/models.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Delete a k-NN model. version: '>= 2.17' diff --git a/tests/default/knn/models/search.yaml b/tests/plugins/knn/knn/models/search.yaml similarity index 97% rename from tests/default/knn/models/search.yaml rename to tests/plugins/knn/knn/models/search.yaml index a67bac7e6..1bc312cb5 100644 --- a/tests/default/knn/models/search.yaml +++ b/tests/plugins/knn/knn/models/search.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../json_schemas/test_story.schema.yaml description: Search for a k-NN model. version: '>= 2.17' diff --git a/tests/default/knn/models/train.yaml b/tests/plugins/knn/knn/models/train.yaml similarity index 98% rename from tests/default/knn/models/train.yaml rename to tests/plugins/knn/knn/models/train.yaml index 91fe9e787..a80b442cf 100644 --- a/tests/default/knn/models/train.yaml +++ b/tests/plugins/knn/knn/models/train.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../json_schemas/test_story.schema.yaml description: Train a k-NN model with disk-based parameters. version: '>= 2.17' diff --git a/tests/default/knn/stats.yaml b/tests/plugins/knn/knn/stats.yaml similarity index 92% rename from tests/default/knn/stats.yaml rename to tests/plugins/knn/knn/stats.yaml index f1fb9c5c9..65e0ff6df 100644 --- a/tests/default/knn/stats.yaml +++ b/tests/plugins/knn/knn/stats.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Get k-NN stats. chapters: diff --git a/tests/default/knn/warmup.yaml b/tests/plugins/knn/knn/warmup.yaml similarity index 91% rename from tests/default/knn/warmup.yaml rename to tests/plugins/knn/knn/warmup.yaml index 656e67ee6..937ff2f38 100644 --- a/tests/default/knn/warmup.yaml +++ b/tests/plugins/knn/knn/warmup.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Load all the native library files for all the shards into native memory. prologues: From 68942c6024e63e304716aeb63d4f40c9cbb6c224 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Fri, 28 Feb 2025 11:19:07 +1300 Subject: [PATCH 18/18] Simplify flow framework delete response Signed-off-by: Thomas Farr --- spec/namespaces/flow_framework.yaml | 2 +- spec/schemas/flow_framework.common.yaml | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index b0a47a30a..777e6a2e6 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -444,7 +444,7 @@ components: content: application/json: schema: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkDeleteResponse' + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' flow_framework.get@200: content: application/json: diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index b6da56940..b3fc551d2 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -76,11 +76,6 @@ components: version: $ref: '#/components/schemas/version' minProperties: 1 - FlowFrameworkDeleteResponse: - allOf: - - $ref: '_common.yaml#/components/schemas/WriteResponseBase' - - type: object - properties: {} FlowFrameworkGetResponse: type: object properties: