Merge branch 'main' into postman-generation

Signed-off-by: Niyazbek Torekeldi <tokesh789@gmail.com>
opensearch-project · Jan 27, 2025 · 229fe06 · 229fe06
2 parents 11e510b + 9df46f8
commit 229fe06
Show file tree

Hide file tree

Showing 164 changed files with 5,349 additions and 1,265 deletions.
diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml
@@ -207,4 +207,4 @@ jobs:
         if: github.event_name == 'pull_request'
         with:
           name: pr-comment
-          path: pr-comment.json
+          path: pr-comment.json
diff --git a/.gitignore b/.gitignore
@@ -19,3 +19,5 @@ build/
 # coverage output
 /coverage/
 
+# log files
+/logs/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,7 +23,6 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `aggregations` to `/nodes/_usage/{metric}` requests and responses ([#615](https://github.com/opensearch-project/opensearch-api-specification/pull/615))
 - Added missing `status` to `/_search/template` response ([#702](https://github.com/opensearch-project/opensearch-api-specification/pull/702))
 - Added `_type` to `rank_eval` API specs ([#704](https://github.com/opensearch-project/opensearch-api-specification/pull/704))
-- Added request body to `_search_shards` API specs ([#709](https://github.com/opensearch-project/opensearch-api-specification/pull/709))
 - Added NodeAttributes as additional property in `shard_stores` API specs ([#715](https://github.com/opensearch-project/opensearch-api-specification/pull/715))
 - Added missing `repository` query parameter to `/_cat/snapshots` ([#700](https://github.com/opensearch-project/opensearch-api-specification/pull/700))
 - Added `sm` namespace API specifications ([#701](https://github.com/opensearch-project/opensearch-api-specification/pull/701))
@@ -46,6 +45,18 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added the ability to skip an individual chapter test ([#765](https://github.com/opensearch-project/opensearch-api-specification/pull/765))
 - Added uploading of test spec logs ([#767](https://github.com/opensearch-project/opensearch-api-specification/pull/767))
 - Added generation of file for Postman ([#770](https://github.com/opensearch-project/opensearch-api-specification/pull/770))
+- Added `aggs` property as an alias to `aggregations` in requestBody of `search` [#774](https://github.com/opensearch-project/opensearch-api-specification/issues/774)
+- Added `POST /_plugins/_ml/memory`, `POST /_plugins/_ml/memory/_search`, `{memory_id}/_search`, `{memory_id}/messages`, `PUT /_plugins/_ml/memory/{memory_id}`, `message/{message_id}`,  `GET /_plugins/_ml/memory`, `GET /_plugins/_ml/memory/{memory_id}`, `_search`, `message/{message_id}`, `{memory_id}/messages`, `{memory_id}/_search`, `message/{message_id}/traces`,  and `DELETE /_plugins/_ml/memory/{memory_id}` ([#771](https://github.com/opensearch-project/opensearch-api-specification/pull/771))
+- Added support for evaluating response payloads in prologues and epilogues ([#772](https://github.com/opensearch-project/opensearch-api-specification/pull/772))
+- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `POST /_plugins/_ml/models/_unload`, `_undeploy`, `_upload`, `meta`, `_register_meta`, `POST /_plugins/_ml/models/{model_id}/_load`, `_predict`, `_unload`, `chunk/{chunk_number}`, `upload_chunk/{chunk_number}`, and `PUT /_plugins/_ml/models/{model_id}` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733))
+- Added `GET`, `POST`, `PUT`, `DELETE /_plugins/_ml/controllers/{model_id}` ([#779](https://github.com/opensearch-project/opensearch-api-specification/pull/779))
+- Added `GET /_plugins/_ml/profile`, `GET /_plugins/_ml/profile/models`, `models/{model_id}`, `tasks`, `tasks/{task_id}` ([#787](https://github.com/opensearch-project/opensearch-api-specification/pull/787))
+- Added `GET /_plugins/_ml/stats/`, `stats/{stat}`, `{nodeId}/stats/`, `{nodeId}/stats/{stat}` ([#794](https://github.com/opensearch-project/opensearch-api-specification/pull/794))
+- Added `GET`, `POST /_plugins/_ml/tasks/_search`, `GET /_plugins/_ml/tools`, `tools/{tool_name}` ([#797](https://github.com/opensearch-project/opensearch-api-specification/pull/797))
+- Added `POST /_plugins/_ml/agents/{agent_id}/_execute`, `GET /_plugins/_ml/agents/{agent_id}`, `GET`, `POST /_plugins/_ml/agents/_search` ([#798](https://github.com/opensearch-project/opensearch-api-specification/pull/798))
+- Added a warning for test file names that don't match the API being tested ([#793](https://github.com/opensearch-project/opensearch-api-specification/pull/793))
+- Added `time` field to the `GetStats` schema in `_common.yml` ([#803](https://github.com/opensearch-project/opensearch-api-specification/pull/803))
+- Added version for `POST /_plugins/_ml/_train/{algorithm_name}`, `_predict/{algorithm_name}/{model_id}`, and `_train_predict/{algorithm_name}` ([#763](https://github.com/opensearch-project/opensearch-api-specification/pull/763))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
@@ -92,12 +103,19 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed schema validation for invalid properties in `retry` configuration ([#758](https://github.com/opensearch-project/opensearch-api-specification/pull/758))
 - Fixed nodes API schemas ([#761](https://github.com/opensearch-project/opensearch-api-specification/pull/761))
 - Fixed ingest API schemas ([#766](https://github.com/opensearch-project/opensearch-api-specification/pull/766))
+- Fixed CAT API schemas ([#769](https://github.com/opensearch-project/opensearch-api-specification/pull/769))
+- Fixed core API schemas ([#777](https://github.com/opensearch-project/opensearch-api-specification/pull/777))
+- Fixed Memory Circuit Breaker flaky error in ML APIs ([#788](https://github.com/opensearch-project/opensearch-api-specification/pull/788))
+- Fixed GeoDistanceSort schema ([#805](https://github.com/opensearch-project/opensearch-api-specification/pull/805))
+- Fixed Aggregation schemas([#801](https://github.com/opensearch-project/opensearch-api-specification/pull/801))
 
 ### Changed
 - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683))
 - Changed `SearchModelsQuery`, `CreateConnectorRequest` & `RegisterAgentsRequest` to be defined inline of request bodies ([#725](https://github.com/opensearch-project/opensearch-api-specification/pull/725))
 - Changed `indices.data_streams_stats:DataStreamsStatsItem` to instead be `indices._common:DataStreamStats` ([#725](https://github.com/opensearch-project/opensearch-api-specification/pull/725)) 
 - Changed naming of `snapshot._common`'s `Status`, `ShardsStats`, `ShardsStatsStage`, `ShardsStatsSummary` and `ShardsStatsSummaryItem` schemas to be prefixed with `Snapshot` ([#730](https://github.com/opensearch-project/opensearch-api-specification/pull/730))
+- Changed `ml.get_memory` and `ml.get_message` to split out `get_all` variants ([#796](https://github.com/opensearch-project/opensearch-api-specification/pull/796))
+- Changed `ml.get_tools` to have two different operation groups `ml.get_all_tools` and `ml.get_tool` ([#799](https://github.com/opensearch-project/opensearch-api-specification/pull/799))
 
 ## [0.1.0] - 2024-10-25
 

diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md
@@ -20,6 +20,7 @@
     - [Waiting for Tasks](#waiting-for-tasks)
     - [Warnings](#warnings)
       - [multiple-paths-detected](#multiple-paths-detected)
+      - [invalid-path-detected](#invalid-path-detected)
       - [Suppressing Warnings](#suppressing-warnings)
   - [Collecting Test Coverage](#collecting-test-coverage)
     - [Coverage Summary](#coverage-summary)
@@ -358,7 +359,7 @@ For example, an ML task returns `CREATED` when created, and `COMPLETED` when it'
 
 #### multiple-paths-detected
 
-The test runner expects all tests in the same file to be variation of the same path in order to keep tests well-organized. Otherwise, a warning will be emitted.
+The test runner expects all tests in the same file to be variation of the same path in order to keep tests well-organized. Prerequisites to the API being tested should be moved to `prologues`, and any cleanup to `epilogues`. Otherwise, a warning will be emitted.
 
 ```
 WARNING Multiple paths detected, please group similar tests together and move paths not being tested to prologues or epilogues.
@@ -367,6 +368,15 @@ WARNING Multiple paths detected, please group similar tests together and move pa
   /{index}
 ```
 
+#### invalid-path-detected
+
+The test file names are expected to match the one API being tested in that file. Otherwise, a warning will be emitted.
+
+```
+PASSED  plugins/ml/ml/train_and_predict.yaml (tests/plugins/ml/ml/train_and_predict.yaml)
+WARNING Invalid path detected, please move /tests/plugins/ml/ml/train_and_predict.yaml to ml/train_predict.yaml.
+```
+
 #### Suppressing Warnings
 
 The test runner may generate warnings that can be suppressed with `warnings:` at story or chapter level. For example, to suppress the multiple paths detected warning.

diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml
@@ -105,6 +105,8 @@ definitions:
         $ref: '#/definitions/Distributions'
       retry:
         $ref: '#/definitions/Retry'
+      response:
+        $ref: '#/definitions/ExpectedResponse'
     required: [method, path]
 
   Output:
@@ -250,4 +252,8 @@ definitions:
         type: boolean
         default: true
         description: Enable/disable warnings about multiple paths being tested in the same story.
+      invalid-path-detected:
+        type: boolean
+        default: true
+        description: Enable/disable warnings about file paths that do not match paths tested in the story.
     additionalProperties: false
diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml
@@ -455,6 +455,7 @@ paths:
         - $ref: '#/components/parameters/reindex::query.max_docs'
         - $ref: '#/components/parameters/reindex::query.refresh'
         - $ref: '#/components/parameters/reindex::query.requests_per_second'
+        - $ref: '#/components/parameters/reindex::query.require_alias'
         - $ref: '#/components/parameters/reindex::query.scroll'
         - $ref: '#/components/parameters/reindex::query.slices'
         - $ref: '#/components/parameters/reindex::query.timeout'
@@ -1028,8 +1029,6 @@ paths:
         - $ref: '#/components/parameters/search_shards::query.local'
         - $ref: '#/components/parameters/search_shards::query.preference'
         - $ref: '#/components/parameters/search_shards::query.routing'
-      requestBody:
-        $ref: '#/components/requestBodies/search_shards'
       responses:
         '200':
           $ref: '#/components/responses/search_shards@200'
@@ -1047,8 +1046,6 @@ paths:
         - $ref: '#/components/parameters/search_shards::query.local'
         - $ref: '#/components/parameters/search_shards::query.preference'
         - $ref: '#/components/parameters/search_shards::query.routing'
-      requestBody:
-        $ref: '#/components/requestBodies/search_shards'
       responses:
         '200':
           $ref: '#/components/responses/search_shards@200'
@@ -2011,8 +2008,6 @@ paths:
         - $ref: '#/components/parameters/search_shards::query.local'
         - $ref: '#/components/parameters/search_shards::query.preference'
         - $ref: '#/components/parameters/search_shards::query.routing'
-      requestBody:
-        $ref: '#/components/requestBodies/search_shards'
       responses:
         '200':
           $ref: '#/components/responses/search_shards@200'
@@ -2031,8 +2026,6 @@ paths:
         - $ref: '#/components/parameters/search_shards::query.local'
         - $ref: '#/components/parameters/search_shards::query.preference'
         - $ref: '#/components/parameters/search_shards::query.routing'
-      requestBody:
-        $ref: '#/components/requestBodies/search_shards'
       responses:
         '200':
           $ref: '#/components/responses/search_shards@200'
@@ -2538,6 +2531,11 @@ components:
                 type: object
                 additionalProperties:
                   $ref: '../schemas/_common.aggregations.yaml#/components/schemas/AggregationContainer'
+              aggs:
+                description: Defines the aggregations that are run as part of the search request.
+                type: object
+                additionalProperties:
+                  $ref: '../schemas/_common.aggregations.yaml#/components/schemas/AggregationContainer'
               collapse:
                 $ref: '../schemas/_core.search.yaml#/components/schemas/FieldCollapse'
               explain:
@@ -2698,46 +2696,6 @@ components:
                 type: string
             description: The search definition template and its parameters.
       required: true
-    search_shards:
-      content:
-        application/json:
-          schema:
-            type: object
-            properties:
-              query:
-                description: Defines the search query to filter shards. Supports any query that is valid for the `_search` endpoint.
-                type: object
-                additionalProperties:
-                  type: object
-              indices_boost:
-                description: Allows increasing the relevance of specific indexes in the search.
-                type: object
-                additionalProperties:
-                  type: number
-              allow_partial_search_results:
-                description: Indicates whether partial search results are allowed if some shards are unavailable.
-                type: boolean
-              _source:
-                description: Specifies which fields to include in the `_source` response or excludes specific fields.
-                oneOf:
-                  - type: boolean
-                  - type: array
-                    items:
-                      type: string
-              from:
-                description: The starting offset (default 0) for paginated results.
-                type: integer
-              size:
-                description: The maximum number of results to return (default 10).
-                type: integer
-              sort:
-                description: A list of fields and directions for sorting the results.
-                type: array
-                items:
-                  type: object
-                  additionalProperties:
-                    type: string
-            description: Defines the parameters that can be used in the `search_shards` endpoint request. See documentation for supported query syntax.
     termvectors:
       content:
         application/json:
@@ -2916,7 +2874,7 @@ components:
               pit_id:
                 type: string
               _shards:
-                $ref: '../schemas/_core._common.yaml#/components/schemas/ShardStatistics'
+                $ref: '../schemas/_common.yaml#/components/schemas/ShardStatistics'
               creation_time:
                 type: integer
                 format: int64
@@ -2939,7 +2897,7 @@ components:
               pits:
                 type: array
                 items:
-                  $ref: '../schemas/_core._common.yaml#/components/schemas/PitsDetailsDeleteAll'
+                  $ref: '../schemas/_core.pit.yaml#/components/schemas/DeletedPit'
     delete_by_query@200:
       content:
         application/json:
@@ -2964,7 +2922,7 @@ components:
               pits:
                 type: array
                 items:
-                  $ref: '../schemas/_core._common.yaml#/components/schemas/DeletedPit'
+                  $ref: '../schemas/_core.pit.yaml#/components/schemas/DeletedPit'
     delete_script@200:
       content:
         application/json:
@@ -3040,7 +2998,7 @@ components:
               pits:
                 type: array
                 items:
-                  $ref: '../schemas/_core._common.yaml#/components/schemas/PitDetail'
+                  $ref: '../schemas/_core.pit.yaml#/components/schemas/PitDetail'
     get_script@200:
       content:
         application/json:
@@ -3186,8 +3144,7 @@ components:
                   $ref: '../schemas/_core.rank_eval.yaml#/components/schemas/RankEvalMetricDetail'
               failures:
                 type: object
-                additionalProperties:
-                  type: object
+                additionalProperties: true
             required:
               - details
               - failures
@@ -3222,8 +3179,7 @@ components:
             properties:
               template_output:
                 type: object
-                additionalProperties:
-                  type: object
+                additionalProperties: true
             required:
               - template_output
     scripts_painless_execute@200:
@@ -3657,10 +3613,6 @@ components:
     count::query.expand_wildcards:
       in: query
       name: expand_wildcards
-      description: |-
-        Type of index that wildcard patterns can match.
-        If the request can target data streams, this argument determines whether wildcard expressions match hidden data streams.
-        Supports comma-separated values, such as `open,hidden`.
       schema:
         $ref: '../schemas/_common.yaml#/components/schemas/ExpandWildcards'
       style: form
@@ -3948,10 +3900,7 @@ components:
       description: Set to `true` or `false` to return the `_source` field or not, or a list of fields to return.
       style: form
       schema:
-        type: array
-        items:
-          type: string
-        description: Set to `true` or `false` to return the `_source` field or not, or a list of fields to return.
+        $ref: '../schemas/_core.search.yaml#/components/schemas/SourceConfigParam'
       explode: true
     delete_by_query::query._source_excludes:
       name: _source_excludes
@@ -5336,6 +5285,11 @@ components:
         format: float
         default: 0
       style: form
+    reindex::query.require_alias:
+      in: query
+      name: require_alias
+      schema:
+        type: boolean
     reindex::query.scroll:
       in: query
       name: scroll
@@ -6290,10 +6244,7 @@ components:
       description: Set to `true` or `false` to return the `_source` field or not, or a list of fields to return.
       style: form
       schema:
-        type: array
-        items:
-          type: string
-        description: Set to `true` or `false` to return the `_source` field or not, or a list of fields to return.
+        $ref: '../schemas/_core.search.yaml#/components/schemas/SourceConfigParam'
       explode: true
     update_by_query::query._source_excludes:
       name: _source_excludes