diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index dad66e2f1..98098bcf9 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -28,40 +28,42 @@ jobs: admin_password: admin - version: 2.0.0 admin_password: admin - - version: 2.18.0 - - version: 2.18.0 + - version: 2.19.0 + - version: 2.19.0 tests: plugins/index_state_management - - version: 2.18.0 + - version: 2.19.0 tests: plugins/ml - - version: 2.18.0 + - version: 2.19.0 tests: routing - - version: 2.18.0 + - version: 2.19.0 tests: snapshot - - version: 2.18.0 + - version: 2.19.0 tests: remote_store - - version: 2.18.0 + - version: 2.19.0 tests: dangling url: http://localhost:9200 - - version: 2.18.0 + - version: 2.19.0 tests: plugins/replication url: http://localhost:9200 - - version: 2.18.0 + - version: 2.19.0 tests: plugins/streaming - - version: 2.18.0 + - version: 2.19.0 tests: plugins/notifications - - version: 2.18.0 + - version: 2.19.0 tests: plugins/query_insights - - version: 2.18.0 + - version: 2.19.0 tests: plugins/workload-management - - version: 2.18.0 + - version: 2.19.0 tests: plugins/analysis - - version: 2.18.0 + - version: 2.19.0 + tests: plugins/knn + - version: 2.19.0 tests: plugins/security cert: tests/plugins/security/.kirk.pem key: tests/plugins/security/.kirk-key.pem - - version: 2.19.0 + - version: 2.20.0 hub: opensearchstaging - ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' + ref: '@sha256:037a2eb5a4f48b9a9a29950be5e5fae4ebd82d41de1066e00bb8d8b0ce7871b1' - version: 3.0.0 hub: opensearchstaging ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de' diff --git a/CHANGELOG.md b/CHANGELOG.md index e93ea4110..36f27fae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `POST /_plugins/_ml/_execute/{algorithm_name}` ([#811](https://github.com/opensearch-project/opensearch-api-specification/pull/811)) - Added search suggester types ([#817](https://github.com/opensearch-project/opensearch-api-specification/pull/817)) - Added `SimplePatternTokenizer` and `SimplePatternSplitTokenizer` ([#820](https://github.com/opensearch-project/opensearch-api-specification/pull/820)) +- Added `optional_extended_plugins` field to `PluginStats` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) +- Added `id` & `group_by` fields to `TopQuery` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) @@ -112,6 +114,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed GeoDistanceSort schema ([#805](https://github.com/opensearch-project/opensearch-api-specification/pull/805)) - Fixed Aggregation schemas ([#801](https://github.com/opensearch-project/opensearch-api-specification/pull/801)) - Fixed FilterQueryRequestProcessor to use correct query type ([#821](https://github.com/opensearch-project/opensearch-api-specification/pull/821)) +- Fixed `knn.train_model`'s request body `method` field to accept an object rather than a string ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) ### Changed - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683)) @@ -120,6 +123,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Changed naming of `snapshot._common`'s `Status`, `ShardsStats`, `ShardsStatsStage`, `ShardsStatsSummary` and `ShardsStatsSummaryItem` schemas to be prefixed with `Snapshot` ([#730](https://github.com/opensearch-project/opensearch-api-specification/pull/730)) - Changed `ml.get_memory` and `ml.get_message` to split out `get_all` variants ([#796](https://github.com/opensearch-project/opensearch-api-specification/pull/796)) - Changed `ml.get_tools` to have two different operation groups `ml.get_all_tools` and `ml.get_tool` ([#799](https://github.com/opensearch-project/opensearch-api-specification/pull/799)) +- Changed `FlowFrameworkDeleteResponse` to utilize `WriteResponseBase` ([#814](https://github.com/opensearch-project/opensearch-api-specification/pull/814)) ## [0.1.0] - 2024-10-25 diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index b0a47a30a..777e6a2e6 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -444,7 +444,7 @@ components: content: application/json: schema: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkDeleteResponse' + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' flow_framework.get@200: content: application/json: diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index c547a7894..5fc8cf768 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -390,7 +390,7 @@ components: in: query description: The default operator for query string query (AND or OR). schema: - $ref: '../schemas/knn._common.yaml#/components/schemas/DefaultOperator' + $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/Operator' knn.search_models::query.df: name: df in: query @@ -518,7 +518,7 @@ components: in: query description: Search operation type. schema: - $ref: '../schemas/knn._common.yaml#/components/schemas/SearchType' + $ref: '../schemas/_common.yaml#/components/schemas/SearchType' knn.search_models::query.seq_no_primary_term: name: seq_no_primary_term in: query @@ -580,7 +580,7 @@ components: in: query description: Specify suggest mode. schema: - $ref: '../schemas/knn._common.yaml#/components/schemas/SuggestMode' + $ref: '../schemas/_common.yaml#/components/schemas/SuggestMode' knn.search_models::query.suggest_size: name: suggest_size in: query diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index de722eebe..698f02dd5 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1516,6 +1516,11 @@ components: type: ['null', string] opensearch_version: $ref: '#/components/schemas/VersionString' + optional_extended_plugins: + type: array + items: + type: string + x-version-added: '2.19' required: - classname - description diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index d31caa059..b3fc551d2 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -76,30 +76,6 @@ components: version: $ref: '#/components/schemas/version' minProperties: 1 - FlowFrameworkDeleteResponse: - type: object - properties: - _index: - type: string - description: The name of the index where the document was stored. - _id: - type: string - description: The unique identifier of the deleted document. - _version: - type: integer - description: The version of the document after the deletion. - result: - type: string - description: The result of the deletion operation. - enum: [deleted, not_found] - _shards: - $ref: '#/components/schemas/shards' - _seq_no: - type: integer - description: The sequence number assigned to the document after the deletion. - _primary_term: - type: integer - description: The primary term assigned to the document after the deletion. FlowFrameworkGetResponse: type: object properties: @@ -148,14 +124,6 @@ components: skipped: type: integer description: The number of shards where the deletion was skipped. - query: - type: object - description: The search query to match workflows. Use `match_all` to retrieve all workflows, or `match` to search by specific fields like `use_case`. - properties: - match: - type: object - match_all: - type: object total: type: object properties: @@ -224,6 +192,23 @@ components: type: string user: $ref: '#/components/schemas/user' + provision_start_time: + $ref: '_common.yaml#/components/schemas/DateTime' + provision_end_time: + $ref: '_common.yaml#/components/schemas/DateTime' + resources_created: + $ref: '#/components/schemas/ResourcesCreated' + ResourcesCreated: + type: object + properties: + workflow_step_name: + type: string + workflow_step_id: + type: string + resource_type: + type: string + resource_id: + type: string user: type: object properties: @@ -367,7 +352,7 @@ components: type: object properties: query: - $ref: '#/components/schemas/query' + $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' WorkflowSearchResponse: type: object properties: diff --git a/spec/schemas/insights._common.yaml b/spec/schemas/insights._common.yaml index 26907f160..b30ceb977 100644 --- a/spec/schemas/insights._common.yaml +++ b/spec/schemas/insights._common.yaml @@ -19,6 +19,10 @@ components: TopQuery: type: object properties: + id: + type: string + group_by: + $ref: '#/components/schemas/GroupingType' timestamp: type: integer description: The timestamp of the query execution. @@ -56,6 +60,13 @@ components: measurements: type: object $ref: '#/components/schemas/Measurements' + GroupingType: + type: string + enum: + - NONE + - SIMILARITY + - none + - similarity TaskResourceUsages: type: object properties: diff --git a/spec/schemas/knn._common.yaml b/spec/schemas/knn._common.yaml index 3efddc192..d0a6140be 100644 --- a/spec/schemas/knn._common.yaml +++ b/spec/schemas/knn._common.yaml @@ -6,25 +6,6 @@ info: paths: {} components: schemas: - DefaultOperator: - type: string - description: The default operator for query string query (AND or OR). - enum: - - AND - - OR - SearchType: - type: string - description: Search operation type. - enum: - - dfs_query_then_fetch - - query_then_fetch - SuggestMode: - type: string - description: Specify suggest mode. - enum: - - always - - missing - - popular Stats: type: object properties: @@ -143,6 +124,20 @@ components: $ref: '_common.yaml#/components/schemas/ByteCount' current_size_in_bytes: $ref: '_common.yaml#/components/schemas/ByteCount' + KnnMethod: + type: object + properties: + name: + type: string + space_type: + type: string + engine: + type: string + parameters: + type: object + additionalProperties: true + required: + - name TrainedModel: type: object properties: @@ -166,7 +161,7 @@ components: compression_level: type: string method: - type: string + $ref: '#/components/schemas/KnnMethod' spaceType: type: string required: diff --git a/tests/default/flow_framework/workflow/state/search.yaml b/tests/default/flow_framework/workflow/state/search.yaml index 67138eb1e..5f5aa0f76 100644 --- a/tests/default/flow_framework/workflow/state/search.yaml +++ b/tests/default/flow_framework/workflow/state/search.yaml @@ -18,13 +18,15 @@ prologues: output: workflow_id: payload.workflow_id chapters: - - synopsis: Search for a workflow. + - synopsis: Search for a workflow state. path: /_plugins/_flow_framework/workflow/state/_search method: GET request: payload: query: - match_all: {} + term: + workflow_id: + value: ${workflow.workflow_id} response: status: 200 payload: @@ -32,5 +34,4 @@ chapters: hits: - _index: .plugins-flow-framework-state _source: - state: NOT_STARTED - + workflow_id: ${workflow.workflow_id} \ No newline at end of file diff --git a/tests/plugins/knn/docker-compose.yml b/tests/plugins/knn/docker-compose.yml new file mode 100644 index 000000000..04c267ec5 --- /dev/null +++ b/tests/plugins/knn/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3' + +services: + opensearch-cluster: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node diff --git a/tests/default/knn/models.yaml b/tests/plugins/knn/knn/models.yaml similarity index 85% rename from tests/default/knn/models.yaml rename to tests/plugins/knn/knn/models.yaml index 3589fc162..d94fcf797 100644 --- a/tests/default/knn/models.yaml +++ b/tests/plugins/knn/knn/models.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Delete a k-NN model. version: '>= 2.17' @@ -41,6 +41,15 @@ prologues: mode: on_disk compression_level: 32x spaceType: l2 + method: + name: ivf + engine: faiss + parameters: + nlist: 2 + encoder: + name: pq + parameters: + code_size: 1 status: [200] epilogues: - path: /movies diff --git a/tests/default/knn/models/search.yaml b/tests/plugins/knn/knn/models/search.yaml similarity index 78% rename from tests/default/knn/models/search.yaml rename to tests/plugins/knn/knn/models/search.yaml index 52711b0f5..1bc312cb5 100644 --- a/tests/default/knn/models/search.yaml +++ b/tests/plugins/knn/knn/models/search.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../json_schemas/test_story.schema.yaml description: Search for a k-NN model. version: '>= 2.17' @@ -41,6 +41,25 @@ prologues: mode: on_disk compression_level: 32x spaceType: l2 + method: + name: ivf + engine: faiss + parameters: + nlist: 2 + encoder: + name: pq + parameters: + code_size: 1 + status: [200] + - method: GET + path: /_plugins/_knn/models/model-1 + retry: + count: 3 + response: + status: 200 + payload: + model_id: model-1 + state: created status: [200] epilogues: - path: /movies diff --git a/tests/default/knn/models/train.yaml b/tests/plugins/knn/knn/models/train.yaml similarity index 81% rename from tests/default/knn/models/train.yaml rename to tests/plugins/knn/knn/models/train.yaml index fbcfb19d5..a80b442cf 100644 --- a/tests/default/knn/models/train.yaml +++ b/tests/plugins/knn/knn/models/train.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../json_schemas/test_story.schema.yaml description: Train a k-NN model with disk-based parameters. version: '>= 2.17' @@ -70,6 +70,15 @@ chapters: mode: on_disk compression_level: 32x spaceType: l2 + method: + name: ivf + engine: faiss + parameters: + nlist: 9 + encoder: + name: pq + parameters: + code_size: 3 response: status: 200 output: @@ -104,5 +113,28 @@ chapters: mode: on_disk compression_level: 32x spaceType: l2 + method: + name: ivf + engine: faiss + parameters: + nlist: 9 + encoder: + name: pq + parameters: + code_size: 3 response: status: 200 + - synopsis: Wait for the model to get trained (2). + warnings: + multiple-paths-detected: false + method: GET + path: /_plugins/_knn/models/{model_id} + parameters: + model_id: model-1 + retry: + count: 3 + response: + status: 200 + payload: + model_id: model-1 + state: created \ No newline at end of file diff --git a/tests/default/knn/stats.yaml b/tests/plugins/knn/knn/stats.yaml similarity index 92% rename from tests/default/knn/stats.yaml rename to tests/plugins/knn/knn/stats.yaml index f1fb9c5c9..65e0ff6df 100644 --- a/tests/default/knn/stats.yaml +++ b/tests/plugins/knn/knn/stats.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Get k-NN stats. chapters: diff --git a/tests/default/knn/warmup.yaml b/tests/plugins/knn/knn/warmup.yaml similarity index 91% rename from tests/default/knn/warmup.yaml rename to tests/plugins/knn/knn/warmup.yaml index 656e67ee6..937ff2f38 100644 --- a/tests/default/knn/warmup.yaml +++ b/tests/plugins/knn/knn/warmup.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Load all the native library files for all the shards into native memory. prologues: