Add simple pattern tokenizers

Signed-off-by: Thomas Farr <tsfarr@amazon.com>
opensearch-project · Feb 19, 2025 · 6ab5d18 · 6ab5d18
1 parent 60a9b3b
commit 6ab5d18
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -59,6 +59,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `POST _plugins/_security/api/internalusers/{username}` response `201` ([#810](https://github.com/opensearch-project/opensearch-api-specification/pull/810))
 - Added `POST /_plugins/_ml/_execute/{algorithm_name}` ([#811](https://github.com/opensearch-project/opensearch-api-specification/pull/811))
 - Added search suggester types ([#817](https://github.com/opensearch-project/opensearch-api-specification/pull/817))
+- Added `SimplePatternTokenizer` and `SimplePatternSplitTokenizer` ([#820](https://github.com/opensearch-project/opensearch-api-specification/pull/820))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))

diff --git a/spec/schemas/_common.analysis.yaml b/spec/schemas/_common.analysis.yaml
@@ -1588,6 +1588,8 @@ components:
         - $ref: '#/components/schemas/WhitespaceTokenizer'
         - $ref: '#/components/schemas/KuromojiTokenizer'
         - $ref: '#/components/schemas/PatternTokenizer'
+        - $ref: '#/components/schemas/SimplePatternTokenizer'
+        - $ref: '#/components/schemas/SimplePatternSplitTokenizer'
         - $ref: '#/components/schemas/IcuTokenizer'
         - $ref: '#/components/schemas/SmartcnTokenizer'
     CharGroupTokenizer:
@@ -1831,6 +1833,32 @@ components:
               type: string
           required:
             - type
+    SimplePatternTokenizer:
+      allOf:
+        - $ref: '#/components/schemas/TokenizerBase'
+        - type: object
+          properties:
+            type:
+              type: string
+              enum:
+                - simple_pattern
+              pattern:
+                type: string
+          required:
+            - type
+    SimplePatternSplitTokenizer:
+      allOf:
+        - $ref: '#/components/schemas/TokenizerBase'
+        - type: object
+          properties:
+            type:
+              type: string
+              enum:
+                - simple_pattern_split
+              pattern:
+                type: string
+          required:
+            - type
     SmartcnTokenizer:
       allOf:
         - $ref: '#/components/schemas/TokenizerBase'