From 58ee10d6d0d3ba7061774a665069d13dddead0ab Mon Sep 17 00:00:00 2001 From: Huy Nguyen <73027756+huyaboo@users.noreply.github.com> Date: Wed, 26 Feb 2025 22:22:20 +0000 Subject: [PATCH 1/2] Refactor changes Signed-off-by: Huy Nguyen <73027756+huyaboo@users.noreply.github.com> --- .../data_importer_app.test.tsx.snap | 407 ++++++++++-------- .../import_type_selector.test.tsx.snap | 179 ++++---- .../components/data_importer_app.test.tsx | 1 + .../public/components/data_importer_app.tsx | 119 ++--- .../public/lib/cat_indices.test.ts | 35 ++ .../data_importer/public/lib/cat_indices.ts | 17 + .../public/lib/import_file.test.ts | 58 ++- .../data_importer/public/lib/import_file.ts | 34 +- .../public/lib/import_text.test.ts | 51 ++- .../data_importer/public/lib/import_text.ts | 32 +- .../data_importer/public/lib/preview.ts | 2 + src/plugins/data_importer/public/types.ts | 4 + .../server/routes/cat_indices.ts | 13 +- .../server/routes/import_file.ts | 33 +- .../server/routes/import_text.ts | 27 +- .../data_importer/server/routes/preview.ts | 5 +- 16 files changed, 634 insertions(+), 383 deletions(-) create mode 100644 src/plugins/data_importer/public/lib/cat_indices.test.ts create mode 100644 src/plugins/data_importer/public/lib/cat_indices.ts diff --git a/src/plugins/data_importer/public/components/__snapshots__/data_importer_app.test.tsx.snap b/src/plugins/data_importer/public/components/__snapshots__/data_importer_app.test.tsx.snap index 12aee687b351..a452893f4fce 100644 --- a/src/plugins/data_importer/public/components/__snapshots__/data_importer_app.test.tsx.snap +++ b/src/plugins/data_importer/public/components/__snapshots__/data_importer_app.test.tsx.snap @@ -10,109 +10,6 @@ exports[`App should render with MDS 1`] = ` useDefaultBehaviors={true} /> - - - - - Data Source Options - - - - -
- <[object Object] - componentConfig={ - Object { - "fullWidth": true, - "notifications": Object { - "toasts": Object { - "add": [MockFunction], - "addDanger": [MockFunction], - "addError": [MockFunction], - "addInfo": [MockFunction], - "addSuccess": [MockFunction], - "addWarning": [MockFunction], - "get$": [MockFunction], - "remove": [MockFunction], - }, - }, - "onSelectedDataSources": [Function], - "savedObjects": Object { - "bulkCreate": [MockFunction], - "bulkGet": [MockFunction], - "bulkUpdate": [MockFunction], - "create": [MockFunction], - "delete": [MockFunction], - "find": [MockFunction], - "get": [MockFunction], - "setCurrentWorkspace": [MockFunction], - "update": [MockFunction], - }, - "selectedOption": undefined, - } - } - componentType="DataSourceSelectable" - dataSourceManagement={ - Object { - "dataSourceSelection": DataSourceSelectionService { - "getSelection$": [Function], - "getSelectionValue": [Function], - "remove": [Function], - "removedComponentIds": Array [], - "selectDataSource": [Function], - "selectedDataSource$": BehaviorSubject { - "_isScalar": false, - "_value": Map {}, - "closed": false, - "hasError": false, - "isStopped": false, - "observers": Array [], - "thrownError": null, - }, - }, - "getDefaultDataSourceId": [Function], - "getDefaultDataSourceId$": [Function], - "registerAuthenticationMethod": [Function], - "ui": Object { - "DataSourceSelector": [Function], - "getDataSourceMenu": [MockFunction] { - "calls": Array [ - Array [], - ], - "results": Array [ - Object { - "type": "return", - "value":
- DataSourceMenu -
, - }, - ], - }, - }, - } - } - /> - -
- - Import - - @@ -122,39 +19,147 @@ exports[`App should render with MDS 1`] = ` >

- - -

- + + + + + Select target data source + + +
+ <[object Object] + componentConfig={ + Object { + "fullWidth": true, + "notifications": Object { + "toasts": Object { + "add": [MockFunction], + "addDanger": [MockFunction], + "addError": [MockFunction], + "addInfo": [MockFunction], + "addSuccess": [MockFunction], + "addWarning": [MockFunction], + "get$": [MockFunction], + "remove": [MockFunction], + }, + }, + "onManageDataSource": [Function], + "onSelectedDataSources": [Function], + "savedObjects": Object { + "bulkCreate": [MockFunction], + "bulkGet": [MockFunction], + "bulkUpdate": [MockFunction], + "create": [MockFunction], + "delete": [MockFunction], + "find": [MockFunction], + "get": [MockFunction], + "setCurrentWorkspace": [MockFunction], + "update": [MockFunction], + }, + } + } + componentType="DataSourceSelectable" + onManageDataSource={[Function]} + /> + +
+ + + + Create/Select Index Name + + + + + + + + + Preview + + + + Import + + + +
+ -

- - - + + + @@ -172,32 +177,6 @@ exports[`App should render without MDS 1`] = ` useDefaultBehaviors={true} /> - - - - - Data Source Options - - - - - - Import - - @@ -207,39 +186,101 @@ exports[`App should render without MDS 1`] = ` >

- - -

- + + + + + + Create/Select Index Name + + + + + + + + + Preview + + + + Import + + + +
+ -

- - - + + + diff --git a/src/plugins/data_importer/public/components/__snapshots__/import_type_selector.test.tsx.snap b/src/plugins/data_importer/public/components/__snapshots__/import_type_selector.test.tsx.snap index 42a06daddffb..3176b5f039b3 100644 --- a/src/plugins/data_importer/public/components/__snapshots__/import_type_selector.test.tsx.snap +++ b/src/plugins/data_importer/public/components/__snapshots__/import_type_selector.test.tsx.snap @@ -13,119 +13,128 @@ exports[`ImportTypeSelector should render 1`] = `
-
-
-
-
- -
-
-
-
-
-
- -
- +
{ const savedObjectsMock = coreMock.createStart().savedObjects; const navigationMock = navigationPluginMock.createStartContract(); const mockConfig: PublicConfigSchema = { + filePreviewDocumentsCount: 10, enabledFileTypes: DEFAULT_SUPPORTED_FILE_TYPES_LIST, maxFileSizeBytes: 104857600, maxTextCount: 10000, diff --git a/src/plugins/data_importer/public/components/data_importer_app.tsx b/src/plugins/data_importer/public/components/data_importer_app.tsx index 2bf9212e197f..1a41443cb25d 100644 --- a/src/plugins/data_importer/public/components/data_importer_app.tsx +++ b/src/plugins/data_importer/public/components/data_importer_app.tsx @@ -46,6 +46,7 @@ import { CSV_FILE_TYPE, CSV_SUPPORTED_DELIMITERS } from '../../common/constants' import { DelimiterSelect } from './delimiter_select'; import { previewFile } from '../lib/preview'; import { PreviewComponent } from './preview_table'; +import { catIndices } from '../lib/cat_indices'; interface DataImporterPluginAppProps { basename: string; @@ -92,31 +93,11 @@ export const DataImporterPluginApp = ({ const [indexOptions, setIndexOptions] = useState>([]); const [createMode, setCreateMode] = useState(false); - useEffect(() => { - const fetchIndices = async () => { - try { - const response = await http.get('/api/data_importer/_cat_indices'); - setIndexOptions(response.indices.map((index: string) => ({ label: index }))); - } catch (error) { - notifications.toasts.addDanger( - i18n.translate('dataImporter.indicesFetchError', { - defaultMessage: `Failed to fetch indices: {error}`, - values: { error }, - }) - ); - } - }; - - fetchIndices(); - }, [http, notifications.toasts]); - const onImportTypeChange = (type: ImportChoices) => { if (type === IMPORT_CHOICE_FILE) { setInputFile(undefined); - setShowDelimiterChoice(true); } else if (type === IMPORT_CHOICE_TEXT) { setText(undefined); - setShowDelimiterChoice(false); } setImportType(type); }; @@ -126,7 +107,6 @@ export const DataImporterPluginApp = ({ setDelimiter(undefined); } setDataType(type); - setShowDelimiterChoice(type === CSV_FILE_TYPE && importType === IMPORT_CHOICE_FILE); }; const onFileInput = (file?: File) => { @@ -168,7 +148,7 @@ export const DataImporterPluginApp = ({ if (importType === IMPORT_CHOICE_FILE) { if (inputFile) { const fileExtension = extname(inputFile.name); - setIsLoadingPreview(true); // Set loading state to true + setIsLoadingPreview(true); try { const response = await previewFile( http, @@ -176,10 +156,11 @@ export const DataImporterPluginApp = ({ createMode, fileExtension, indexName!, + config.filePreviewDocumentsCount, delimiter, dataSourceId ); - setIsLoadingPreview(false); // Set loading state to false + setIsLoadingPreview(false); if (response) { setFilePreviewData(response); notifications.toasts.addSuccess( @@ -195,7 +176,6 @@ export const DataImporterPluginApp = ({ ); } } catch (error) { - // console.error(error, 'error'); setIsLoadingPreview(false); } } @@ -207,25 +187,28 @@ export const DataImporterPluginApp = ({ try { if (importType === IMPORT_CHOICE_FILE) { - response = await importFile( + response = await importFile({ http, - inputFile!, - indexName!, + file: inputFile!, + indexName: indexName!, createMode, // TODO This should be determined from the file type selectable - extname(inputFile!.name), + fileExtension: extname(inputFile!.name), delimiter, - dataSourceId - ); + selectedDataSourceId: dataSourceId, + mapping: filePreviewData.mapping, + }); } else if (importType === IMPORT_CHOICE_TEXT) { - response = await importText( + response = await importText({ http, - inputText!, - dataType!, - indexName!, + text: inputText!, + textFormat: dataType!, + createMode, + indexName: indexName!, delimiter, - dataSourceId - ); + selectedDataSourceId: dataSourceId, + mapping: filePreviewData.mapping, + }); } } catch (error) { notifications.toasts.addDanger( @@ -247,6 +230,21 @@ export const DataImporterPluginApp = ({ }, }) ); + + if (createMode) { + try { + const catIndicesResponse = await catIndices({ http, dataSourceId }); + setIndexOptions(catIndicesResponse.indices.map((index: string) => ({ label: index }))); + setCreateMode(false); + } catch (error) { + notifications.toasts.addDanger( + i18n.translate('dataImporter.indicesFetchError', { + defaultMessage: `Failed to fetch indices: {error}`, + values: { error }, + }) + ); + } + } } else { const errorMessage = response ? `: ${response.message.message}` : ''; notifications.toasts.addDanger( @@ -258,6 +256,24 @@ export const DataImporterPluginApp = ({ } }; + useEffect(() => { + async function fetchIndices() { + try { + const response = await catIndices({ http, dataSourceId }); + setIndexOptions(response.indices.map((index: string) => ({ label: index }))); + } catch (error) { + notifications.toasts.addDanger( + i18n.translate('dataImporter.indicesFetchError', { + defaultMessage: `Failed to fetch indices: {error}`, + values: { error }, + }) + ); + } + } + + fetchIndices(); + }, [http, dataSourceId, notifications.toasts, filePreviewData]); + useEffect(() => { setDisableImport(shouldDisableImportButton()); setShowDelimiterChoice(shouldShowDelimiter()); @@ -295,7 +311,7 @@ export const DataImporterPluginApp = ({ savedObjects: savedObjects.client, notifications, onSelectedDataSources: onDataSourceSelect, - onManageDataSource: () => {}, // Add a proper handler if needed + onManageDataSource: () => {}, }} onManageDataSource={() => {}} /> @@ -319,7 +335,12 @@ export const DataImporterPluginApp = ({ } function shouldShowDelimiter() { - return importType === IMPORT_CHOICE_FILE && dataType === CSV_FILE_TYPE; + return ( + (inputFile && + importType === IMPORT_CHOICE_FILE && + extname(inputFile.name) === `.${CSV_FILE_TYPE}`) || + (importType === IMPORT_CHOICE_TEXT && dataType === CSV_FILE_TYPE) + ); } const loadMoreRows = () => { @@ -378,19 +399,17 @@ export const DataImporterPluginApp = ({ /> + {showDelimiterChoice && ( + + )} {importType === IMPORT_CHOICE_FILE && ( - <> - {showDelimiterChoice && ( - - )} - - + )} {importType === IMPORT_CHOICE_FILE && ( diff --git a/src/plugins/data_importer/public/lib/cat_indices.test.ts b/src/plugins/data_importer/public/lib/cat_indices.test.ts new file mode 100644 index 000000000000..14e3234ae108 --- /dev/null +++ b/src/plugins/data_importer/public/lib/cat_indices.test.ts @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { httpServiceMock } from '../../../../core/public/mocks'; +import { catIndices, CatIndicesProps } from './cat_indices'; + +describe('catIndices()', () => { + const httpMock = httpServiceMock.createStartContract(); + const mockIndexNames = ['foo', 'bar']; + httpMock.get.mockResolvedValue({ + indices: mockIndexNames, + }); + + it.each([ + { + http: httpMock, + dataSourceId: undefined, + }, + { + http: httpMock, + dataSourceId: 'data-source-id', + }, + ])( + 'should call /api/data_importer/_cat_indices with the correct args', + async ({ http, dataSourceId }) => { + const response = await catIndices({ http, dataSourceId }); + expect(response.indices).toMatchObject([...mockIndexNames]); + expect(http.get).toBeCalledWith('/api/data_importer/_cat_indices', { + ...(dataSourceId && { query: { dataSource: dataSourceId } }), + }); + } + ); +}); diff --git a/src/plugins/data_importer/public/lib/cat_indices.ts b/src/plugins/data_importer/public/lib/cat_indices.ts new file mode 100644 index 000000000000..f2dc4849afe7 --- /dev/null +++ b/src/plugins/data_importer/public/lib/cat_indices.ts @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpStart } from 'opensearch-dashboards/public'; +import { CatIndicesResponse } from '../types'; + +export interface CatIndicesProps { + http: HttpStart; + dataSourceId?: string; +} + +export async function catIndices({ http, dataSourceId }: CatIndicesProps) { + const query = dataSourceId ? { dataSource: dataSourceId } : undefined; + return http.get('/api/data_importer/_cat_indices', { query }); +} diff --git a/src/plugins/data_importer/public/lib/import_file.test.ts b/src/plugins/data_importer/public/lib/import_file.test.ts index 2c1677e350a4..3951f1687fb6 100644 --- a/src/plugins/data_importer/public/lib/import_file.test.ts +++ b/src/plugins/data_importer/public/lib/import_file.test.ts @@ -4,7 +4,7 @@ */ import { httpServiceMock } from '../../../../core/public/mocks'; -import { importFile } from './import_file'; +import { importFile, ImportFileProps } from './import_file'; describe('importFile()', () => { const httpMock = httpServiceMock.createStartContract(); @@ -23,56 +23,100 @@ describe('importFile()', () => { httpMock.post.mockClear(); }); - it.each([ + it.each([ { + http: httpMock, file: new File([''], 'test.csv'), indexName: 'foo', delimiter: ';', + createMode: true, + fileExtension: '.csv', + mapping: { foo: 'bar' }, selectedDataSourceId: undefined, }, { + http: httpMock, file: new File([''], 'mds.csv'), indexName: 'bar', delimiter: ',', + createMode: true, + fileExtension: '.csv', + mapping: { foo: 'bar' }, selectedDataSourceId: 'datasource-csv', }, { + http: httpMock, file: new File([''], 'test.ndjson'), indexName: 'bar', - selectedDataSourceId: undefined, + createMode: false, delimiter: undefined, + fileExtension: '.ndjson', + selectedDataSourceId: undefined, }, { + http: httpMock, file: new File([''], 'mds.ndjson'), indexName: 'bar', + createMode: true, + fileExtension: '.ndjson', selectedDataSourceId: 'datasource-ndjson', + mapping: { foo: 'bar' }, delimiter: undefined, }, { + http: httpMock, file: new File([''], 'test.json'), indexName: 'baz', - selectedDataSourceId: undefined, + createMode: false, + fileExtension: '.json', delimiter: undefined, + selectedDataSourceId: undefined, }, { + http: httpMock, file: new File([''], 'mds.json'), indexName: 'qux', - selectedDataSourceId: 'datasource-json', + createMode: false, + fileExtension: '.json', delimiter: undefined, + selectedDataSourceId: 'datasource-json', }, ])( 'should call /api/data_importer/_import_file with the correct args', - async ({ file, indexName, delimiter, selectedDataSourceId }) => { - const response = await importFile(httpMock, file, indexName, delimiter, selectedDataSourceId); + async ({ + http, + file, + indexName, + createMode, + fileExtension, + delimiter, + selectedDataSourceId, + mapping, + }) => { + const response = await importFile({ + http, + file, + createMode, + fileExtension, + indexName, + delimiter, + selectedDataSourceId, + mapping, + }); expect(response.success).toBe(true); expect(response.message.total).toBe(5); const formData = new FormData(); formData.append('file', file); + if (mapping) { + formData.append('mapping', JSON.stringify(mapping)); + } expect(httpMock.post).toBeCalledWith('/api/data_importer/_import_file', { query: { indexName, delimiter, + createMode, + fileExtension, ...(selectedDataSourceId && { dataSource: selectedDataSourceId }), }, headers: { diff --git a/src/plugins/data_importer/public/lib/import_file.ts b/src/plugins/data_importer/public/lib/import_file.ts index 8b59f769e166..aaec6abeab65 100644 --- a/src/plugins/data_importer/public/lib/import_file.ts +++ b/src/plugins/data_importer/public/lib/import_file.ts @@ -6,17 +6,33 @@ import { HttpStart } from '../../../../core/public'; import { ImportResponse } from '../types'; -export async function importFile( - http: HttpStart, - file: File, - indexName: string, - createMode: boolean, - fileExtension: string, - delimiter?: string, - selectedDataSourceId?: string -) { +export interface ImportFileProps { + http: HttpStart; + file: File; + indexName: string; + createMode: boolean; + fileExtension: string; + delimiter?: string; + selectedDataSourceId?: string; + mapping?: Record; +} + +export async function importFile({ + http, + file, + indexName, + createMode, + fileExtension, + delimiter, + selectedDataSourceId, + mapping, +}: ImportFileProps) { const formData = new FormData(); formData.append('file', file); + if (mapping) { + formData.append('mapping', JSON.stringify(mapping)); + } + const query = { indexName, createMode, diff --git a/src/plugins/data_importer/public/lib/import_text.test.ts b/src/plugins/data_importer/public/lib/import_text.test.ts index 17916985905c..2b040b42731b 100644 --- a/src/plugins/data_importer/public/lib/import_text.test.ts +++ b/src/plugins/data_importer/public/lib/import_text.test.ts @@ -4,15 +4,7 @@ */ import { httpServiceMock } from '../../../../core/public/mocks'; -import { importText } from './import_text'; - -interface ImportTextTestCaseFormat { - text: string; - textFormat: string; - indexName: string; - delimiter?: string; - selectedDataSourceId?: string; -} +import { importText, ImportTextProps } from './import_text'; describe('importText()', () => { const httpMock = httpServiceMock.createStartContract(); @@ -31,60 +23,86 @@ describe('importText()', () => { httpMock.post.mockClear(); }); - it.each([ + it.each([ { + http: httpMock, text: `{"foo":"bar"}\n{"baz":"qux"}`, textFormat: 'ndjson', indexName: 'test-ndjson', + createMode: true, + mapping: { foo: 'bar' }, delimiter: undefined, selectedDataSourceId: undefined, }, { + http: httpMock, text: `{"Product ID":7631,"SKU":"HEH-9133","Name":"On Cloud Nine Pillow","Product URL":"https://www.domain.com/product/heh-9133","Price":24.99,"Retail Price":24.99,"Thumbnail URL":"https://www.domain.com/images/heh-9133_600x600.png","Search Keywords":"lorem, ipsum, dolor, ...","Description":"Sociosqu facilisis duis ...","Category":"Home>Home Decor>Pillows|Back In Stock","Category ID":"298|511","Brand":"FabDecor","Child SKU":"","Child Price":"","Color":"White","Color Family":"White","Color Swatches":"","Size":"","Shoe Size":"","Pants Size":"","Occassion":"","Season":"","Badges":"","Rating Avg":4.2,"Rating Count":8,"Inventory Count":21,"Date Created":"2018-03-03 17:41:13"}\n{"Product ID":7615,"SKU":"HEH-2245","Name":"Simply Sweet Blouse","Product URL":"https://www.domain.com/product/heh-2245","Price":42,"Retail Price":59.95,"Thumbnail URL":"https://www.domain.com/images/heh-2245_600x600.png","Search Keywords":"lorem, ipsum, dolor, ...","Description":"Sociosqu facilisis duis ...","Category":"Clothing>Tops>Blouses|Clearance|Tops On Sale","Category ID":"285|512|604","Brand":"Entity Apparel","Child SKU":"HEH-2245-RSWD-SM|HEH-2245-RSWD-MD|HEH-2245-THGR-SM|EH-2245-THGR-MD|HEH-2245-DKCH-SM|HEH-2245-DKCH-MD","Child Price":"42|59.99","Color":"Rosewood|Thyme Green|Dark Charcoal","Color Family":"Red|Green|Grey","Color Swatches":"[{\"color\":\"Rosewood\", \"family\":\"Red\", \"swatch_hex\":\"#65000b\", \"thumbnail\":\"/images/heh-2245-rswd-sm_600x600.png\", \"price\":42}, {\"color\":\"Thyme Green\", \"family\":\"Green\", \"swatch_img\":\"/swatches/thyme_green.png\", \"thumbnail\":\"/images/heh-2245-thgr-sm_600x600.png\", \"price\":59.99}, {\"color\":\"Dark Charcoal\", \"family\":\"Grey\", \"swatch_hex\":\"#36454f\", \"thumbnail\":\"/images/heh-2245-dkch-sm_600x600.png\", \"price\":59.99}]","Size":"Small|Medium","Shoe Size":"","Pants Size":"","Occassion":"","Season":"Summer|Spring","Badges":"Exclusive|Clearance","Rating Avg":4.5,"Rating Count":10,"Inventory Count":8,"Date Created":"2018-03-20 22:24:21"}`, textFormat: 'ndjson', indexName: 'mds-ndjson', delimiter: undefined, selectedDataSourceId: 'datasource-ndjson', + createMode: false, }, { + http: httpMock, text: `field1;field2;field3\n111;Test CSV;"I am awesome"\n852;`, textFormat: 'csv', indexName: 'test-csv', delimiter: ';', selectedDataSourceId: undefined, + createMode: false, }, { + http: httpMock, text: `field1|field2|field3|field4\nMary Jane,26,TX,\nJohn Smith,22,WA,john@example.com`, textFormat: 'csv', indexName: 'mds-csv', delimiter: '|', selectedDataSourceId: 'datasource-csv', + createMode: true, + mapping: { foo: 'bar' }, }, { + http: httpMock, text: `{"name":"John", "age":30, "car":null}`, textFormat: 'json', indexName: 'test-json', delimiter: undefined, selectedDataSourceId: undefined, + createMode: true, + mapping: { foo: 'bar' }, }, { + http: httpMock, text: `{"name":"JohnDoe","age":30,"isStudent":false,"skills":["JavaScript","Python","HTML","CSS"],"address":{"street":"123ElmStreet","city":"Springfield","state":"IL","postalCode":"62704"},"hobbies":[{"name":"Photography","experience":"Intermediate"},{"name":"Hiking","experience":"Beginner"}]}`, textFormat: 'json', indexName: 'mds-json', delimiter: undefined, selectedDataSourceId: 'datasource-json', + createMode: false, }, ])( 'should call /api/data_importer/_import_text with the correct args for a $textFormat file', - async ({ text, textFormat, indexName, delimiter, selectedDataSourceId }) => { - const response = await importText( - httpMock, + async ({ + http, + text, + textFormat, + indexName, + delimiter, + createMode, + selectedDataSourceId, + mapping, + }) => { + const response = await importText({ + http, text, textFormat, indexName, + createMode, delimiter, - selectedDataSourceId - ); + selectedDataSourceId, + mapping, + }); expect(response.success).toBe(true); expect(response.message.total).toBe(5); @@ -92,10 +110,11 @@ describe('importText()', () => { query: { indexName, delimiter, + createMode, fileType: textFormat, ...(selectedDataSourceId && { dataSource: selectedDataSourceId }), }, - body: JSON.stringify({ text }), + body: JSON.stringify({ text, ...(mapping && { mapping: JSON.stringify(mapping) }) }), }); } ); diff --git a/src/plugins/data_importer/public/lib/import_text.ts b/src/plugins/data_importer/public/lib/import_text.ts index 316c8eb39ade..4637e0daf76a 100644 --- a/src/plugins/data_importer/public/lib/import_text.ts +++ b/src/plugins/data_importer/public/lib/import_text.ts @@ -6,23 +6,37 @@ import { HttpStart } from '../../../../core/public'; import { ImportResponse } from '../types'; -export async function importText( - http: HttpStart, - text: string, - textFormat: string, - indexName: string, - delimiter?: string, - selectedDataSourceId?: string -) { +export interface ImportTextProps { + http: HttpStart; + text: string; + textFormat: string; + indexName: string; + createMode: boolean; + delimiter?: string; + selectedDataSourceId?: string; + mapping?: Record; +} + +export async function importText({ + http, + text, + textFormat, + indexName, + createMode, + delimiter, + selectedDataSourceId, + mapping, +}: ImportTextProps) { const query = { indexName, fileType: textFormat, + createMode, ...(selectedDataSourceId && { dataSource: selectedDataSourceId }), delimiter, }; return await http.post('/api/data_importer/_import_text', { - body: JSON.stringify({ text }), + body: JSON.stringify({ text, ...(mapping && { mapping: JSON.stringify(mapping) }) }), query, }); } diff --git a/src/plugins/data_importer/public/lib/preview.ts b/src/plugins/data_importer/public/lib/preview.ts index 8193fe549b9b..d75839a4ca89 100644 --- a/src/plugins/data_importer/public/lib/preview.ts +++ b/src/plugins/data_importer/public/lib/preview.ts @@ -12,6 +12,7 @@ export async function previewFile( createMode: boolean, fileExtension: string, indexName: string, + previewCount: number, delimiter?: string, selectedDataSourceId?: string ) { @@ -21,6 +22,7 @@ export async function previewFile( indexName, fileExtension, createMode, + previewCount, ...(selectedDataSourceId && { dataSource: selectedDataSourceId }), delimiter, }; diff --git a/src/plugins/data_importer/public/types.ts b/src/plugins/data_importer/public/types.ts index bafa7e5f5e36..7c39b162ef2f 100644 --- a/src/plugins/data_importer/public/types.ts +++ b/src/plugins/data_importer/public/types.ts @@ -34,3 +34,7 @@ export interface PreviewResponse { documents: Array>; existingMapping?: Record; } + +export interface CatIndicesResponse { + indices: string[]; +} diff --git a/src/plugins/data_importer/server/routes/cat_indices.ts b/src/plugins/data_importer/server/routes/cat_indices.ts index b1d7768b6789..170d3ef8766e 100644 --- a/src/plugins/data_importer/server/routes/cat_indices.ts +++ b/src/plugins/data_importer/server/routes/cat_indices.ts @@ -6,6 +6,7 @@ import { IRouter } from 'src/core/server'; import { schema, TypeOf } from '@osd/config-schema'; import _ from 'lodash'; +import { CatIndicesIndicesRecord } from '@opensearch-project/opensearch/api/types'; import { configSchema } from '../../config'; import { decideClient } from '../utils/util'; @@ -24,11 +25,7 @@ export function catIndicesRoute( }, }, async (context, request, response) => { - const client = await decideClient( - dataSourceEnabled, - context, - (request.query as { dataSource?: string }).dataSource - ); + const client = await decideClient(dataSourceEnabled, context, request.query.dataSource); if (!!!client) { return response.notFound({ body: 'Data source is not enabled or does not exist', @@ -41,7 +38,11 @@ export function catIndicesRoute( }); return response.ok({ body: { - indices: indices.body.map((index: { index?: string }) => index.index || 'unknown'), + indices: indices.body + .filter((index: CatIndicesIndicesRecord) => { + return !index.index?.startsWith('.'); + }) + .map((index: CatIndicesIndicesRecord) => index.index || 'unknown'), }, }); } catch (e) { diff --git a/src/plugins/data_importer/server/routes/import_file.ts b/src/plugins/data_importer/server/routes/import_file.ts index 1e2597a206ea..a3c36e873ebe 100644 --- a/src/plugins/data_importer/server/routes/import_file.ts +++ b/src/plugins/data_importer/server/routes/import_file.ts @@ -69,23 +69,28 @@ export function importFileRoute( }); } - if (!request.query.createMode) { - try { - const indexExists = await client.indices.exists({ - index: request.query.indexName, - }); + try { + const indexExists = await client.indices.exists({ + index: request.query.indexName, + }); - if (!indexExists.body) { - return response.notFound({ - body: `Index ${request.query.indexName} does not exist`, - }); - } - } catch (e) { - return response.internalError({ - body: `Error checking if index exists: ${e}`, + if (!request.query.createMode && !indexExists.body) { + return response.notFound({ + body: `Index ${request.query.indexName} does not exist`, }); } - } else { + if (request.query.createMode && indexExists.body) { + return response.badRequest({ + body: `Index ${request.query.indexName} already exists`, + }); + } + } catch (e) { + return response.internalError({ + body: `Error checking if index exists: ${e}`, + }); + } + + if (request.query.createMode) { try { const mapping = request.body.mapping ? JSON.parse(request.body.mapping) : {}; await client.indices.create({ diff --git a/src/plugins/data_importer/server/routes/import_text.ts b/src/plugins/data_importer/server/routes/import_text.ts index a5f01c2cbff6..94ab3cccabc0 100644 --- a/src/plugins/data_importer/server/routes/import_text.ts +++ b/src/plugins/data_importer/server/routes/import_text.ts @@ -28,7 +28,8 @@ export function importTextRoute( } }, }), - indexName: schema.string(), + indexName: schema.string({ minLength: 1 }), + createMode: schema.boolean(), delimiter: schema.maybe( schema.string({ validate(value: string) { @@ -42,6 +43,7 @@ export function importTextRoute( }), body: schema.object({ text: schema.string({ minLength: 1, maxLength: config.maxTextCount }), + mapping: schema.maybe(schema.string({ minLength: 1 })), }), }, }, @@ -66,17 +68,38 @@ export function importTextRoute( index: request.query.indexName, }); - if (!indexExists.body) { + if (!request.query.createMode && !indexExists.body) { return response.notFound({ body: `Index ${request.query.indexName} does not exist`, }); } + if (request.query.createMode && indexExists.body) { + return response.badRequest({ + body: `Index ${request.query.indexName} already exists`, + }); + } } catch (e) { return response.internalError({ body: `Error checking if index exists: ${e}`, }); } + if (request.query.createMode) { + try { + const mapping = request.body.mapping ? JSON.parse(request.body.mapping) : {}; + await client.indices.create({ + index: request.query.indexName, + body: { + mappings: mapping, + }, + }); + } catch (e) { + return response.internalError({ + body: `Error creating index: ${e}`, + }); + } + } + let isValid; try { isValid = await parser.validateText(request.body.text, { diff --git a/src/plugins/data_importer/server/routes/preview.ts b/src/plugins/data_importer/server/routes/preview.ts index 11f7be60fc09..7794c4022d86 100644 --- a/src/plugins/data_importer/server/routes/preview.ts +++ b/src/plugins/data_importer/server/routes/preview.ts @@ -43,6 +43,7 @@ export function previewRoute( }, }) ), + previewCount: schema.number({ min: 1, max: config.filePreviewDocumentsCount }), }), body: schema.object({ file: schema.stream(), @@ -88,10 +89,10 @@ export function previewRoute( const file = request.body.file as FileStream; const documents = ( - await parser.parseFile(file, config.filePreviewDocumentsCount, { + await parser.parseFile(file, request.query.previewCount, { delimiter: request.query.delimiter, }) - ).slice(0, config.filePreviewDocumentsCount); + ).slice(0, request.query.previewCount); try { // Ensure OpenSearch can handle the deeply nested objects From 14ea06930adb408a848bb855daf21bee306a9d13 Mon Sep 17 00:00:00 2001 From: "opensearch-changeset-bot[bot]" <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:18:09 +0000 Subject: [PATCH 2/2] Changeset file for PR #9462 created/updated --- changelogs/fragments/9462.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/9462.yml diff --git a/changelogs/fragments/9462.yml b/changelogs/fragments/9462.yml new file mode 100644 index 000000000000..35388d272da3 --- /dev/null +++ b/changelogs/fragments/9462.yml @@ -0,0 +1,2 @@ +fix: +- Remove system indexes from the index name selector ([#9462](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9462)) \ No newline at end of file