From ff2c85769bdebd8255a7a22329fa3ef13099f3b6 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 4 Sep 2024 16:41:29 -0700 Subject: [PATCH 1/7] Add options; auto-populate sample docs on index selection Signed-off-by: Tyler Ohlsen --- .../workflow_detail/resizable_workspace.tsx | 8 +- .../pages/workflow_detail/workflow_detail.tsx | 3 + .../ingest_inputs/source_data.tsx | 146 +++++++++++++++--- .../input_fields/select_field.tsx | 4 +- .../input_transform_modal.tsx | 6 +- .../output_transform_modal.tsx | 6 +- .../configure_search_request.tsx | 31 ++-- .../workflow_inputs/workflow_inputs.tsx | 6 +- 8 files changed, 151 insertions(+), 59 deletions(-) diff --git a/public/pages/workflow_detail/resizable_workspace.tsx b/public/pages/workflow_detail/resizable_workspace.tsx index 8c089c06..5f68e2f7 100644 --- a/public/pages/workflow_detail/resizable_workspace.tsx +++ b/public/pages/workflow_detail/resizable_workspace.tsx @@ -4,7 +4,6 @@ */ import React, { useRef, useState, useEffect } from 'react'; -import { useSelector } from 'react-redux'; import { Form, Formik } from 'formik'; import * as yup from 'yup'; import { @@ -21,6 +20,7 @@ import { WorkflowConfig, WorkflowFormValues, WorkflowSchema, + customStringify, } from '../../../common'; import { isValidUiWorkflow, @@ -286,11 +286,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { - {JSON.stringify( - reduceToTemplate(props.workflow as Workflow), - undefined, - 2 - )} + {customStringify(reduceToTemplate(props.workflow as Workflow))} diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 5f1cd16b..a0ec10df 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -21,6 +21,7 @@ import { getCore } from '../../services'; import { WorkflowDetailHeader } from './components'; import { AppState, + catIndices, getWorkflow, searchModels, useAppDispatch, @@ -101,9 +102,11 @@ export function WorkflowDetail(props: WorkflowDetailProps) { // On initial load: // - fetch workflow // - fetch available models as their IDs may be used when building flows + // - fetch all indices useEffect(() => { dispatch(getWorkflow({ workflowId, dataSourceId })); dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId })); + dispatch(catIndices({ pattern: '*,-.*', dataSourceId })); }, []); return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) || diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index 941894b6..0d8d64c1 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -4,6 +4,7 @@ */ import React, { useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; import { useFormikContext } from 'formik'; import { EuiSmallButton, @@ -18,19 +19,44 @@ import { EuiSpacer, EuiText, EuiTitle, + EuiFilterGroup, + EuiSmallFilterButton, + EuiSuperSelectOption, + EuiCompressedSuperSelect, } from '@elastic/eui'; import { JsonField } from '../input_fields'; -import { WorkspaceFormValues } from '../../../../../common'; +import { + FETCH_ALL_QUERY, + SearchHit, + WorkspaceFormValues, + customStringify, +} from '../../../../../common'; +import { AppState, searchIndex, useAppDispatch } from '../../../../store'; +import { getDataSourceId } from '../../../../utils'; interface SourceDataProps { setIngestDocs: (docs: string) => void; } +enum SOURCE_OPTIONS { + MANUAL = 'manual', + UPLOAD = 'upload', + EXISTING_INDEX = 'existing_index', +} + /** * Input component for configuring the source data for ingest. */ export function SourceData(props: SourceDataProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); const { values, setFieldValue } = useFormikContext(); + const indices = useSelector((state: AppState) => state.opensearch.indices); + + // selected option state + const [selectedOption, setSelectedOption] = useState( + SOURCE_OPTIONS.MANUAL + ); // edit modal state const [isEditModalOpen, setIsEditModalOpen] = useState(false); @@ -43,8 +69,40 @@ export function SourceData(props: SourceDataProps) { } }; - // Hook to listen when the docs form value changes. - // Try to set the ingestDocs if possible + // selected index state. when an index is selected, update the form value + const [selectedIndex, setSelectedIndex] = useState( + undefined + ); + useEffect(() => { + if (selectedIndex !== undefined) { + dispatch( + searchIndex({ + apiBody: { + index: selectedIndex, + body: FETCH_ALL_QUERY, + searchPipeline: '_none', + }, + dataSourceId, + }) + ) + .unwrap() + .then((resp) => { + const docObjs = resp.hits?.hits + ?.slice(0, 5) + ?.map((hit: SearchHit) => hit?._source); + setFieldValue('ingest.docs', customStringify(docObjs)); + }); + } + }, [selectedIndex]); + + // hook to clear out the selected index when switching options + useEffect(() => { + if (selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX) { + setSelectedIndex(undefined); + } + }, [selectedOption]); + + // hook to listen when the docs form value changes. useEffect(() => { if (values?.ingest?.docs) { props.setIngestDocs(values.ingest.docs); @@ -65,22 +123,74 @@ export function SourceData(props: SourceDataProps) { <> - - Upload a JSON file or enter manually. - {' '} - - { - if (files && files.length > 0) { - fileReader.readAsText(files[0]); + + setSelectedOption(SOURCE_OPTIONS.MANUAL)} + > + Manual + + setSelectedOption(SOURCE_OPTIONS.UPLOAD)} + > + Upload + + - + onClick={() => + setSelectedOption(SOURCE_OPTIONS.EXISTING_INDEX) + } + > + Existing index + + + + {selectedOption === SOURCE_OPTIONS.UPLOAD && ( + <> + { + if (files && files.length > 0) { + fileReader.readAsText(files[0]); + } + }} + display="default" + /> + + + )} + {selectedOption === SOURCE_OPTIONS.EXISTING_INDEX && ( + <> + + ({ + value: option.name, + inputDisplay: {option.name}, + disabled: false, + } as EuiSuperSelectOption) + )} + valueOfSelected={selectedIndex} + onChange={(option) => { + setSelectedIndex(option); + }} + isInvalid={false} + /> + + + Up to 5 sample documents will be automatically populated. + + + + )} { return ( - { setSourceInput( - JSON.stringify( + customStringify( resp.hits.hits.map( (hit: SearchHit) => hit._source - ), - undefined, - 2 + ) ) ); }) diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx index 6e84bb82..393459c1 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx @@ -210,12 +210,10 @@ export function OutputTransformModal(props: OutputTransformModalProps) { .unwrap() .then(async (resp) => { setSourceInput( - JSON.stringify( + customStringify( resp.hits.hits.map( (hit: SearchHit) => hit._source - ), - undefined, - 2 + ) ) ); }) diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx index be239bfc..06772f77 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx @@ -12,20 +12,19 @@ import { EuiFlexGroup, EuiFlexItem, EuiCompressedFormRow, - EuiSuperSelect, + EuiCompressedSuperSelect, EuiSuperSelectOption, EuiText, EuiTitle, EuiSpacer, } from '@elastic/eui'; -import { SearchHit, WorkflowFormValues } from '../../../../../common'; -import { JsonField } from '../input_fields'; import { - AppState, - catIndices, - searchIndex, - useAppDispatch, -} from '../../../../store'; + SearchHit, + WorkflowFormValues, + customStringify, +} from '../../../../../common'; +import { JsonField } from '../input_fields'; +import { AppState, searchIndex, useAppDispatch } from '../../../../store'; import { getDataSourceId } from '../../../../utils/utils'; import { EditQueryModal } from './edit_query_modal'; @@ -74,14 +73,6 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) { } }, [values?.search?.request]); - // Initialization hook to fetch available indices (if applicable) - useEffect(() => { - if (!ingestEnabled) { - // Fetch all indices besides system indices - dispatch(catIndices({ pattern: '*,-.*', dataSourceId })); - } - }, []); - return ( <> {isEditModalOpen && ( @@ -104,7 +95,7 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) { readOnly={true} /> ) : ( - ({ @@ -162,10 +153,8 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) { .unwrap() .then(async (resp) => { props.setQueryResponse( - JSON.stringify( - resp.hits.hits.map((hit: SearchHit) => hit._source), - undefined, - 2 + customStringify( + resp.hits.hits.map((hit: SearchHit) => hit._source) ) ); }) diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx index e392ef41..f3d47fe5 100644 --- a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx @@ -534,10 +534,8 @@ export function WorkflowInputs(props: WorkflowInputsProps) { .unwrap() .then(async (resp) => { props.setQueryResponse( - JSON.stringify( - resp.hits.hits.map((hit: SearchHit) => hit._source), - undefined, - 2 + customStringify( + resp.hits.hits.map((hit: SearchHit) => hit._source) ) ); }) From 417afde7b0fa86efaa2fdced4455dfb6c6bad4f6 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 5 Sep 2024 10:05:38 -0700 Subject: [PATCH 2/7] onboard getmappings api; set default mappings and ml processor configs optionally Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + common/interfaces.ts | 4 +- .../ingest_inputs/ingest_inputs.tsx | 5 +- .../ingest_inputs/source_data.tsx | 85 +++++++++++++++++-- public/route_service.ts | 18 ++++ public/store/reducers/opensearch_reducer.ts | 33 +++++++ server/routes/opensearch_routes_service.ts | 56 ++++++++++++ 7 files changed, 192 insertions(+), 10 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index 107227af..2d0bf50d 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -34,6 +34,7 @@ export const BASE_NODE_API_PATH = '/api/flow_framework'; // OpenSearch node APIs export const BASE_OPENSEARCH_NODE_API_PATH = `${BASE_NODE_API_PATH}/opensearch`; export const CAT_INDICES_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/catIndices`; +export const GET_MAPPINGS_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/mappings`; export const SEARCH_INDEX_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/search`; export const INGEST_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/ingest`; export const BULK_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/bulk`; diff --git a/common/interfaces.ts b/common/interfaces.ts index f0cb927b..2d46da26 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -238,12 +238,12 @@ export type NormalizationProcessor = SearchProcessor & { }; export type IndexConfiguration = { - settings: {}; + settings: { [key: string]: any }; mappings: IndexMappings; }; export type IndexMappings = { - properties: {}; + properties: { [key: string]: any }; }; export type TemplateNode = { diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx index 6b3c7f0d..cc696633 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx @@ -23,7 +23,10 @@ export function IngestInputs(props: IngestInputsProps) { return ( - + diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index 0d8d64c1..f8eeb2f7 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -5,7 +5,7 @@ import React, { useEffect, useState } from 'react'; import { useSelector } from 'react-redux'; -import { useFormikContext } from 'formik'; +import { getIn, useFormikContext } from 'formik'; import { EuiSmallButton, EuiCompressedFilePicker, @@ -27,14 +27,23 @@ import { import { JsonField } from '../input_fields'; import { FETCH_ALL_QUERY, + IndexMappings, + MapEntry, SearchHit, + WorkflowConfig, WorkspaceFormValues, customStringify, } from '../../../../../common'; -import { AppState, searchIndex, useAppDispatch } from '../../../../store'; +import { + AppState, + getMappings, + searchIndex, + useAppDispatch, +} from '../../../../store'; import { getDataSourceId } from '../../../../utils'; interface SourceDataProps { + uiConfig: WorkflowConfig; setIngestDocs: (docs: string) => void; } @@ -69,12 +78,13 @@ export function SourceData(props: SourceDataProps) { } }; - // selected index state. when an index is selected, update the form value + // selected index state. when an index is selected, update several form values const [selectedIndex, setSelectedIndex] = useState( undefined ); useEffect(() => { if (selectedIndex !== undefined) { + // 1. fetch and set sample docs dispatch( searchIndex({ apiBody: { @@ -92,6 +102,63 @@ export function SourceData(props: SourceDataProps) { ?.map((hit: SearchHit) => hit?._source); setFieldValue('ingest.docs', customStringify(docObjs)); }); + + // 2. fetch and set index mappings + dispatch(getMappings({ index: selectedIndex, dataSourceId })) + .unwrap() + .then((resp: IndexMappings) => { + setFieldValue('ingest.index.mappings', customStringify(resp)); + + // 3. try to set default key/values for the ML processor input/output maps, if applicable + const ingestProcessorId = + props.uiConfig.ingest.enrich.processors[0]?.id; + const ingestProcessorInputMapEntry = + (getIn( + values, + `ingest.enrich.${ingestProcessorId}.input_map.0.0`, + undefined + ) as MapEntry) || undefined; + const ingestProcessorOutputMapEntry = + (getIn( + values, + `ingest.enrich.${ingestProcessorId}.output_map.0.0`, + undefined + ) as MapEntry) || undefined; + + if ( + ingestProcessorId !== undefined && + (ingestProcessorInputMapEntry !== undefined || + ingestProcessorOutputMapEntry !== undefined) + ) { + // set/overwrite default text field for the input map. may be empty. + if (ingestProcessorInputMapEntry !== undefined) { + const textFieldFormPath = `ingest.enrich.${ingestProcessorId}.input_map.0.0.value`; + const curTextField = getIn(values, textFieldFormPath) as string; + if (!Object.keys(resp.properties).includes(curTextField)) { + const defaultTextField = + Object.keys(resp.properties).find((fieldName) => { + return resp.properties[fieldName]?.type === 'text'; + }) || ''; + setFieldValue(textFieldFormPath, defaultTextField); + } + } + // set/overwrite default vector field for the output map. may be empty. + if (ingestProcessorOutputMapEntry !== undefined) { + const vectorFieldFormPath = `ingest.enrich.${ingestProcessorId}.output_map.0.0.key`; + const curVectorField = getIn( + values, + vectorFieldFormPath + ) as string; + if (!Object.keys(resp.properties).includes(curVectorField)) { + const defaultVectorField = + Object.keys(resp.properties).find((fieldName) => { + return resp.properties[fieldName]?.type === 'knn_vector'; + }) || ''; + setFieldValue(vectorFieldFormPath, defaultVectorField); + } + } + } + }); } }, [selectedIndex]); @@ -169,6 +236,14 @@ export function SourceData(props: SourceDataProps) { )} {selectedOption === SOURCE_OPTIONS.EXISTING_INDEX && ( <> + + Up to 5 sample documents will be automatically populated. + + + The currently-configured index mappings will be overwritten + to match any selected index. + + @@ -185,10 +260,6 @@ export function SourceData(props: SourceDataProps) { isInvalid={false} /> - - Up to 5 sample documents will be automatically populated. - - )} Promise; + getMappings: ( + index: string, + dataSourceId?: string + ) => Promise; searchIndex: ({ index, body, @@ -270,6 +275,19 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, + getMappings: async (index: string, dataSourceId?: string) => { + try { + const url = dataSourceId + ? `${BASE_NODE_API_PATH}/${dataSourceId}/opensearch/mappings` + : GET_MAPPINGS_NODE_API_PATH; + const response = await core.http.get<{ respString: string }>( + `${url}/${index}` + ); + return response; + } catch (e: any) { + return e as HttpFetchError; + } + }, searchIndex: async ({ index, body, diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts index ddffae86..d40e9396 100644 --- a/public/store/reducers/opensearch_reducer.ts +++ b/public/store/reducers/opensearch_reducer.ts @@ -20,6 +20,7 @@ const initialState = { const OPENSEARCH_PREFIX = 'opensearch'; const CAT_INDICES_ACTION = `${OPENSEARCH_PREFIX}/catIndices`; +const GET_MAPPINGS_ACTION = `${OPENSEARCH_PREFIX}/mappings`; const SEARCH_INDEX_ACTION = `${OPENSEARCH_PREFIX}/search`; const INGEST_ACTION = `${OPENSEARCH_PREFIX}/ingest`; const BULK_ACTION = `${OPENSEARCH_PREFIX}/bulk`; @@ -47,6 +48,26 @@ export const catIndices = createAsyncThunk( } ); +export const getMappings = createAsyncThunk( + GET_MAPPINGS_ACTION, + async ( + { index, dataSourceId }: { index: string; dataSourceId?: string }, + { rejectWithValue } + ) => { + const response: any | HttpFetchError = await getRouteService().getMappings( + index, + dataSourceId + ); + if (response instanceof HttpFetchError) { + return rejectWithValue( + 'Error getting index mappings: ' + response.body.message + ); + } else { + return response; + } + } +); + export const searchIndex = createAsyncThunk( SEARCH_INDEX_ACTION, async ( @@ -169,6 +190,10 @@ const opensearchSlice = createSlice({ state.loading = true; state.errorMessage = ''; }) + .addCase(getMappings.pending, (state, action) => { + state.loading = true; + state.errorMessage = ''; + }) .addCase(searchIndex.pending, (state, action) => { state.loading = true; state.errorMessage = ''; @@ -186,6 +211,10 @@ const opensearchSlice = createSlice({ state.loading = false; state.errorMessage = ''; }) + .addCase(getMappings.fulfilled, (state, action) => { + state.loading = false; + state.errorMessage = ''; + }) .addCase(searchIndex.fulfilled, (state, action) => { state.loading = false; state.errorMessage = ''; @@ -198,6 +227,10 @@ const opensearchSlice = createSlice({ state.errorMessage = action.payload as string; state.loading = false; }) + .addCase(getMappings.rejected, (state, action) => { + state.errorMessage = action.payload as string; + state.loading = false; + }) .addCase(searchIndex.rejected, (state, action) => { state.errorMessage = action.payload as string; state.loading = false; diff --git a/server/routes/opensearch_routes_service.ts b/server/routes/opensearch_routes_service.ts index dfcd61af..4f9f7c04 100644 --- a/server/routes/opensearch_routes_service.ts +++ b/server/routes/opensearch_routes_service.ts @@ -15,8 +15,10 @@ import { BASE_NODE_API_PATH, BULK_NODE_API_PATH, CAT_INDICES_NODE_API_PATH, + GET_MAPPINGS_NODE_API_PATH, INGEST_NODE_API_PATH, Index, + IndexMappings, IngestPipelineConfig, SEARCH_INDEX_NODE_API_PATH, SIMULATE_PIPELINE_NODE_API_PATH, @@ -57,6 +59,29 @@ export function registerOpenSearchRoutes( }, opensearchRoutesService.catIndices ); + router.get( + { + path: `${GET_MAPPINGS_NODE_API_PATH}/{index}`, + validate: { + params: schema.object({ + index: schema.string(), + }), + }, + }, + opensearchRoutesService.getMappings + ); + router.get( + { + path: `${BASE_NODE_API_PATH}/{data_source_id}/opensearch/mappings/{index}`, + validate: { + params: schema.object({ + index: schema.string(), + data_source_id: schema.string(), + }), + }, + }, + opensearchRoutesService.getMappings + ); router.post( { path: `${SEARCH_INDEX_NODE_API_PATH}/{index}`, @@ -252,6 +277,37 @@ export class OpenSearchRoutesService { } }; + getMappings = async ( + context: RequestHandlerContext, + req: OpenSearchDashboardsRequest, + res: OpenSearchDashboardsResponseFactory + ): Promise> => { + const { index } = req.params as { index: string }; + const { data_source_id = '' } = req.params as { data_source_id?: string }; + try { + const callWithRequest = getClientBasedOnDataSource( + context, + this.dataSourceEnabled, + req, + data_source_id, + this.client + ); + + const response = await callWithRequest('indices.getMapping', { + index, + }); + + // Response will be a dict with key being the index name. Attempt to + // pull out the mappings. If any errors found (missing index, etc.), an error + // will be thrown. + const mappings = response[index]?.mappings as IndexMappings; + + return res.ok({ body: mappings }); + } catch (err: any) { + return generateCustomError(res, err); + } + }; + searchIndex = async ( context: RequestHandlerContext, req: OpenSearchDashboardsRequest, From 010641bb9378751d8b611f1e12f16925d984e287 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 5 Sep 2024 11:11:33 -0700 Subject: [PATCH 3/7] clear out some config on manual or uploaded doc input changes Signed-off-by: Tyler Ohlsen --- common/utils.ts | 13 ++- .../ingest_inputs/ingest_inputs.tsx | 4 +- .../ingest_inputs/source_data.tsx | 103 ++++++++++++++---- .../workflow_inputs/workflow_inputs.tsx | 1 + 4 files changed, 95 insertions(+), 26 deletions(-) diff --git a/common/utils.ts b/common/utils.ts index 2992ee71..dc5b780f 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -4,7 +4,7 @@ */ import moment from 'moment'; -import { DATE_FORMAT_PATTERN } from './'; +import { DATE_FORMAT_PATTERN, WORKFLOW_TYPE, Workflow } from './'; import { isEmpty } from 'lodash'; export function toFormattedDate(timestampMillis: number): String { @@ -39,3 +39,14 @@ export function getCharacterLimitedString( export function customStringify(jsonObj: {}): string { return JSON.stringify(jsonObj, undefined, 2); } + +export function isVectorSearchUseCase(workflow: Workflow | undefined): boolean { + return ( + workflow?.ui_metadata?.type !== undefined && + [ + WORKFLOW_TYPE.HYBRID_SEARCH, + WORKFLOW_TYPE.MULTIMODAL_SEARCH, + WORKFLOW_TYPE.SEMANTIC_SEARCH, + ].includes(workflow?.ui_metadata?.type) + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx index cc696633..527def78 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx @@ -8,12 +8,13 @@ import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule } from '@elastic/eui'; import { SourceData } from './source_data'; import { EnrichData } from './enrich_data'; import { IngestData } from './ingest_data'; -import { WorkflowConfig } from '../../../../../common'; +import { Workflow, WorkflowConfig } from '../../../../../common'; interface IngestInputsProps { setIngestDocs: (docs: string) => void; uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; + workflow: Workflow | undefined; } /** @@ -24,6 +25,7 @@ export function IngestInputs(props: IngestInputsProps) { diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index f8eeb2f7..df49a076 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -30,9 +30,11 @@ import { IndexMappings, MapEntry, SearchHit, + Workflow, WorkflowConfig, WorkspaceFormValues, customStringify, + isVectorSearchUseCase, } from '../../../../../common'; import { AppState, @@ -43,6 +45,7 @@ import { import { getDataSourceId } from '../../../../utils'; interface SourceDataProps { + workflow: Workflow | undefined; uiConfig: WorkflowConfig; setIngestDocs: (docs: string) => void; } @@ -78,12 +81,12 @@ export function SourceData(props: SourceDataProps) { } }; - // selected index state. when an index is selected, update several form values + // selected index state. when an index is selected, update several form values (if vector search) const [selectedIndex, setSelectedIndex] = useState( undefined ); useEffect(() => { - if (selectedIndex !== undefined) { + if (selectedIndex !== undefined && isVectorSearchUseCase(props.workflow)) { // 1. fetch and set sample docs dispatch( searchIndex({ @@ -110,29 +113,18 @@ export function SourceData(props: SourceDataProps) { setFieldValue('ingest.index.mappings', customStringify(resp)); // 3. try to set default key/values for the ML processor input/output maps, if applicable - const ingestProcessorId = - props.uiConfig.ingest.enrich.processors[0]?.id; - const ingestProcessorInputMapEntry = - (getIn( - values, - `ingest.enrich.${ingestProcessorId}.input_map.0.0`, - undefined - ) as MapEntry) || undefined; - const ingestProcessorOutputMapEntry = - (getIn( - values, - `ingest.enrich.${ingestProcessorId}.output_map.0.0`, - undefined - ) as MapEntry) || undefined; - + const { + processorId, + inputMapEntry, + outputMapEntry, + } = getProcessorInfo(props.uiConfig, values); if ( - ingestProcessorId !== undefined && - (ingestProcessorInputMapEntry !== undefined || - ingestProcessorOutputMapEntry !== undefined) + processorId !== undefined && + (inputMapEntry !== undefined || outputMapEntry !== undefined) ) { // set/overwrite default text field for the input map. may be empty. - if (ingestProcessorInputMapEntry !== undefined) { - const textFieldFormPath = `ingest.enrich.${ingestProcessorId}.input_map.0.0.value`; + if (inputMapEntry !== undefined) { + const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; const curTextField = getIn(values, textFieldFormPath) as string; if (!Object.keys(resp.properties).includes(curTextField)) { const defaultTextField = @@ -143,8 +135,8 @@ export function SourceData(props: SourceDataProps) { } } // set/overwrite default vector field for the output map. may be empty. - if (ingestProcessorOutputMapEntry !== undefined) { - const vectorFieldFormPath = `ingest.enrich.${ingestProcessorId}.output_map.0.0.key`; + if (outputMapEntry !== undefined) { + const vectorFieldFormPath = `ingest.enrich.${processorId}.output_map.0.0.key`; const curVectorField = getIn( values, vectorFieldFormPath @@ -174,6 +166,38 @@ export function SourceData(props: SourceDataProps) { if (values?.ingest?.docs) { props.setIngestDocs(values.ingest.docs); } + + // try to clear out any default values for the ML ingest processor, if applicable + if ( + isVectorSearchUseCase(props.workflow) && + isEditModalOpen && + selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX + ) { + let sampleDoc = undefined as {} | undefined; + try { + sampleDoc = JSON.parse(values.ingest.docs)[0]; + } catch (error) {} + if (sampleDoc !== undefined) { + const { processorId, inputMapEntry, outputMapEntry } = getProcessorInfo( + props.uiConfig, + values + ); + if ( + processorId !== undefined && + (inputMapEntry !== undefined || outputMapEntry !== undefined) + ) { + // clear any default text field for the input map, if the sample doc + // doesn't contain the currently configured field + if (inputMapEntry !== undefined) { + const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; + const curTextField = getIn(values, textFieldFormPath) as string; + if (!Object.keys(sampleDoc).includes(curTextField)) { + setFieldValue(textFieldFormPath, ''); + } + } + } + } + } }, [values?.ingest?.docs]); return ( @@ -310,3 +334,34 @@ export function SourceData(props: SourceDataProps) { ); } + +// helper fn to parse out some useful info from the ML ingest processor config, if applicable +// takes on the assumption the first processor is an ML inference processor, and should +// only be executed for workflows coming from preset vector search use cases. +function getProcessorInfo( + uiConfig: WorkflowConfig, + values: WorkspaceFormValues +): { + processorId: string | undefined; + inputMapEntry: MapEntry | undefined; + outputMapEntry: MapEntry | undefined; +} { + const ingestProcessorId = uiConfig.ingest.enrich.processors[0]?.id as + | string + | undefined; + return { + processorId: ingestProcessorId, + inputMapEntry: + (getIn( + values, + `ingest.enrich.${ingestProcessorId}.input_map.0.0`, + undefined + ) as MapEntry) || undefined, + outputMapEntry: + (getIn( + values, + `ingest.enrich.${ingestProcessorId}.output_map.0.0`, + undefined + ) as MapEntry) || undefined, + }; +} diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx index f3d47fe5..952ab5a7 100644 --- a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx @@ -725,6 +725,7 @@ export function WorkflowInputs(props: WorkflowInputsProps) { setIngestDocs={props.setIngestDocs} uiConfig={props.uiConfig} setUiConfig={props.setUiConfig} + workflow={props.workflow} /> ) : ( Date: Thu, 5 Sep 2024 11:18:18 -0700 Subject: [PATCH 4/7] change default index name for custom Signed-off-by: Tyler Ohlsen --- public/pages/workflows/new_workflow/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index 8988c739..6eadbc6e 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -82,7 +82,7 @@ function fetchEmptyMetadata(): UIState { name: { id: 'indexName', type: 'string', - value: 'my-new-index', + value: generateId('my_index', 6), }, mappings: { id: 'indexMappings', From d38c4fdc5d998d3b723e427ebef76d6f4ec10315 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 5 Sep 2024 11:23:56 -0700 Subject: [PATCH 5/7] revert overwriting of index mappings Signed-off-by: Tyler Ohlsen --- .../workflow_inputs/ingest_inputs/source_data.tsx | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index df49a076..77187e66 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -106,13 +106,10 @@ export function SourceData(props: SourceDataProps) { setFieldValue('ingest.docs', customStringify(docObjs)); }); - // 2. fetch and set index mappings + // 2. fetch index mappings, and try to set default key/values for the ML processor input/output maps, if applicable dispatch(getMappings({ index: selectedIndex, dataSourceId })) .unwrap() .then((resp: IndexMappings) => { - setFieldValue('ingest.index.mappings', customStringify(resp)); - - // 3. try to set default key/values for the ML processor input/output maps, if applicable const { processorId, inputMapEntry, @@ -263,10 +260,6 @@ export function SourceData(props: SourceDataProps) { Up to 5 sample documents will be automatically populated. - - The currently-configured index mappings will be overwritten - to match any selected index. - Date: Thu, 5 Sep 2024 11:48:58 -0700 Subject: [PATCH 6/7] revert vector field overriding Signed-off-by: Tyler Ohlsen --- .../ingest_inputs/source_data.tsx | 47 ++++--------------- 1 file changed, 8 insertions(+), 39 deletions(-) diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index 77187e66..99f3ed15 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -106,19 +106,15 @@ export function SourceData(props: SourceDataProps) { setFieldValue('ingest.docs', customStringify(docObjs)); }); - // 2. fetch index mappings, and try to set default key/values for the ML processor input/output maps, if applicable + // 2. fetch index mappings, and try to set defaults for the ML processor configs, if applicable dispatch(getMappings({ index: selectedIndex, dataSourceId })) .unwrap() .then((resp: IndexMappings) => { - const { - processorId, - inputMapEntry, - outputMapEntry, - } = getProcessorInfo(props.uiConfig, values); - if ( - processorId !== undefined && - (inputMapEntry !== undefined || outputMapEntry !== undefined) - ) { + const { processorId, inputMapEntry } = getProcessorInfo( + props.uiConfig, + values + ); + if (processorId !== undefined && inputMapEntry !== undefined) { // set/overwrite default text field for the input map. may be empty. if (inputMapEntry !== undefined) { const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; @@ -131,21 +127,6 @@ export function SourceData(props: SourceDataProps) { setFieldValue(textFieldFormPath, defaultTextField); } } - // set/overwrite default vector field for the output map. may be empty. - if (outputMapEntry !== undefined) { - const vectorFieldFormPath = `ingest.enrich.${processorId}.output_map.0.0.key`; - const curVectorField = getIn( - values, - vectorFieldFormPath - ) as string; - if (!Object.keys(resp.properties).includes(curVectorField)) { - const defaultVectorField = - Object.keys(resp.properties).find((fieldName) => { - return resp.properties[fieldName]?.type === 'knn_vector'; - }) || ''; - setFieldValue(vectorFieldFormPath, defaultVectorField); - } - } } }); } @@ -175,16 +156,11 @@ export function SourceData(props: SourceDataProps) { sampleDoc = JSON.parse(values.ingest.docs)[0]; } catch (error) {} if (sampleDoc !== undefined) { - const { processorId, inputMapEntry, outputMapEntry } = getProcessorInfo( + const { processorId, inputMapEntry } = getProcessorInfo( props.uiConfig, values ); - if ( - processorId !== undefined && - (inputMapEntry !== undefined || outputMapEntry !== undefined) - ) { - // clear any default text field for the input map, if the sample doc - // doesn't contain the currently configured field + if (processorId !== undefined && inputMapEntry !== undefined) { if (inputMapEntry !== undefined) { const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; const curTextField = getIn(values, textFieldFormPath) as string; @@ -337,7 +313,6 @@ function getProcessorInfo( ): { processorId: string | undefined; inputMapEntry: MapEntry | undefined; - outputMapEntry: MapEntry | undefined; } { const ingestProcessorId = uiConfig.ingest.enrich.processors[0]?.id as | string @@ -350,11 +325,5 @@ function getProcessorInfo( `ingest.enrich.${ingestProcessorId}.input_map.0.0`, undefined ) as MapEntry) || undefined, - outputMapEntry: - (getIn( - values, - `ingest.enrich.${ingestProcessorId}.output_map.0.0`, - undefined - ) as MapEntry) || undefined, }; } From 7dd41bae96a4767c6b58deb0e6b66105549eb67b Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 5 Sep 2024 11:58:59 -0700 Subject: [PATCH 7/7] Move vector search check Signed-off-by: Tyler Ohlsen --- .../ingest_inputs/source_data.tsx | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index 99f3ed15..d8447804 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -86,7 +86,7 @@ export function SourceData(props: SourceDataProps) { undefined ); useEffect(() => { - if (selectedIndex !== undefined && isVectorSearchUseCase(props.workflow)) { + if (selectedIndex !== undefined) { // 1. fetch and set sample docs dispatch( searchIndex({ @@ -107,28 +107,30 @@ export function SourceData(props: SourceDataProps) { }); // 2. fetch index mappings, and try to set defaults for the ML processor configs, if applicable - dispatch(getMappings({ index: selectedIndex, dataSourceId })) - .unwrap() - .then((resp: IndexMappings) => { - const { processorId, inputMapEntry } = getProcessorInfo( - props.uiConfig, - values - ); - if (processorId !== undefined && inputMapEntry !== undefined) { - // set/overwrite default text field for the input map. may be empty. - if (inputMapEntry !== undefined) { - const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; - const curTextField = getIn(values, textFieldFormPath) as string; - if (!Object.keys(resp.properties).includes(curTextField)) { - const defaultTextField = - Object.keys(resp.properties).find((fieldName) => { - return resp.properties[fieldName]?.type === 'text'; - }) || ''; - setFieldValue(textFieldFormPath, defaultTextField); + if (isVectorSearchUseCase(props.workflow)) { + dispatch(getMappings({ index: selectedIndex, dataSourceId })) + .unwrap() + .then((resp: IndexMappings) => { + const { processorId, inputMapEntry } = getProcessorInfo( + props.uiConfig, + values + ); + if (processorId !== undefined && inputMapEntry !== undefined) { + // set/overwrite default text field for the input map. may be empty. + if (inputMapEntry !== undefined) { + const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; + const curTextField = getIn(values, textFieldFormPath) as string; + if (!Object.keys(resp.properties).includes(curTextField)) { + const defaultTextField = + Object.keys(resp.properties).find((fieldName) => { + return resp.properties[fieldName]?.type === 'text'; + }) || ''; + setFieldValue(textFieldFormPath, defaultTextField); + } } } - } - }); + }); + } } }, [selectedIndex]);