From 42bc5f8d58ae316e3ada345f8f614cf86b1f6e80 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 3 Sep 2024 11:33:01 -0700 Subject: [PATCH 1/3] Nest processors in accordions Signed-off-by: Tyler Ohlsen --- .../workflow_inputs/processors_list.tsx | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index a69d1a4f..a4c854ba 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -10,10 +10,10 @@ import { EuiContextMenu, EuiFlexGroup, EuiFlexItem, - EuiHorizontalRule, EuiPanel, EuiPopover, - EuiText, + EuiAccordion, + EuiSpacer, } from '@elastic/eui'; import { cloneDeep } from 'lodash'; import { useFormikContext } from 'formik'; @@ -138,36 +138,37 @@ export function ProcessorsList(props: ProcessorsListProps) { {processors.map((processor: IProcessorConfig, processorIndex) => { return ( - - - - {processor.name || 'Processor'} - - - { - deleteProcessor(processor.id); - }} - /> - - - - - + { + deleteProcessor(processor.id); + }} + /> + } + > + + + + + ); })} @@ -178,7 +179,6 @@ export function ProcessorsList(props: ProcessorsListProps) { { setPopover(!isPopoverOpen); }} From 6a4bb7c3ca9862ec4934a0a3e0a444c3b30768a6 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 3 Sep 2024 15:18:16 -0700 Subject: [PATCH 2/3] clean up input/output map defaults; simplify prediction in advanced modals; Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + .../pages/workflow_detail/workflow_detail.tsx | 4 +- .../input_fields/map_field.tsx | 10 +- .../select_with_custom_options.tsx | 19 +-- .../input_transform_modal.tsx | 22 ++- .../output_transform_modal.tsx | 24 +-- .../workflow_inputs/processors_list.tsx | 8 + .../new_workflow/quick_configure_modal.tsx | 154 +++++++++++++++--- 8 files changed, 173 insertions(+), 69 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index 7027e3ce..107227af 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -396,6 +396,7 @@ export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A'; export const EMPTY_FIELD_STRING = '--'; export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception'; export const ERROR_GETTING_WORKFLOW_MSG = 'Failed to retrieve template'; +export const NO_TEMPLATES_FOUND_MSG = 'There are no templates'; export const NO_MODIFICATIONS_FOUND_TEXT = 'Template does not contain any modifications'; export const JSONPATH_ROOT_SELECTOR = '$.'; diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 0008025c..5f1cd16b 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -30,6 +30,7 @@ import { ERROR_GETTING_WORKFLOW_MSG, FETCH_ALL_QUERY, MAX_WORKFLOW_NAME_TO_DISPLAY, + NO_TEMPLATES_FOUND_MSG, getCharacterLimitedString, } from '../../../common'; import { MountPoint } from '../../../../../src/core/public'; @@ -105,7 +106,8 @@ export function WorkflowDetail(props: WorkflowDetailProps) { dispatch(searchModels({ apiBody: FETCH_ALL_QUERY, dataSourceId })); }, []); - return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) ? ( + return errorMessage.includes(ERROR_GETTING_WORKFLOW_MSG) || + errorMessage.includes(NO_TEMPLATES_FOUND_MSG) ? ( ) : ( @@ -137,10 +134,6 @@ export function MapField(props: MapFieldProps) { placeholder={ props.valuePlaceholder || 'Output' } - autofill={ - props.valueOptions?.length === 1 && - idx === 0 - } /> ) : ( ([]); - // update the selected option when the form is updated. if the form is empty, - // default to the top option. by default, this will re-trigger this hook with a populated - // value, to then finally update the displayed option. + // set the visible option when the underlying form is updated. useEffect(() => { - if (props.autofill) { - const formValue = getIn(values, props.fieldPath); - if (!isEmpty(formValue)) { - setSelectedOption([{ label: getIn(values, props.fieldPath) }]); - } else { - if (props.options.length > 0) { - setFieldValue(props.fieldPath, props.options[0].label); - } - } + const formValue = getIn(values, props.fieldPath); + if (!isEmpty(formValue)) { + setSelectedOption([{ label: formValue }]); } }, [getIn(values, props.fieldPath)]); @@ -73,7 +64,7 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) { return ( <> - Expected output for} - options={outputOptions} - value={selectedOutputOption} - onChange={(e) => { - setSelectedOutputOption(Number(e.target.value)); - setTransformedOutput('{}'); - }} - /> + {outputOptions.length === 1 ? ( + Expected output + ) : ( + Expected output for} + options={outputOptions} + value={selectedOutputOption} + onChange={(e) => { + setSelectedOutputOption(Number(e.target.value)); + setTransformedOutput('{}'); + }} + /> + )} ({ value: idx, - text: `Prediction output ${idx + 1}`, + text: `Prediction ${idx + 1}`, })) as EuiSelectOption[]; const [selectedOutputOption, setSelectedOutputOption] = useState< number | undefined @@ -257,15 +257,19 @@ export function OutputTransformModal(props: OutputTransformModalProps) { <> - Expected output for} - options={outputOptions} - value={selectedOutputOption} - onChange={(e) => { - setSelectedOutputOption(Number(e.target.value)); - setTransformedOutput('{}'); - }} - /> + {outputOptions.length === 1 ? ( + Expected output + ) : ( + Expected output for} + options={outputOptions} + value={selectedOutputOption} + onChange={(e) => { + setSelectedOutputOption(Number(e.target.value)); + setTransformedOutput('{}'); + }} + /> + )} (); + // Processor added state. Used to automatically open accordion when a new + // processor is added, assuming users want to immediately configure it. + const [processorAdded, setProcessorAdded] = useState(false); + // Popover state when adding new processors const [isPopoverOpen, setPopover] = useState(false); const closePopover = () => { @@ -75,6 +79,7 @@ export function ProcessorsList(props: ProcessorsListProps) { // (getting any updated/interim values along the way) and add to // the list of processors function addProcessor(processor: IProcessorConfig): void { + setProcessorAdded(true); const existingConfig = cloneDeep(props.uiConfig as WorkflowConfig); let newConfig = formikToUiConfig(values, existingConfig); switch (props.context) { @@ -139,6 +144,9 @@ export function ProcessorsList(props: ProcessorsListProps) { return ( state.ml); + + // model interface state + const [modelInterface, setModelInterface] = useState< + ModelInterface | undefined + >(undefined); // workflow name state const [workflowName, setWorkflowName] = useState( @@ -73,6 +86,14 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) { ); } + // fetching model interface if available. used to prefill some + // of the input/output maps + useEffect(() => { + setModelInterface( + models[quickConfigureFields.embeddingModelId || '']?.interface + ); + }, [models, quickConfigureFields.embeddingModelId]); + return ( props.onClose()} style={{ width: '30vw' }}> @@ -115,7 +136,8 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) { if (!isEmpty(quickConfigureFields)) { workflowToCreate = injectQuickConfigureFields( workflowToCreate, - quickConfigureFields + quickConfigureFields, + modelInterface ); } dispatch( @@ -155,7 +177,8 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) { // helper fn to populate UI config values if there are some quick configure fields available function injectQuickConfigureFields( workflow: Workflow, - quickConfigureFields: QuickConfigureFields + quickConfigureFields: QuickConfigureFields, + modelInterface: ModelInterface | undefined ): Workflow { if (workflow.ui_metadata?.type) { switch (workflow.ui_metadata?.type) { @@ -167,7 +190,8 @@ function injectQuickConfigureFields( if (!isEmpty(quickConfigureFields) && workflow.ui_metadata?.config) { workflow.ui_metadata.config = updateIngestProcessorConfig( workflow.ui_metadata.config, - quickConfigureFields + quickConfigureFields, + modelInterface ); workflow.ui_metadata.config = updateIndexConfig( workflow.ui_metadata.config, @@ -179,7 +203,8 @@ function injectQuickConfigureFields( ); workflow.ui_metadata.config = updateSearchRequestProcessorConfig( workflow.ui_metadata.config, - quickConfigureFields + quickConfigureFields, + modelInterface ); } break; @@ -196,32 +221,56 @@ function injectQuickConfigureFields( // prefill ML ingest processor config, if applicable function updateIngestProcessorConfig( config: WorkflowConfig, - fields: QuickConfigureFields + fields: QuickConfigureFields, + modelInterface: ModelInterface | undefined ): WorkflowConfig { config.ingest.enrich.processors[0].fields.forEach((field) => { if (field.id === 'model' && fields.embeddingModelId) { field.value = { id: fields.embeddingModelId }; } - if (field.id === 'input_map' && (fields.textField || fields.imageField)) { - const inputMap = [] as MapFormValue; + if (field.id === 'input_map') { + const inputMap = generateMapFromModelInputs(modelInterface); if (fields.textField) { - inputMap.push({ - key: '', - value: fields.textField, - }); + if (inputMap.length > 0) { + inputMap[0] = { + ...inputMap[0], + value: fields.textField, + }; + } else { + inputMap.push({ + key: '', + value: fields.textField, + }); + } } if (fields.imageField) { - inputMap.push({ - key: '', - value: fields.imageField, - }); + if (inputMap.length > 1) { + inputMap[1] = { + ...inputMap[1], + value: fields.imageField, + }; + } else { + inputMap.push({ + key: '', + value: fields.imageField, + }); + } } field.value = [inputMap] as MapArrayFormValue; } - if (field.id === 'output_map' && fields.vectorField) { - field.value = [ - [{ key: fields.vectorField, value: '' }], - ] as MapArrayFormValue; + if (field.id === 'output_map') { + const outputMap = generateMapFromModelOutputs(modelInterface); + if (fields.vectorField) { + if (outputMap.length > 0) { + outputMap[0] = { + ...outputMap[0], + key: fields.vectorField, + }; + } else { + outputMap.push({ key: fields.vectorField, value: '' }); + } + } + field.value = [outputMap] as MapArrayFormValue; } }); @@ -232,20 +281,37 @@ function updateIngestProcessorConfig( // including populating placeholders in any pre-configured query_template function updateSearchRequestProcessorConfig( config: WorkflowConfig, - fields: QuickConfigureFields + fields: QuickConfigureFields, + modelInterface: ModelInterface | undefined ): WorkflowConfig { config.search.enrichRequest.processors[0].fields.forEach((field) => { if (field.id === 'model' && fields.embeddingModelId) { field.value = { id: fields.embeddingModelId }; } if (field.id === 'input_map') { + const inputMap = generateMapFromModelInputs(modelInterface); // TODO: pre-populate more if the query becomes standard - field.value = [[EMPTY_MAP_ENTRY]] as MapArrayFormValue; + field.value = + inputMap.length > 0 + ? [inputMap] + : ([[EMPTY_MAP_ENTRY]] as MapArrayFormValue); } if (field.id === 'output_map') { // prepopulate 'vector' constant as the model output transformed field, // so it is consistent and used in the downstream query_template, if configured. - field.value = [[{ key: VECTOR, value: '' }]] as MapArrayFormValue; + const outputMap = generateMapFromModelOutputs(modelInterface); + if (outputMap.length > 0) { + outputMap[0] = { + ...outputMap[0], + key: VECTOR, + }; + } else { + outputMap.push({ + key: VECTOR, + value: '', + }); + } + field.value = [outputMap]; } }); config.search.enrichRequest.processors[0].optionalFields = config.search.enrichRequest.processors[0].optionalFields?.map( @@ -348,3 +414,39 @@ function injectPlaceholderValues( return finalRequestString; } + +// generate a set of mappings s.t. each key is +// a unique model input. +function generateMapFromModelInputs( + modelInterface?: ModelInterface +): MapFormValue { + const inputMap = [] as MapFormValue; + if (modelInterface) { + const modelInputs = parseModelInputs(modelInterface); + modelInputs.forEach((modelInput) => { + inputMap.push({ + key: modelInput.label, + value: '', + }); + }); + } + return inputMap; +} + +// generate a set of mappings s.t. each value is +// a unique model output +function generateMapFromModelOutputs( + modelInterface?: ModelInterface +): MapFormValue { + const outputMap = [] as MapFormValue; + if (modelInterface) { + const modelOutputs = parseModelOutputs(modelInterface); + modelOutputs.forEach((modelOutput) => { + outputMap.push({ + key: '', + value: modelOutput.label, + }); + }); + } + return outputMap; +} From 561e8158819c9a69729179433a9091aabe974d0b Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 3 Sep 2024 16:40:42 -0700 Subject: [PATCH 3/3] Simplify state to just persist interface Signed-off-by: Tyler Ohlsen --- .../input_transform_modal.tsx | 11 +++++-- .../processor_inputs/ml_processor_inputs.tsx | 29 +++++++------------ .../output_transform_modal.tsx | 7 +++-- public/utils/utils.ts | 4 +-- 4 files changed, 24 insertions(+), 27 deletions(-) diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index 7edf58fa..b1dda8a1 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -28,6 +28,7 @@ import { JSONPATH_ROOT_SELECTOR, ML_INFERENCE_DOCS_LINK, MapArrayFormValue, + ModelInterface, PROCESSOR_CONTEXT, SearchHit, SimulateIngestPipelineResponse, @@ -47,7 +48,7 @@ import { useAppDispatch, } from '../../../../store'; import { getCore } from '../../../../services'; -import { getDataSourceId } from '../../../../utils/utils'; +import { getDataSourceId, parseModelInputs } from '../../../../utils/utils'; import { MapArrayField } from '../input_fields'; interface InputTransformModalProps { @@ -56,7 +57,7 @@ interface InputTransformModalProps { context: PROCESSOR_CONTEXT; inputMapField: IConfigField; inputMapFieldPath: string; - inputFields: any[]; + modelInterface: ModelInterface | undefined; onClose: () => void; } @@ -88,6 +89,10 @@ export function InputTransformModal(props: InputTransformModalProps) { number | undefined >((outputOptions[0]?.value as number) ?? undefined); + // TODO: integrated with Ajv to fetch any model interface and perform validation + // on the produced output on-the-fly. For examples, see + // https://www.npmjs.com/package/ajv + return ( @@ -254,7 +259,7 @@ export function InputTransformModal(props: InputTransformModalProps) { ? 'Query field' : 'Document field' } - keyOptions={props.inputFields} + keyOptions={parseModelInputs(props.modelInterface)} // If the map we are adding is the first one, populate the selected option to index 0 onMapAdd={(curArray) => { if (isEmpty(curArray)) { diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx index 9abfa2c2..7baf2aee 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx @@ -22,10 +22,9 @@ import { PROCESSOR_CONTEXT, WorkflowConfig, JSONPATH_ROOT_SELECTOR, - ModelInputFormField, - ModelOutputFormField, ML_INFERENCE_DOCS_LINK, WorkflowFormValues, + ModelInterface, } from '../../../../../common'; import { MapArrayField, ModelField } from '../input_fields'; import { isEmpty } from 'lodash'; @@ -108,9 +107,9 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { >(false); // model interface state - const [hasModelInterface, setHasModelInterface] = useState(true); - const [inputFields, setInputFields] = useState([]); - const [outputFields, setOutputFields] = useState([]); + const [modelInterface, setModelInterface] = useState< + ModelInterface | undefined + >(undefined); // Hook to listen when the selected model has changed. We do a few checks here: // 1: update model interface states @@ -136,15 +135,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { // reusable function to update interface states based on the model ID function updateModelInterfaceStates(modelId: string) { const newSelectedModel = models[modelId]; - if (newSelectedModel?.interface !== undefined) { - setInputFields(parseModelInputs(newSelectedModel.interface)); - setOutputFields(parseModelOutputs(newSelectedModel.interface)); - setHasModelInterface(true); - } else { - setInputFields([]); - setOutputFields([]); - setHasModelInterface(false); - } + setModelInterface(newSelectedModel?.interface); } return ( @@ -156,7 +147,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { context={props.context} inputMapField={inputMapField} inputMapFieldPath={inputMapFieldPath} - inputFields={inputFields} + modelInterface={modelInterface} onClose={() => setIsInputTransformModalOpen(false)} /> )} @@ -167,14 +158,14 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { context={props.context} outputMapField={outputMapField} outputMapFieldPath={outputMapFieldPath} - outputFields={outputFields} + modelInterface={modelInterface} onClose={() => setIsOutputTransformModalOpen(false)} /> )} {!isEmpty(getIn(values, modelFieldPath)?.id) && ( @@ -226,7 +217,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { ? 'Query field' : 'Document field' } - keyOptions={inputFields} + keyOptions={parseModelInputs(modelInterface)} /> @@ -274,7 +265,7 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { : 'Document field' } valuePlaceholder="Model output field" - valueOptions={outputFields} + valueOptions={parseModelOutputs(modelInterface)} /> {inputMapValue.length !== outputMapValue.length && diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx index 44b7face..26dbf6e4 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx @@ -28,6 +28,7 @@ import { JSONPATH_ROOT_SELECTOR, ML_INFERENCE_DOCS_LINK, MapArrayFormValue, + ModelInterface, PROCESSOR_CONTEXT, SearchHit, SearchPipelineConfig, @@ -49,7 +50,7 @@ import { } from '../../../../store'; import { getCore } from '../../../../services'; import { MapArrayField } from '../input_fields'; -import { getDataSourceId } from '../../../../utils/utils'; +import { getDataSourceId, parseModelOutputs } from '../../../../utils/utils'; interface OutputTransformModalProps { uiConfig: WorkflowConfig; @@ -57,7 +58,7 @@ interface OutputTransformModalProps { context: PROCESSOR_CONTEXT; outputMapField: IConfigField; outputMapFieldPath: string; - outputFields: any[]; + modelInterface: ModelInterface | undefined; onClose: () => void; } @@ -237,7 +238,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { helpLink={ML_INFERENCE_DOCS_LINK} keyPlaceholder="Document field" valuePlaceholder="Model output field" - valueOptions={props.outputFields} + valueOptions={parseModelOutputs(props.modelInterface)} // If the map we are adding is the first one, populate the selected option to index 0 onMapAdd={(curArray) => { if (isEmpty(curArray)) { diff --git a/public/utils/utils.ts b/public/utils/utils.ts index 67d9ca88..d573eb9f 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -203,7 +203,7 @@ export function generateTransform(input: {}, map: MapFormValue): {} { // Derive the collection of model inputs from the model interface JSONSchema into a form-ready list export function parseModelInputs( - modelInterface: ModelInterface + modelInterface: ModelInterface | undefined ): ModelInputFormField[] { const modelInputsObj = get( modelInterface, @@ -223,7 +223,7 @@ export function parseModelInputs( // Derive the collection of model outputs from the model interface JSONSchema into a form-ready list export function parseModelOutputs( - modelInterface: ModelInterface + modelInterface: ModelInterface | undefined ): ModelOutputFormField[] { const modelOutputsObj = get(modelInterface, 'output.properties', {}) as { [key: string]: ModelOutput;