diff --git a/changelogs/fragments/9379.yml b/changelogs/fragments/9379.yml new file mode 100644 index 000000000000..2565522db045 --- /dev/null +++ b/changelogs/fragments/9379.yml @@ -0,0 +1,2 @@ +fix: +- Make PPL time column respect time zone and date format ([#9379](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9379)) \ No newline at end of file diff --git a/cypress/integration/core_opensearch_dashboards/opensearch_dashboards/apps/query_enhancements/inspect.spec.js b/cypress/integration/core_opensearch_dashboards/opensearch_dashboards/apps/query_enhancements/inspect.spec.js index f97753998169..c5d946a9d9ca 100644 --- a/cypress/integration/core_opensearch_dashboards/opensearch_dashboards/apps/query_enhancements/inspect.spec.js +++ b/cypress/integration/core_opensearch_dashboards/opensearch_dashboards/apps/query_enhancements/inspect.spec.js @@ -99,10 +99,17 @@ const inspectTestSuite = () => { ) { cy.log(`Skipped for ${key}`); continue; + } else if (config.language === QueryLanguages.PPL.name && key === 'timestamp') { + // PPL date field will be formatted + docTable + .getExpandedDocTableRowFieldValue(key) + .should('have.text', 'Dec 31, 2020 @ 16:00:00.000'); + continue; + } else { + docTable + .getExpandedDocTableRowFieldValue(key) + .should('have.text', value === null ? ' - ' : value); } - docTable - .getExpandedDocTableRowFieldValue(key) - .should('have.text', value === null ? ' - ' : value); } }); }); diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts index c1bc6812129e..947e4016c94d 100644 --- a/src/plugins/data/common/constants.ts +++ b/src/plugins/data/common/constants.ts @@ -111,4 +111,6 @@ export const UI_SETTINGS = { SEARCH_QUERY_LANGUAGE_BLOCKLIST: 'search:queryLanguageBlocklist', NEW_HOME_PAGE: 'home:useNewHomePage', DATA_WITH_LONG_NUMERALS: 'data:withLongNumerals', + DATE_FORMAT: 'dateFormat', + DATE_FORMAT_TIMEZONE: 'dateFormat:tz', } as const; diff --git a/src/plugins/data/common/data_frames/utils.test.ts b/src/plugins/data/common/data_frames/utils.test.ts index 5ba877c963c2..e922accb44e3 100644 --- a/src/plugins/data/common/data_frames/utils.test.ts +++ b/src/plugins/data/common/data_frames/utils.test.ts @@ -4,7 +4,17 @@ */ import datemath from '@opensearch/datemath'; -import { formatTimePickerDate } from '.'; +import { + convertResult, + DATA_FRAME_TYPES, + formatTimePickerDate, + IDataFrameErrorResponse, + IDataFrameResponse, +} from '.'; +import moment from 'moment'; +import { ISearchOptions, SearchSourceFields } from '../search'; +import { IIndexPatternFieldList, IndexPattern, IndexPatternField } from '../index_patterns'; +import { OSD_FIELD_TYPES } from '../types'; describe('formatTimePickerDate', () => { const mockDateFormat = 'YYYY-MM-DD HH:mm:ss'; @@ -25,3 +35,264 @@ describe('formatTimePickerDate', () => { expect(datemath.parse).toHaveBeenCalledWith('now/d', { roundUp: true }); }); }); + +describe('convertResult', () => { + const mockDateString = '2025-02-13 00:51:50'; + const expectedFormattedDate = moment.utc(mockDateString).format('YYYY-MM-DDTHH:mm:ssZ'); + + it('should handle empty response', () => { + const response: IDataFrameResponse = { + took: 0, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: 0, + max_score: 0, + hits: [], + }, + body: { + fields: [], + size: 0, + name: 'test-index', + values: [], + }, + type: DATA_FRAME_TYPES.DEFAULT, + }; + + const result = convertResult({ response }); + expect(result.hits.hits).toEqual([]); + expect(result.took).toBe(0); + }); + + it('should convert simple date fields', () => { + const response: IDataFrameResponse = { + took: 100, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: 0, + max_score: 0, + hits: [], + }, + body: { + fields: [ + { name: 'timestamp', type: 'date', values: [mockDateString] }, + { name: 'message', type: 'keyword', values: ['test message'] }, + ], + size: 1, + name: 'test-index', + }, + type: DATA_FRAME_TYPES.DEFAULT, + }; + + // Custom date formatter + const customFormatter = (dateStr: string, type: OSD_FIELD_TYPES) => { + if (type === OSD_FIELD_TYPES.DATE) { + return moment.utc(dateStr).format('YYYY-MM-DDTHH:mm:ssZ'); + } + }; + + const options: ISearchOptions = { + formatter: customFormatter, + }; + + const result = convertResult({ response, options }); + expect(result.hits.hits[0]._source.timestamp).toBe(expectedFormattedDate); + expect(result.hits.hits[0]._source.message).toBe('test message'); + }); + + it('should handle nested objects with dates', () => { + const response: IDataFrameResponse = { + took: 100, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: 0, + max_score: 0, + hits: [], + }, + body: { + fields: [ + { + name: 'metadata', + type: 'object', + values: [{ created_at: mockDateString, status: 'active' }], + }, + ], + size: 1, + name: 'test-index', + }, + type: DATA_FRAME_TYPES.DEFAULT, + }; + + // Create proper IndexPatternField instances + const createdAtField = new IndexPatternField( + { + name: 'metadata.created_at', + type: 'date', + esTypes: ['date'], + searchable: true, + aggregatable: true, + }, + 'metadata.created_at' + ); + + const statusField = new IndexPatternField( + { + name: 'metadata.status', + type: 'keyword', + esTypes: ['keyword'], + searchable: true, + aggregatable: true, + }, + 'metadata.status' + ); + + // Create a mock of IIndexPatternFieldList with the required methods + const mockFields = [createdAtField, statusField] as IIndexPatternFieldList; + // Add required methods + mockFields.getAll = jest.fn().mockReturnValue([createdAtField, statusField]); + mockFields.getByName = jest.fn((name) => + name === 'metadata.created_at' + ? createdAtField + : name === 'metadata.status' + ? statusField + : undefined + ); + mockFields.getByType = jest.fn((type) => + type === 'date' ? [createdAtField] : type === 'keyword' ? [statusField] : [] + ); + + // Mock IndexPattern with fields property using IIndexPatternFieldList + const mockIndexPattern: IndexPattern = { + fields: mockFields, + title: 'test-index', + timeFieldName: 'timestamp', + }; + + // Correctly structured SearchSourceFields + const fields: SearchSourceFields = { + index: mockIndexPattern, + }; + + // Custom date formatter + const customFormatter = (dateStr: string, type: OSD_FIELD_TYPES) => { + if (type === OSD_FIELD_TYPES.DATE) { + return moment.utc(dateStr).format('YYYY-MM-DDTHH:mm:ssZ'); + } + }; + + const options: ISearchOptions = { + formatter: customFormatter, + }; + + const result = convertResult({ response, fields, options }); + expect(result.hits.hits[0]._source.metadata.created_at).toBe(expectedFormattedDate); + expect(result.hits.hits[0]._source.metadata.status).toBe('active'); + }); + + it('should handle aggregations with date histogram', () => { + const response: IDataFrameResponse = { + took: 100, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: 0, + max_score: 0, + hits: [], + }, + body: { + fields: [], + size: 0, + name: 'test-index', + aggs: { + timestamp_histogram: [ + { key: mockDateString, value: 10 }, + { key: '2025-02-13 01:51:50', value: 20 }, + ], + }, + meta: { + date_histogram: true, + }, + }, + type: DATA_FRAME_TYPES.DEFAULT, + }; + + const result = convertResult({ response }); + expect(result.aggregations?.timestamp_histogram.buckets).toHaveLength(2); + expect(result.aggregations?.timestamp_histogram.buckets[0].doc_count).toBe(10); + }); + + it('should handle error response', () => { + const errorResponse: IDataFrameErrorResponse = { + type: DATA_FRAME_TYPES.ERROR, + took: 0, + body: { + error: 'Some error message', + timed_out: false, + took: 0, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: 0, + max_score: 0, + hits: [], + }, + }, + }; + + const result = convertResult({ response: errorResponse as IDataFrameResponse }); + expect(result).toEqual(errorResponse); + }); + + it('should use default processing when no formatter is provided', () => { + const response: IDataFrameResponse = { + took: 100, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: 0, + max_score: 0, + hits: [], + }, + body: { + fields: [{ name: 'timestamp', type: 'date', values: [mockDateString] }], + size: 1, + name: 'test-index', + }, + type: DATA_FRAME_TYPES.DEFAULT, + }; + + const result = convertResult({ response }); + expect(result.hits.hits[0]._source.timestamp).toBe(mockDateString); + }); +}); diff --git a/src/plugins/data/common/data_frames/utils.ts b/src/plugins/data/common/data_frames/utils.ts index 7e280478630a..ed02e3fdb497 100644 --- a/src/plugins/data/common/data_frames/utils.ts +++ b/src/plugins/data/common/data_frames/utils.ts @@ -8,7 +8,6 @@ import datemath from '@opensearch/datemath'; import { DATA_FRAME_TYPES, DataFrameAggConfig, - IDataFrame, IDataFrameWithAggs, IDataFrameResponse, PartialDataFrame, @@ -16,7 +15,9 @@ import { } from './types'; import { IFieldType } from './fields'; import { IndexPatternFieldMap, IndexPatternSpec } from '../index_patterns'; -import { TimeRange } from '../types'; +import { OSD_FIELD_TYPES, TimeRange } from '../types'; +import { IDataFrame } from './types'; +import { ISearchOptions, SearchSourceFields } from '../search'; /** * Converts the data frame response to a search response. @@ -26,7 +27,15 @@ import { TimeRange } from '../types'; * @param response - data frame response object * @returns converted search response */ -export const convertResult = (response: IDataFrameResponse): SearchResponse => { +export const convertResult = ({ + response, + fields, + options, +}: { + response: IDataFrameResponse; + fields?: SearchSourceFields; + options?: ISearchOptions; +}): SearchResponse => { const body = response.body; if (body.hasOwnProperty('error')) { return response; @@ -52,9 +61,60 @@ export const convertResult = (response: IDataFrameResponse): SearchResponse if (data && data.fields && data.fields.length > 0) { for (let index = 0; index < data.size; index++) { const hit: { [key: string]: any } = {}; + + const processNestedFieldEntry = (field: any, value: any, formatter: any): any => { + Object.entries(value).forEach(([nestedField, nestedValue]) => { + // Need to get the flattened field name for nested fields ex.products.created_on + const flattenedFieldName = `${field.name}.${nestedField}`; + + // Go through search source fields to find the field type of the nested field + fields?.index?.fields.forEach((searchSourceField) => { + if ( + searchSourceField.displayName === flattenedFieldName && + searchSourceField.type === 'date' + ) { + value[nestedField] = formatter(nestedValue as string, OSD_FIELD_TYPES.DATE); + } + }); + }); + return value; + }; + + const processNestedField = (field: any, value: any, formatter: any): any => { + const nestedHit: { [key: string]: any } = value; + // if nestedHit is an array, we need to process each element + if (Array.isArray(nestedHit)) { + return nestedHit.map((nestedValue) => { + return processNestedFieldEntry(field, nestedValue, formatter); + }); + } else { + return processNestedFieldEntry(field, nestedHit, formatter); + } + }; + + const processField = (field: any, value: any): any => { + if (options && options.formatter) { + // Handle date fields + if (field.type === 'date') { + return options.formatter(value, OSD_FIELD_TYPES.DATE); + } + // Handle nested objects with potential date fields + else if (field.type === 'object') { + return processNestedField(field, value, options.formatter); + } else { + // Default case when the field is either a date type or object type + return value; + } + } else { + // Default case when we don't have a formatter + return value; + } + }; + data.fields.forEach((field) => { - hit[field.name] = field.values[index]; + hit[field.name] = processField(field, field.values[index]); }); + hits.push({ _index: data.name, _source: hit, diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts b/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts index 773badbde732..89c11582fc0f 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts @@ -384,6 +384,7 @@ export class IndexPattern implements IIndexPattern { return (this.fieldsLoading = status); }; + // DQL and Lucene already calling this formatter, we should add ppl formatter in the language config /** * Provide a field, get its formatter * @param field diff --git a/src/plugins/data/common/search/aggs/buckets/date_histogram.ts b/src/plugins/data/common/search/aggs/buckets/date_histogram.ts index 765aad2d3fd9..b28df914cd9b 100644 --- a/src/plugins/data/common/search/aggs/buckets/date_histogram.ts +++ b/src/plugins/data/common/search/aggs/buckets/date_histogram.ts @@ -133,7 +133,7 @@ export const getDateHistogramBucketAgg = ({ buckets = new TimeBuckets({ 'histogram:maxBars': getConfig(UI_SETTINGS.HISTOGRAM_MAX_BARS), 'histogram:barTarget': getConfig(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: getConfig('dateFormat'), + dateFormat: getConfig(UI_SETTINGS.DATE_FORMAT), 'dateFormat:scaled': getConfig('dateFormat:scaled'), }); updateTimeBuckets(this, calculateBounds, buckets); diff --git a/src/plugins/data/common/search/aggs/utils/calculate_auto_time_expression.ts b/src/plugins/data/common/search/aggs/utils/calculate_auto_time_expression.ts index 05abf7235194..57a0a4436c17 100644 --- a/src/plugins/data/common/search/aggs/utils/calculate_auto_time_expression.ts +++ b/src/plugins/data/common/search/aggs/utils/calculate_auto_time_expression.ts @@ -45,7 +45,7 @@ export function getCalculateAutoTimeExpression(getConfig: (key: string) => any) const buckets = new TimeBuckets({ 'histogram:maxBars': getConfig(UI_SETTINGS.HISTOGRAM_MAX_BARS), 'histogram:barTarget': getConfig(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: getConfig('dateFormat'), + dateFormat: getConfig(UI_SETTINGS.DATE_FORMAT), 'dateFormat:scaled': getConfig('dateFormat:scaled'), }); diff --git a/src/plugins/data/common/search/opensearch_search/types.ts b/src/plugins/data/common/search/opensearch_search/types.ts index f90a3f1de245..b9cbb71a80d2 100644 --- a/src/plugins/data/common/search/opensearch_search/types.ts +++ b/src/plugins/data/common/search/opensearch_search/types.ts @@ -31,6 +31,7 @@ import { SearchResponse } from 'elasticsearch'; import { Search } from '@opensearch-project/opensearch/api/requestParams'; import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types'; +import { OSD_FIELD_TYPES } from '../../types'; export const OPENSEARCH_SEARCH_STRATEGY = 'opensearch'; export const OPENSEARCH_SEARCH_WITH_LONG_NUMERALS_STRATEGY = 'opensearch-with-long-numerals'; @@ -48,6 +49,10 @@ export interface ISearchOptions { * Use this option to enable support for long numerals. */ withLongNumeralsSupport?: boolean; + /** + * Use this option to format the fields in the search response. + */ + formatter?: (value: any, type: OSD_FIELD_TYPES) => any; } export type ISearchRequestParams> = { diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index df57a8800ed1..329294febbc3 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -124,7 +124,7 @@ import { handleQueryResults } from '../../utils/helpers'; /** @internal */ export const searchSourceRequiredUiSettings = [ - 'dateFormat:tz', + UI_SETTINGS.DATE_FORMAT_TIMEZONE, UI_SETTINGS.COURIER_BATCH_SEARCHES, UI_SETTINGS.COURIER_CUSTOM_REQUEST_PREFERENCE, UI_SETTINGS.COURIER_IGNORE_FILTER_IF_FIELD_NOT_IN_INDEX, @@ -444,7 +444,14 @@ export class SearchSource { if ((response as IDataFrameResponse).type === DATA_FRAME_TYPES.DEFAULT) { const dataFrameResponse = response as IDataFrameDefaultResponse; await this.setDataFrame(dataFrameResponse.body as IDataFrame); - return onResponse(searchRequest, convertResult(response as IDataFrameResponse)); + return onResponse( + searchRequest, + convertResult({ + response: response as IDataFrameResponse, + fields: this.getFields(), + options, + }) + ); } if ((response as IDataFrameResponse).type === DATA_FRAME_TYPES.POLLING) { const startTime = Date.now(); @@ -477,7 +484,14 @@ export class SearchSource { (results as any).took = elapsedMs; await this.setDataFrame((results as QuerySuccessStatusResponse).body as IDataFrame); - return onResponse(searchRequest, convertResult(results as IDataFrameResponse)); + return onResponse( + searchRequest, + convertResult({ + response: results as IDataFrameResponse, + fields: this.getFields(), + options, + }) + ); } if ((response as IDataFrameResponse).type === DATA_FRAME_TYPES.ERROR) { const dataFrameError = response as IDataFrameError; diff --git a/src/plugins/data/public/query/query_string/language_service/types.ts b/src/plugins/data/public/query/query_string/language_service/types.ts index 5bcc33b3edaa..b69201c35a63 100644 --- a/src/plugins/data/public/query/query_string/language_service/types.ts +++ b/src/plugins/data/public/query/query_string/language_service/types.ts @@ -5,6 +5,7 @@ import { ISearchInterceptor } from '../../../search'; import { + OSD_FIELD_TYPES, Query, QueryEditorExtensionConfig, QueryStringContract, @@ -54,6 +55,7 @@ export interface LanguageConfig { sortable?: boolean; filterable?: boolean; visualizable?: boolean; + formatter?: (value: any, type: OSD_FIELD_TYPES) => any; }; showDocLinks?: boolean; docLink?: { diff --git a/src/plugins/discover/public/application/view_components/utils/use_search.ts b/src/plugins/discover/public/application/view_components/utils/use_search.ts index 9acb3e0e9d5f..3e7690358485 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_search.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_search.ts @@ -243,10 +243,18 @@ export const useSearch = (services: DiscoverViewServices) => { trackQueryMetric(query); } + const languageConfig = data.query.queryString + .getLanguageService() + .getLanguage(query!.language); + // Execute the search const fetchResp = await searchSource.fetch({ abortSignal: fetchStateRef.current.abortController.signal, withLongNumeralsSupport: await services.uiSettings.get(UI_SETTINGS.DATA_WITH_LONG_NUMERALS), + ...(languageConfig && + languageConfig.fields?.formatter && { + formatter: languageConfig.fields.formatter, + }), }); inspectorRequest diff --git a/src/plugins/query_enhancements/public/plugin.tsx b/src/plugins/query_enhancements/public/plugin.tsx index 71ad2302b99e..119acaec2dba 100644 --- a/src/plugins/query_enhancements/public/plugin.tsx +++ b/src/plugins/query_enhancements/public/plugin.tsx @@ -4,8 +4,9 @@ */ import { i18n } from '@osd/i18n'; import { BehaviorSubject } from 'rxjs'; +import moment from 'moment'; import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '../../../core/public'; -import { DataStorage } from '../../data/common'; +import { DataStorage, OSD_FIELD_TYPES } from '../../data/common'; import { createEditor, DefaultInput, @@ -66,7 +67,20 @@ export class QueryEnhancementsPlugin usageCollector: data.search.usageCollector, }), getQueryString: (currentQuery: Query) => `source = ${currentQuery.dataset?.title}`, - fields: { sortable: false, filterable: false, visualizable: false }, + fields: { + sortable: false, + filterable: false, + visualizable: false, + formatter: (value: string, type: OSD_FIELD_TYPES) => { + switch (type) { + case OSD_FIELD_TYPES.DATE: + return moment.utc(value).format('YYYY-MM-DDTHH:mm:ssZ'); // PPL date fields need special formatting in order for discover table formatter to render in the correct time zone + + default: + return value; + } + }, + }, docLink: { title: i18n.translate('queryEnhancements.pplLanguage.docLink', { defaultMessage: 'PPL documentation',