Render PPL time column using the correct time zone

changelogs/fragments/9379.yml

@@ -0,0 +1,2 @@
+fix:
+- Make PPL time column respect time zone and date format ([#9379](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9379))

src/plugins/data/common/constants.ts

@@ -111,4 +111,6 @@ export const UI_SETTINGS = {
   SEARCH_QUERY_LANGUAGE_BLOCKLIST: 'search:queryLanguageBlocklist',
   NEW_HOME_PAGE: 'home:useNewHomePage',
   DATA_WITH_LONG_NUMERALS: 'data:withLongNumerals',
+  DATE_FORMAT: 'dateFormat',
+  DATE_FORMAT_TIMEZONE: 'dateFormat:tz',
 } as const;

src/plugins/data/common/data_frames/utils.test.ts

@@ -4,7 +4,17 @@
  */
 
 import datemath from '@opensearch/datemath';
-import { formatTimePickerDate } from '.';
+import {
+  convertResult,
+  DATA_FRAME_TYPES,
+  formatTimePickerDate,
+  IDataFrameErrorResponse,
+  IDataFrameResponse,
+} from '.';
+import moment from 'moment';
+import { ISearchOptions, SearchSourceFields } from '../search';
+import { IIndexPatternFieldList, IndexPattern, IndexPatternField } from '../index_patterns';
+import { OSD_FIELD_TYPES } from '../types';
 
 describe('formatTimePickerDate', () => {
   const mockDateFormat = 'YYYY-MM-DD HH:mm:ss';
@@ -25,3 +35,264 @@ describe('formatTimePickerDate', () => {
     expect(datemath.parse).toHaveBeenCalledWith('now/d', { roundUp: true });
   });
 });
+
+describe('convertResult', () => {
+  const mockDateString = '2025-02-13 00:51:50';
+  const expectedFormattedDate = moment.utc(mockDateString).format('YYYY-MM-DDTHH:mm:ssZ');
+
+  it('should handle empty response', () => {
+    const response: IDataFrameResponse = {
+      took: 0,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [],
+        size: 0,
+        name: 'test-index',
+        values: [],
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    const result = convertResult({ response });
+    expect(result.hits.hits).toEqual([]);
+    expect(result.took).toBe(0);
+  });
+
+  it('should convert simple date fields', () => {
+    const response: IDataFrameResponse = {
+      took: 100,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [
+          { name: 'timestamp', type: 'date', values: [mockDateString] },
+          { name: 'message', type: 'keyword', values: ['test message'] },
+        ],
+        size: 1,
+        name: 'test-index',
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    // Custom date formatter
+    const customFormatter = (dateStr: string, type: OSD_FIELD_TYPES) => {
+      if (type === OSD_FIELD_TYPES.DATE) {
+        return moment.utc(dateStr).format('YYYY-MM-DDTHH:mm:ssZ');
+      }
+    };
+
+    const options: ISearchOptions = {
+      formatter: customFormatter,
+    };
+
+    const result = convertResult({ response, options });
+    expect(result.hits.hits[0]._source.timestamp).toBe(expectedFormattedDate);
+    expect(result.hits.hits[0]._source.message).toBe('test message');
+  });
+
+  it('should handle nested objects with dates', () => {
+    const response: IDataFrameResponse = {
+      took: 100,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [
+          {
+            name: 'metadata',
+            type: 'object',
+            values: [{ created_at: mockDateString, status: 'active' }],
+          },
+        ],
+        size: 1,
+        name: 'test-index',
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    // Create proper IndexPatternField instances
+    const createdAtField = new IndexPatternField(
+      {
+        name: 'metadata.created_at',
+        type: 'date',
+        esTypes: ['date'],
+        searchable: true,
+        aggregatable: true,
+      },
+      'metadata.created_at'
+    );
+
+    const statusField = new IndexPatternField(
+      {
+        name: 'metadata.status',
+        type: 'keyword',
+        esTypes: ['keyword'],
+        searchable: true,
+        aggregatable: true,
+      },
+      'metadata.status'
+    );
+
+    // Create a mock of IIndexPatternFieldList with the required methods
+    const mockFields = [createdAtField, statusField] as IIndexPatternFieldList;
+    // Add required methods
+    mockFields.getAll = jest.fn().mockReturnValue([createdAtField, statusField]);
+    mockFields.getByName = jest.fn((name) =>
+      name === 'metadata.created_at'
+        ? createdAtField
+        : name === 'metadata.status'
+        ? statusField
+        : undefined
+    );
+    mockFields.getByType = jest.fn((type) =>
+      type === 'date' ? [createdAtField] : type === 'keyword' ? [statusField] : []
+    );
+
+    // Mock IndexPattern with fields property using IIndexPatternFieldList
+    const mockIndexPattern: IndexPattern = {
+      fields: mockFields,
+      title: 'test-index',
+      timeFieldName: 'timestamp',
+    };
+
+    // Correctly structured SearchSourceFields
+    const fields: SearchSourceFields = {
+      index: mockIndexPattern,
+    };
+
+    // Custom date formatter
+    const customFormatter = (dateStr: string, type: OSD_FIELD_TYPES) => {
+      if (type === OSD_FIELD_TYPES.DATE) {
+        return moment.utc(dateStr).format('YYYY-MM-DDTHH:mm:ssZ');
+      }
+    };
+
+    const options: ISearchOptions = {
+      formatter: customFormatter,
+    };
+
+    const result = convertResult({ response, fields, options });
+    expect(result.hits.hits[0]._source.metadata.created_at).toBe(expectedFormattedDate);
+    expect(result.hits.hits[0]._source.metadata.status).toBe('active');
+  });
+
+  it('should handle aggregations with date histogram', () => {
+    const response: IDataFrameResponse = {
+      took: 100,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [],
+        size: 0,
+        name: 'test-index',
+        aggs: {
+          timestamp_histogram: [
+            { key: mockDateString, value: 10 },
+            { key: '2025-02-13 01:51:50', value: 20 },
+          ],
+        },
+        meta: {
+          date_histogram: true,
+        },
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    const result = convertResult({ response });
+    expect(result.aggregations?.timestamp_histogram.buckets).toHaveLength(2);
+    expect(result.aggregations?.timestamp_histogram.buckets[0].doc_count).toBe(10);
+  });
+
+  it('should handle error response', () => {
+    const errorResponse: IDataFrameErrorResponse = {
+      type: DATA_FRAME_TYPES.ERROR,
+      took: 0,
+      body: {
+        error: 'Some error message',
+        timed_out: false,
+        took: 0,
+        _shards: {
+          total: 1,
+          successful: 1,
+          skipped: 0,
+          failed: 0,
+        },
+        hits: {
+          total: 0,
+          max_score: 0,
+          hits: [],
+        },
+      },
+    };
+
+    const result = convertResult({ response: errorResponse as IDataFrameResponse });
+    expect(result).toEqual(errorResponse);
+  });
+
+  it('should use default processing when no formatter is provided', () => {
+    const response: IDataFrameResponse = {
+      took: 100,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [{ name: 'timestamp', type: 'date', values: [mockDateString] }],
+        size: 1,
+        name: 'test-index',
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    const result = convertResult({ response });
+    expect(result.hits.hits[0]._source.timestamp).toBe(mockDateString);
+  });
+});

src/plugins/data/common/data_frames/utils.ts

@@ -8,15 +8,16 @@ import datemath from '@opensearch/datemath';
 import {
   DATA_FRAME_TYPES,
   DataFrameAggConfig,
-  IDataFrame,
   IDataFrameWithAggs,
   IDataFrameResponse,
   PartialDataFrame,
   DataFrameQueryConfig,
 } from './types';
 import { IFieldType } from './fields';
 import { IndexPatternFieldMap, IndexPatternSpec } from '../index_patterns';
-import { TimeRange } from '../types';
+import { OSD_FIELD_TYPES, TimeRange } from '../types';
+import { IDataFrame } from './types';
+import { ISearchOptions, SearchSourceFields } from '../search';
 
 /**
  * Converts the data frame response to a search response.
@@ -26,7 +27,15 @@ import { TimeRange } from '../types';
  * @param response - data frame response object
  * @returns converted search response
  */
-export const convertResult = (response: IDataFrameResponse): SearchResponse<any> => {
+export const convertResult = ({
+  response,
+  fields,
+  options,
+}: {
+  response: IDataFrameResponse;
+  fields?: SearchSourceFields;
+  options?: ISearchOptions;
+}): SearchResponse<any> => {
   const body = response.body;
   if (body.hasOwnProperty('error')) {
     return response;
@@ -52,9 +61,49 @@ export const convertResult = (response: IDataFrameResponse): SearchResponse<any>
   if (data && data.fields && data.fields.length > 0) {
     for (let index = 0; index < data.size; index++) {
       const hit: { [key: string]: any } = {};
+
+      const processNestedField = (field: any, value: any, formatter: any): any => {
+        const nestedHit: { [key: string]: any } = value;
+        Object.entries(value).forEach(([nestedField, nestedValue]) => {
+          // Need to get the flattened field name for nested fields ex.products.created_on
+          const flattenedFieldName = `${field.name}.${nestedField}`;
+
+          // Go through search source fields to find the field type of the nested field
+          fields?.index?.fields.forEach((searchSourceField) => {
+            if (
+              searchSourceField.displayName === flattenedFieldName &&
+              searchSourceField.type === 'date'
+            ) {
+              nestedHit[nestedField] = formatter(nestedValue as string, OSD_FIELD_TYPES.DATE);
+            }
+          });
+        });
+        return nestedHit;
+      };
+
+      const processField = (field: any, value: any): any => {
+        if (options && options.formatter) {
+          // Handle date fields
+          if (field.type === 'date') {
+            return options.formatter(value, OSD_FIELD_TYPES.DATE);
+          }
+          // Handle nested objects with potential date fields
+          else if (field.type === 'object') {
+            return processNestedField(field, value, options.formatter);
+          } else {
+            // Default case when the field is either a date type or object type
+            return value;
+          }
+        } else {
+          // Default case when we don't have a formatter
+          return value;
+        }
+      };
+
       data.fields.forEach((field) => {
-        hit[field.name] = field.values[index];
+        hit[field.name] = processField(field, field.values[index]);
       });
+
       hits.push({
         _index: data.name,
         _source: hit,

src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts

@@ -384,6 +384,7 @@ export class IndexPattern implements IIndexPattern {
     return (this.fieldsLoading = status);
   };
 
+  // DQL and Lucene already calling this formatter, we should add ppl formatter in the language config
   /**
    * Provide a field, get its formatter
    * @param field

src/plugins/data/common/search/aggs/buckets/date_histogram.ts

@@ -133,7 +133,7 @@ export const getDateHistogramBucketAgg = ({
             buckets = new TimeBuckets({
               'histogram:maxBars': getConfig(UI_SETTINGS.HISTOGRAM_MAX_BARS),
               'histogram:barTarget': getConfig(UI_SETTINGS.HISTOGRAM_BAR_TARGET),
-              dateFormat: getConfig('dateFormat'),
+              dateFormat: getConfig(UI_SETTINGS.DATE_FORMAT),
               'dateFormat:scaled': getConfig('dateFormat:scaled'),
             });
             updateTimeBuckets(this, calculateBounds, buckets);