Render PPL time column using the correct time zone (opensearch-projec…

…t#9379)

* progress

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

* add timezone and date format

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

* correctly handle time zone

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

* add unit test

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

* Changeset file for PR opensearch-project#9379 created/updated

* progress

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

* use osd field types to check date type

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

* update unit test

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

* make dateFieldsFormatter to formatter so more general

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

* need to consider nested fields have array value

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

* fix cypress

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>

---------

Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>
Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>

changelogs/fragments/9379.yml

@@ -0,0 +1,2 @@
+fix:
+- Make PPL time column respect time zone and date format ([#9379](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9379))

cypress/integration/core_opensearch_dashboards/opensearch_dashboards/apps/query_enhancements/inspect.spec.js

@@ -99,10 +99,17 @@ const inspectTestSuite = () => {
             ) {
               cy.log(`Skipped for ${key}`);
               continue;
+            } else if (config.language === QueryLanguages.PPL.name && key === 'timestamp') {
+              // PPL date field will be formatted
+              docTable
+                .getExpandedDocTableRowFieldValue(key)
+                .should('have.text', 'Dec 31, 2020 @ 16:00:00.000');
+              continue;
+            } else {
+              docTable
+                .getExpandedDocTableRowFieldValue(key)
+                .should('have.text', value === null ? ' - ' : value);
             }
-            docTable
-              .getExpandedDocTableRowFieldValue(key)
-              .should('have.text', value === null ? ' - ' : value);
           }
         });
       });

src/plugins/data/common/constants.ts

@@ -111,4 +111,6 @@ export const UI_SETTINGS = {
   SEARCH_QUERY_LANGUAGE_BLOCKLIST: 'search:queryLanguageBlocklist',
   NEW_HOME_PAGE: 'home:useNewHomePage',
   DATA_WITH_LONG_NUMERALS: 'data:withLongNumerals',
+  DATE_FORMAT: 'dateFormat',
+  DATE_FORMAT_TIMEZONE: 'dateFormat:tz',
 } as const;

src/plugins/data/common/data_frames/utils.test.ts

@@ -4,7 +4,17 @@
  */
 
 import datemath from '@opensearch/datemath';
-import { formatTimePickerDate } from '.';
+import {
+  convertResult,
+  DATA_FRAME_TYPES,
+  formatTimePickerDate,
+  IDataFrameErrorResponse,
+  IDataFrameResponse,
+} from '.';
+import moment from 'moment';
+import { ISearchOptions, SearchSourceFields } from '../search';
+import { IIndexPatternFieldList, IndexPattern, IndexPatternField } from '../index_patterns';
+import { OSD_FIELD_TYPES } from '../types';
 
 describe('formatTimePickerDate', () => {
   const mockDateFormat = 'YYYY-MM-DD HH:mm:ss';
@@ -25,3 +35,264 @@ describe('formatTimePickerDate', () => {
     expect(datemath.parse).toHaveBeenCalledWith('now/d', { roundUp: true });
   });
 });
+
+describe('convertResult', () => {
+  const mockDateString = '2025-02-13 00:51:50';
+  const expectedFormattedDate = moment.utc(mockDateString).format('YYYY-MM-DDTHH:mm:ssZ');
+
+  it('should handle empty response', () => {
+    const response: IDataFrameResponse = {
+      took: 0,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [],
+        size: 0,
+        name: 'test-index',
+        values: [],
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    const result = convertResult({ response });
+    expect(result.hits.hits).toEqual([]);
+    expect(result.took).toBe(0);
+  });
+
+  it('should convert simple date fields', () => {
+    const response: IDataFrameResponse = {
+      took: 100,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [
+          { name: 'timestamp', type: 'date', values: [mockDateString] },
+          { name: 'message', type: 'keyword', values: ['test message'] },
+        ],
+        size: 1,
+        name: 'test-index',
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    // Custom date formatter
+    const customFormatter = (dateStr: string, type: OSD_FIELD_TYPES) => {
+      if (type === OSD_FIELD_TYPES.DATE) {
+        return moment.utc(dateStr).format('YYYY-MM-DDTHH:mm:ssZ');
+      }
+    };
+
+    const options: ISearchOptions = {
+      formatter: customFormatter,
+    };
+
+    const result = convertResult({ response, options });
+    expect(result.hits.hits[0]._source.timestamp).toBe(expectedFormattedDate);
+    expect(result.hits.hits[0]._source.message).toBe('test message');
+  });
+
+  it('should handle nested objects with dates', () => {
+    const response: IDataFrameResponse = {
+      took: 100,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [
+          {
+            name: 'metadata',
+            type: 'object',
+            values: [{ created_at: mockDateString, status: 'active' }],
+          },
+        ],
+        size: 1,
+        name: 'test-index',
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    // Create proper IndexPatternField instances
+    const createdAtField = new IndexPatternField(
+      {
+        name: 'metadata.created_at',
+        type: 'date',
+        esTypes: ['date'],
+        searchable: true,
+        aggregatable: true,
+      },
+      'metadata.created_at'
+    );
+
+    const statusField = new IndexPatternField(
+      {
+        name: 'metadata.status',
+        type: 'keyword',
+        esTypes: ['keyword'],
+        searchable: true,
+        aggregatable: true,
+      },
+      'metadata.status'
+    );
+
+    // Create a mock of IIndexPatternFieldList with the required methods
+    const mockFields = [createdAtField, statusField] as IIndexPatternFieldList;
+    // Add required methods
+    mockFields.getAll = jest.fn().mockReturnValue([createdAtField, statusField]);
+    mockFields.getByName = jest.fn((name) =>
+      name === 'metadata.created_at'
+        ? createdAtField
+        : name === 'metadata.status'
+        ? statusField
+        : undefined
+    );
+    mockFields.getByType = jest.fn((type) =>
+      type === 'date' ? [createdAtField] : type === 'keyword' ? [statusField] : []
+    );
+
+    // Mock IndexPattern with fields property using IIndexPatternFieldList
+    const mockIndexPattern: IndexPattern = {
+      fields: mockFields,
+      title: 'test-index',
+      timeFieldName: 'timestamp',
+    };
+
+    // Correctly structured SearchSourceFields
+    const fields: SearchSourceFields = {
+      index: mockIndexPattern,
+    };
+
+    // Custom date formatter
+    const customFormatter = (dateStr: string, type: OSD_FIELD_TYPES) => {
+      if (type === OSD_FIELD_TYPES.DATE) {
+        return moment.utc(dateStr).format('YYYY-MM-DDTHH:mm:ssZ');
+      }
+    };
+
+    const options: ISearchOptions = {
+      formatter: customFormatter,
+    };
+
+    const result = convertResult({ response, fields, options });
+    expect(result.hits.hits[0]._source.metadata.created_at).toBe(expectedFormattedDate);
+    expect(result.hits.hits[0]._source.metadata.status).toBe('active');
+  });
+
+  it('should handle aggregations with date histogram', () => {
+    const response: IDataFrameResponse = {
+      took: 100,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [],
+        size: 0,
+        name: 'test-index',
+        aggs: {
+          timestamp_histogram: [
+            { key: mockDateString, value: 10 },
+            { key: '2025-02-13 01:51:50', value: 20 },
+          ],
+        },
+        meta: {
+          date_histogram: true,
+        },
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    const result = convertResult({ response });
+    expect(result.aggregations?.timestamp_histogram.buckets).toHaveLength(2);
+    expect(result.aggregations?.timestamp_histogram.buckets[0].doc_count).toBe(10);
+  });
+
+  it('should handle error response', () => {
+    const errorResponse: IDataFrameErrorResponse = {
+      type: DATA_FRAME_TYPES.ERROR,
+      took: 0,
+      body: {
+        error: 'Some error message',
+        timed_out: false,
+        took: 0,
+        _shards: {
+          total: 1,
+          successful: 1,
+          skipped: 0,
+          failed: 0,
+        },
+        hits: {
+          total: 0,
+          max_score: 0,
+          hits: [],
+        },
+      },
+    };
+
+    const result = convertResult({ response: errorResponse as IDataFrameResponse });
+    expect(result).toEqual(errorResponse);
+  });
+
+  it('should use default processing when no formatter is provided', () => {
+    const response: IDataFrameResponse = {
+      took: 100,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+      body: {
+        fields: [{ name: 'timestamp', type: 'date', values: [mockDateString] }],
+        size: 1,
+        name: 'test-index',
+      },
+      type: DATA_FRAME_TYPES.DEFAULT,
+    };
+
+    const result = convertResult({ response });
+    expect(result.hits.hits[0]._source.timestamp).toBe(mockDateString);
+  });
+});