Update cypress data to have random id, missing value fields and uniqu…

…e field

* Random id generation in TestDataGenerator
* Add a unique field to test modifyColumnsOnSwitch
* Update mapping and include a never present field to test hide
missing values

Signed-off-by: Anan <ananzh@amazon.com>

cypress/fixtures/query_enhancements/data_logs_1/data_logs_large_time_1.mapping.json

@@ -22,6 +22,12 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
+      "unique_category": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_1/data_logs_large_time_2.mapping.json

@@ -22,6 +22,9 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_1/data_logs_small_no_time_1.mapping.json

@@ -22,6 +22,12 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
+      "unique_category": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_1/data_logs_small_no_time_2.mapping.json

@@ -22,6 +22,9 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_1/data_logs_small_time_1.mapping.json

@@ -22,6 +22,12 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
+      "unique_category": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_1/data_logs_small_time_2.mapping.json

@@ -22,6 +22,9 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_2/data_logs_large_time_1.mapping.json

@@ -22,6 +22,12 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
+      "unique_category": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_2/data_logs_large_time_2.mapping.json

@@ -22,6 +22,9 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_2/data_logs_small_no_time_1.mapping.json

@@ -22,6 +22,12 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
+      "unique_category": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_2/data_logs_small_no_time_2.mapping.json

@@ -22,6 +22,9 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_2/data_logs_small_time_1.mapping.json

@@ -22,6 +22,12 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
+      "unique_category": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/fixtures/query_enhancements/data_logs_2/data_logs_small_time_2.mapping.json

@@ -22,6 +22,9 @@
       "service_endpoint": {
         "type": "keyword"
       },
+      "never_present_field": {
+        "type": "keyword"
+      },
       "personal": {
         "type": "nested",
         "properties": {

cypress/integration/core_opensearch_dashboards/opensearch_dashboards/apps/query_enhancements/a_check.spec.js

@@ -3,46 +3,42 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { DATASOURCE_NAME } from '../../../../../utils/apps/constants';
-import { SECONDARY_ENGINE } from '../../../../../utils/constants';
-import { getRandomizedWorkspaceName } from '../../../../../utils/apps/query_enhancements/shared';
+import { SECONDARY_ENGINE, INDEX_WITH_TIME_1 } from '../../../../../utils/constants';
+import {
+  getRandomizedWorkspaceName,
+  getRandomizedDatasourceName,
+} from '../../../../../utils/apps/query_enhancements/shared';
 
-const workspace = getRandomizedWorkspaceName();
+const workspaceName = getRandomizedWorkspaceName();
+const dataSourceName = getRandomizedDatasourceName();
 
 describe('No Index Pattern Check Test', () => {
   before(() => {
     // Load test data
     cy.setupTestData(
       SECONDARY_ENGINE.url,
-      [
-        'cypress/fixtures/query_enhancements/data_logs_1/data_logs_small_time_1.mapping.json',
-        'cypress/fixtures/query_enhancements/data_logs_2/data_logs_small_time_2.mapping.json',
-      ],
-      [
-        'cypress/fixtures/query_enhancements/data_logs_1/data_logs_small_time_1.data.ndjson',
-        'cypress/fixtures/query_enhancements/data_logs_2/data_logs_small_time_2.data.ndjson',
-      ]
+      ['cypress/fixtures/query_enhancements/data_logs_1/data_logs_small_time_1.mapping.json'],
+      ['cypress/fixtures/query_enhancements/data_logs_1/data_logs_small_time_1.data.ndjson']
     );
 
     // Add data source
     cy.addDataSource({
-      name: `${DATASOURCE_NAME}`,
-      url: `${SECONDARY_ENGINE.url}`,
+      name: dataSourceName,
+      url: SECONDARY_ENGINE.url,
       authType: 'no_auth',
     });
     // Create workspace
-    cy.deleteWorkspaceByName(workspace);
+    cy.deleteWorkspaceByName(workspaceName);
     cy.visit('/app/home');
-    cy.osd.createInitialWorkspaceWithDataSource(DATASOURCE_NAME, workspace);
+    cy.osd.createInitialWorkspaceWithDataSource(dataSourceName, workspaceName);
     cy.wait(2000);
   });
 
   after(() => {
-    cy.deleteWorkspaceByName(`${workspace}`);
-    cy.deleteDataSourceByName(`${DATASOURCE_NAME}`);
+    cy.deleteWorkspaceByName(workspaceName);
     // TODO: Modify deleteIndex to handle an array of index and remove hard code
-    cy.deleteIndex('data_logs_small_time_1');
-    cy.deleteIndex('data_logs_small_time_2');
+    cy.deleteDataSourceByName(dataSourceName);
+    cy.deleteIndex(INDEX_WITH_TIME_1);
   });
 
   describe('empty state', () => {

cypress/integration/core_opensearch_dashboards/opensearch_dashboards/apps/query_enhancements/queries.spec.js

@@ -59,7 +59,7 @@ describe('query enhancement queries', { scrollBehavior: false }, () => {
       cy.setQueryLanguage('DQL');
       cy.setTopNavDate(START_TIME, END_TIME);
 
-      const query = `_id:1`;
+      const query = `_id:N9srQ8opwBxGdIoQU3TW`;
       cy.setQueryEditor(query);
       cy.waitForLoader(true);
       cy.waitForSearch();
@@ -74,7 +74,7 @@ describe('query enhancement queries', { scrollBehavior: false }, () => {
       cy.setQueryLanguage('Lucene');
       cy.setTopNavDate(START_TIME, END_TIME);
 
-      const query = `_id:1`;
+      const query = `_id:N9srQ8opwBxGdIoQU3TW`;
       cy.setQueryEditor(query);
       cy.waitForLoader(true);
       cy.waitForSearch();

cypress/integration/core_opensearch_dashboards/opensearch_dashboards/apps/query_enhancements/saved_search_in_dashboards.spec.js

@@ -114,7 +114,7 @@ export const runSavedSearchTests = () => {
         START_TIME,
         'Oct 1, 2022 @ 00:00:00.000'
       );
-      cy.getElementByTestId('osdDocTablePagination').contains(/of 15/);
+      cy.getElementByTestId('osdDocTablePagination').contains(/of 13/);
     });
 
     it('Show valid saved searches', () => {

cypress/integration/core_opensearch_dashboards/opensearch_dashboards/apps/query_enhancements/sidebar.spec.js

@@ -84,7 +84,7 @@ const addSidebarFieldsAndCheckDocTableColumns = (
     cy.wait('@query').then(() => {
       checkTableHeaders(testFields);
       if (isIndexPattern) {
-        cy.getElementByTestId('discoverQueryHits').should('have.text', '1,152');
+        cy.getElementByTestId('discoverQueryHits').should('have.text', '1,125');
       }
       checkDocTableColumn(expectedValues, 2);
     });
@@ -133,11 +133,11 @@ export const runSideBarTests = () => {
       sqlQuery: (dataset) => `SELECT * FROM ${dataset} WHERE status_code = 200`,
       simpleFields: {
         fields: ['service_endpoint', 'response_time', 'bytes_transferred', 'request_url'],
-        expectedValues: ['3.32', '2.8', '3.35', '1.68', '4.98'],
+        expectedValues: ['3.91', '4.82', '1.72', '4.08', '3.97'],
       },
       nestedFields: {
         fields: ['personal.name', 'personal.age', 'personal.birthdate', 'personal.address.country'],
-        expectedValues: ['28', '55', '76', '56', '36'],
+        expectedValues: ['28', '52', '65', '21', '79'],
       },
     };
 

cypress/scripts/generate_data/generate_all_data.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Array of configurations
+declare -a configs=(
+    "data_logs_small_time_2 --doc-count 10000"
+    "data_logs_small_no_time_2 --doc-count 10000 --no-timestamp"
+    "data_logs_large_time_2 --doc-count 100000"
+)
+# Log file
+LOG_FILE="data_generation.log"
+echo "Starting data generation at $(date)" > $LOG_FILE
+# Loop through configurations and generate data
+for config in "${configs[@]}"
+do
+    echo "Generating data for: $config"
+    echo "Starting $config at $(date)" >> $LOG_FILE
+
+    if node scripts/generate_data/index.js $config >> $LOG_FILE 2>&1; then
+        echo "Successfully generated data for $config" | tee -a $LOG_FILE
+    else
+        echo "Failed to generate data for $config" | tee -a $LOG_FILE
+        exit 1
+    fi
+done
+echo "All data generation completed successfully!"
+echo "Completed all data generation at $(date)" >> $LOG_FILE

cypress/scripts/generate_data/index.js

@@ -9,8 +9,14 @@ const { join } = require('path');
 const { TestDataGenerator } = require('./test_data_generator');
 
 // Need to update different path for multiple clusters
-const DEFAULT_PATH =
-  './cypress/fixtures/dashboard/opensearch_dashboards/query_enhancement/data_logs_1';
+const DEFAULT_PATH = './fixtures/query_enhancements/data_logs_2';
+const ID_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+function generateRandomId(length = 20) {
+  return Array.from({ length }, () => ID_CHARS[Math.floor(Math.random() * ID_CHARS.length)]).join(
+    ''
+  );
+}
 
 // Helper function to parse integer
 function parseInteger(value) {
@@ -63,7 +69,7 @@ program
 
       // Create documents
       for (let i = 0; i < docCount; i++) {
-        const indexLine = JSON.stringify({ index: { _index: indexName, _id: i } });
+        const indexLine = JSON.stringify({ index: { _index: indexName, _id: generateRandomId() } });
         const docLine = JSON.stringify(generator.createDoc(i));
 
         dataWriter.write(indexLine + '\n');

cypress/scripts/generate_data/test_data_generator.js

@@ -15,6 +15,30 @@ class TestDataGenerator {
     this.categories = ['Network', 'Database', 'Security', 'Application'];
     this.statusCodes = [200, 201, 400, 401, 403, 404, 500, 502, 503];
 
+    // Unique categories for some indices
+    this.uniqueCategories = [
+      'Analytics',
+      'Monitoring',
+      'Deployment',
+      'Testing',
+      'Development',
+      'Production',
+      'Staging',
+      'LoadBalancing',
+      'Caching',
+      'Logging',
+      'Authentication',
+      'Authorization',
+      'Encryption',
+      'Backup',
+      'Recovery',
+      'Configuration',
+      'Integration',
+      'Migration',
+      'Optimization',
+      'Maintenance',
+    ];
+
     // Country-city mapping for controlled location data
     this.locationMap = {
       USA: ['New York', 'Los Angeles', 'Chicago', 'Seattle'],
@@ -31,6 +55,7 @@ class TestDataGenerator {
 
     this.countries = Object.keys(this.locationMap);
     this.categoryCount = this.categories.length;
+    this.uniqueCategoriesCount = this.uniqueCategories.length;
 
     // Only set time-related properties if includeTimestamp is true
     if (includeTimestamp) {
@@ -109,6 +134,10 @@ class TestDataGenerator {
     const location = this.getLocationWithCoordinates();
     const doc = {
       category: this.categories[Math.floor((index * this.categoryCount) / this.docCount)],
+      // Remove for some indices to create a unique field. Test for modifyColumnsOnSwitch.
+      unique_category: this.uniqueCategories[
+        Math.floor((index * this.uniqueCategoriesCount) / this.docCount)
+      ],
       status_code: faker.helpers.arrayElement(this.statusCodes),
       response_time: faker.datatype.float({ min: 0.1, max: 5.0, precision: 0.01 }),
       bytes_transferred: faker.datatype.number({ min: 100, max: 10000 }),
@@ -159,6 +188,10 @@ class TestDataGenerator {
           event_sequence_number: { type: 'long' }, // New field mapping
           request_url: { type: 'keyword' },
           service_endpoint: { type: 'keyword' },
+          // Field that is never present in the data. Test for hide missing values.
+          never_present_field: { type: 'keyword' },
+          // Remove for some indices to create a unique field. Test for modifyColumnsOnSwitch.
+          unique_category: { type: 'keyword' }, // Field with unique values for time_1 indices
           personal: {
             type: 'nested',
             properties: {

cypress/utils/apps/query_enhancements/dataset_selector.js

@@ -72,7 +72,7 @@ export const verifyBaseState = (dataset) => {
     dataset: dataset,
     queryString: query,
     language: language,
-    hitCount: '2,229',
+    hitCount: '2,217',
   });
 };