refactor(es): adapt to 6.x version & fix query build

n9e · Apr 6, 2023 · 2eba9ee · 2eba9ee
1 parent fe6deed
commit 2eba9ee
Show file tree

Hide file tree

Showing 9 changed files with 124 additions and 73 deletions.
diff --git a/src/pages/alertRules/Form/Rule/Rule/Log/ElasticsearchSettings/Values/index.tsx b/src/pages/alertRules/Form/Rule/Rule/Log/ElasticsearchSettings/Values/index.tsx
@@ -36,7 +36,7 @@ export default function index({ prefixField = {}, prefixFields = [], prefixNameF
   const [fieldsOptions, setFieldsOptions] = useState<any[]>([]);
   const { run } = useDebounceFn(
     () => {
-      getFields(datasourceValue, index).then((res) => {
+      getFields(datasourceValue, index, 'number').then((res) => {
         setFieldsOptions(
           _.map(res, (item) => {
             return {

diff --git a/src/pages/dashboard/Editor/QueryEditor/Elasticsearch.tsx b/src/pages/dashboard/Editor/QueryEditor/Elasticsearch.tsx
@@ -79,12 +79,7 @@ export default function Prometheus({ chartForm }) {
                         </Form.Item>
                       </Col>
                     </Row>
-                    <Form.Item
-                      shouldUpdate={(prevValues, curValues) => {
-                        return !_.isEqual(prevValues.datasourceValue, curValues.datasourceValue);
-                      }}
-                      noStyle
-                    >
+                    <Form.Item shouldUpdate noStyle>
                       {({ getFieldValue }) => {
                         const datasourceValue = getFieldValue('datasourceValue');
                         return (

diff --git a/src/pages/dashboard/Renderer/datasource/elasticsearch/index.ts b/src/pages/dashboard/Renderer/datasource/elasticsearch/index.ts
@@ -8,6 +8,7 @@ import { IVariable } from '../../../VariableConfig/definition';
 import { replaceExpressionVars } from '../../../VariableConfig/constant';
 import { getSeriesQuery, getLogsQuery } from './queryBuilder';
 import { processResponseToSeries } from './processResponse';
+import { flattenHits } from '@/pages/explorer/Elasticsearch/utils';
 
 interface IOptions {
   dashboardId: string;
@@ -103,11 +104,12 @@ export default async function elasticSearchQuery(options: IOptions) {
       });
       const res = await getDsQuery(datasourceValue, payload);
       _.forEach(res, (item) => {
-        _.forEach(item?.hits?.hits, (hit: any) => {
+        const { docs } = flattenHits(item?.hits?.hits);
+        _.forEach(docs, (doc: any) => {
           series.push({
-            id: hit._id,
-            name: hit._index,
-            metric: hit.fields,
+            id: doc._id,
+            name: doc._index,
+            metric: doc.fields,
             data: [],
           });
         });

diff --git a/src/pages/dashboard/Renderer/datasource/elasticsearch/processResponse.ts b/src/pages/dashboard/Renderer/datasource/elasticsearch/processResponse.ts
@@ -1,40 +1,85 @@
 import _ from 'lodash';
-import { ITarget } from '../../../types';
+import { getSerieName } from '@/pages/dashboard/Renderer/datasource/utils';
 
 function processAggregations(aggregations: any[], seriesList: any[], metric: { [index: string]: string }, hasCountFunc: boolean) {
   let aggId;
+  const metricObj = _.cloneDeep(metric);
   for (aggId in aggregations) {
     const buckets = aggregations[aggId].buckets;
     if (aggId === 'date') {
       const subAggs = _.omit(buckets[0], ['key', 'key_as_string', 'doc_count']);
       if (hasCountFunc) {
         seriesList.push({
-          name: 'count',
-          metric: _.cloneDeep(metric),
+          metric: {
+            ...metricObj,
+            __name__: 'count',
+          },
           data: [],
         });
       }
       _.forEach(subAggs, (_subAgg, subAggId) => {
-        seriesList.push({
-          name: subAggId,
-          metric: _.cloneDeep(metric),
-          data: [],
-        });
+        if (subAggId.indexOf('percentiles') === 0) {
+          const percentilesField = subAggId.split(' ')[1];
+          const percentiles = _subAgg.values;
+          _.forEach(percentiles, (_percentileValue, percentileKey) => {
+            seriesList.push({
+              metric: {
+                ...metricObj,
+                __name__: `p${percentileKey} ${percentilesField}`,
+              },
+              data: [],
+            });
+          });
+        } else {
+          seriesList.push({
+            metric: {
+              ...metricObj,
+              __name__: subAggId,
+            },
+            data: [],
+          });
+        }
       });
     }
     _.forEach(buckets, (bucket) => {
       const { key, doc_count } = bucket;
       const subAggs = _.omit(bucket, ['key', 'key_as_string', 'doc_count']) as any[];
       if (aggId === 'date') {
-        _.forEach(subAggs, (subAgg, subAggId) => {
-          const { value } = subAgg;
-          const series = _.find(seriesList, (s) => s.name === subAggId);
-          if (series) {
-            series.data.push([key / 1000, value]);
+        _.forEach(subAggs, (subAgg, subAggId: string) => {
+          if (subAggId.indexOf('percentiles') === 0) {
+            const percentilesField = subAggId.split(' ')[1];
+            const percentiles = subAgg.values;
+            _.forEach(percentiles, (percentileValue, percentileKey) => {
+              const series = _.find(seriesList, (s) =>
+                _.isEqual(s.metric, {
+                  ...metric,
+                  __name__: `p${percentileKey} ${percentilesField}`,
+                }),
+              );
+              if (series) {
+                series.data.push([key / 1000, percentileValue]);
+              }
+            });
+          } else {
+            const { value } = subAgg;
+            const series = _.find(seriesList, (s) =>
+              _.isEqual(s.metric, {
+                ...metric,
+                __name__: subAggId,
+              }),
+            );
+            if (series) {
+              series.data.push([key / 1000, value]);
+            }
           }
         });
         if (hasCountFunc) {
-          const series = _.find(seriesList, (s) => s.name === 'count');
+          const series = _.find(seriesList, (s) =>
+            _.isEqual(s.metric, {
+              ...metric,
+              __name__: 'count',
+            }),
+          );
           if (series) {
             series.data.push([key / 1000, doc_count]);
           }
@@ -57,6 +102,10 @@ export function processResponseToSeries(responses: any[], params: any[]) {
     const { aggregations } = response;
     processAggregations(aggregations, seriesList, {}, hasCountFunc(params[idx]));
   });
-
-  return seriesList;
+  return _.map(seriesList, (item) => {
+    return {
+      ...item,
+      name: getSerieName(item.metric),
+    };
+  });
 }
diff --git a/src/pages/dashboard/Renderer/datasource/elasticsearch/queryBuilder.ts b/src/pages/dashboard/Renderer/datasource/elasticsearch/queryBuilder.ts
@@ -26,15 +26,9 @@ export function getLogsQuery(target: ElasticsearchQuery) {
           unmapped_type: 'boolean',
         },
       },
-      {
-        _doc: {
-          order: 'desc',
-        },
-      },
     ],
     script_fields: {},
-    _source: false,
-    fields: ['*'],
+    aggs: {},
   };
   if (target.filter && target.filter !== '') {
     queryObj.query.bool.filter = [
@@ -108,7 +102,7 @@ export function getSeriesQuery(target: ElasticsearchQuery) {
             max: target.end,
           },
           format: 'epoch_millis',
-          fixed_interval: '60s',
+          interval: target.interval,
         };
         break;
       }

diff --git a/src/pages/explorer/Elasticsearch/index.tsx b/src/pages/explorer/Elasticsearch/index.tsx
@@ -45,6 +45,7 @@ export default function index(props: IProps) {
   const [data, setData] = useState<any[]>([]);
   const [series, setSeries] = useState<any[]>([]);
   const [displayTimes, setDisplayTimes] = useState('');
+  const [dateFields, setDateFields] = useState<string[]>([]);
   const [fields, setFields] = useState<string[]>([]);
   const [selectedFields, setSelectedFields] = useState<string[]>([]);
   const [isMore, setIsMore] = useState(true);
@@ -160,6 +161,9 @@ export default function index(props: IProps) {
     (val) => {
       if (datasourceValue && val) {
         getFields(datasourceValue, val).then((res) => {
+          setFields(res);
+        });
+        getFields(datasourceValue, val, 'date').then((res) => {
           const dateFiled = form.getFieldValue(['query', 'date_field']);
           if (!_.includes(res, dateFiled)) {
             if (_.includes(res, '@timestamp')) {
@@ -176,7 +180,7 @@ export default function index(props: IProps) {
               });
             }
           }
-          setFields(res);
+          setDateFields(res);
         });
       }
     },
@@ -274,7 +278,7 @@ export default function index(props: IProps) {
                 ]}
               >
                 <Select dropdownMatchSelectWidth={false} style={{ width: 150 }} showSearch>
-                  {_.map(_.sortBy(_.concat(fields, selectedFields)), (item) => {
+                  {_.map(dateFields, (item) => {
                     return (
                       <Select.Option key={item} value={item}>
                         {item}

diff --git a/src/pages/explorer/Elasticsearch/services.ts b/src/pages/explorer/Elasticsearch/services.ts
@@ -31,14 +31,13 @@ export function getIndices(datasourceValue: number) {
   });
 }
 
-export function getFields(datasourceValue: number, index?: string) {
+export function getFields(datasourceValue: number, index?: string, type?: string) {
   const url = index ? `/${index}/_mapping` : '/_mapping';
   return request(`/api/n9e/proxy/${datasourceValue}${url}?pretty=true`, {
     method: RequestMethod.Get,
     silence: true,
   }).then((res) => {
-    const fields = mappingsToFields(res);
-    return fields;
+    return mappingsToFields(res, type);
   });
 }
 

diff --git a/src/pages/explorer/Elasticsearch/utils.tsx b/src/pages/explorer/Elasticsearch/utils.tsx
@@ -72,13 +72,30 @@ interface Mappings {
   };
 }
 
-export function mappingsToFields(mappings: Mappings) {
+const typeMap: Record<string, string> = {
+  float: 'number',
+  double: 'number',
+  integer: 'number',
+  long: 'number',
+  date: 'date',
+  date_nanos: 'date',
+  string: 'string',
+  text: 'string',
+  scaled_float: 'number',
+  nested: 'nested',
+  histogram: 'number',
+};
+
+export function mappingsToFields(mappings: Mappings, type?: string) {
   const fields: string[] = [];
   _.forEach(mappings, (item: any) => {
     function loop(mappings, prefix = '') {
-      _.forEach(mappings?.properties, (item, key) => {
+      // mappings?.doc?.properties 为了兼容 6.x 版本接口
+      _.forEach(mappings?.doc?.properties || mappings?.properties, (item, key) => {
         if (item.type) {
-          fields.push(`${prefix}${key}`);
+          if (typeMap[item.type] === type || !type) {
+            fields.push(`${prefix}${key}`);
+          }
         } else {
           loop(item, `${key}.`);
         }
@@ -95,7 +112,7 @@ export function normalizeLogsQueryRequestBody(params: any) {
     ignore_unavailable: true,
     index: params.index,
   };
-  const body = {
+  const body: any = {
     size: params.limit,
     query: {
       bool: {
@@ -110,14 +127,6 @@ export function normalizeLogsQueryRequestBody(params: any) {
             },
           },
         ],
-        must: [
-          {
-            query_string: {
-              analyze_wildcard: true,
-              query: params.filter || '*',
-            },
-          },
-        ],
       },
     },
     sort: [
@@ -130,8 +139,15 @@ export function normalizeLogsQueryRequestBody(params: any) {
     ],
     script_fields: {},
     aggs: {},
-    // fields: ['*'],
   };
+  if (params.filter) {
+    body.query.bool.filter.push({
+      query_string: {
+        analyze_wildcard: true,
+        query: params.filter || '*',
+      },
+    });
+  }
   return `${JSON.stringify(header)}\n${JSON.stringify(body)}\n`;
 }
 
@@ -141,7 +157,7 @@ export function normalizeTimeseriesQueryRequestBody(params: any) {
     ignore_unavailable: true,
     index: params.index,
   };
-  const body = {
+  const body: any = {
     size: params.limit,
     query: {
       bool: {
@@ -156,24 +172,8 @@ export function normalizeTimeseriesQueryRequestBody(params: any) {
             },
           },
         ],
-        must: [
-          {
-            query_string: {
-              analyze_wildcard: true,
-              query: params.filter || '*',
-            },
-          },
-        ],
       },
     },
-    sort: [
-      {
-        [params.date_field]: {
-          order: 'desc',
-          unmapped_type: 'boolean',
-        },
-      },
-    ],
     script_fields: {},
     _source: false,
     aggs: {
@@ -186,12 +186,20 @@ export function normalizeTimeseriesQueryRequestBody(params: any) {
             max: params.end,
           },
           format: 'epoch_millis',
-          fixed_interval: params.interval,
+          interval: params.interval,
         },
         aggs: {},
       },
     },
   };
+  if (params.filter) {
+    body.query.bool.filter.push({
+      query_string: {
+        analyze_wildcard: true,
+        query: params.filter || '*',
+      },
+    });
+  }
   return `${JSON.stringify(header)}\n${JSON.stringify(body)}\n`;
 }
 

diff --git a/src/pages/explorer/Explorer.tsx b/src/pages/explorer/Explorer.tsx
@@ -148,11 +148,11 @@ const Panel = ({ defaultPromQL, removePanel, id, cateOptions, type, defaultCate
             const datasourceCate = getFieldValue('datasourceCate');
             const datasourceValue = getFieldValue('datasourceValue');
             if (datasourceCate === DatasourceCateEnum.elasticsearch) {
-              return <Elasticsearch datasourceValue={datasourceValue} form={form} />;
+              return <Elasticsearch key={datasourceValue} datasourceValue={datasourceValue} form={form} />;
             } else if (datasourceCate === DatasourceCateEnum.aliyunSLS) {
               return <AliyunSLS datasourceCate={DatasourceCateEnum.aliyunSLS} datasourceName={datasourceValue} headerExtra={headerExtraRef.current} form={form} />;
             } else if (datasourceCate === DatasourceCateEnum.prometheus) {
-              return <Prometheus defaultPromQL={defaultPromQL} headerExtra={headerExtraRef.current} datasourceValue={datasourceValue} />;
+              return <Prometheus key={datasourceValue} defaultPromQL={defaultPromQL} headerExtra={headerExtraRef.current} datasourceValue={datasourceValue} />;
             }
           }}
         </Form.Item>