From 2eba9eee23a7db4e90b1342b9a166ae9dac9ddb2 Mon Sep 17 00:00:00 2001 From: jsers Date: Thu, 6 Apr 2023 18:19:33 +0800 Subject: [PATCH] refactor(es): adapt to 6.x version & fix query build --- .../ElasticsearchSettings/Values/index.tsx | 2 +- .../Editor/QueryEditor/Elasticsearch.tsx | 7 +- .../datasource/elasticsearch/index.ts | 10 ++- .../elasticsearch/processResponse.ts | 81 +++++++++++++++---- .../datasource/elasticsearch/queryBuilder.ts | 10 +-- src/pages/explorer/Elasticsearch/index.tsx | 8 +- src/pages/explorer/Elasticsearch/services.ts | 5 +- src/pages/explorer/Elasticsearch/utils.tsx | 70 +++++++++------- src/pages/explorer/Explorer.tsx | 4 +- 9 files changed, 124 insertions(+), 73 deletions(-) diff --git a/src/pages/alertRules/Form/Rule/Rule/Log/ElasticsearchSettings/Values/index.tsx b/src/pages/alertRules/Form/Rule/Rule/Log/ElasticsearchSettings/Values/index.tsx index 160fe7c2c..85e6c17e9 100644 --- a/src/pages/alertRules/Form/Rule/Rule/Log/ElasticsearchSettings/Values/index.tsx +++ b/src/pages/alertRules/Form/Rule/Rule/Log/ElasticsearchSettings/Values/index.tsx @@ -36,7 +36,7 @@ export default function index({ prefixField = {}, prefixFields = [], prefixNameF const [fieldsOptions, setFieldsOptions] = useState([]); const { run } = useDebounceFn( () => { - getFields(datasourceValue, index).then((res) => { + getFields(datasourceValue, index, 'number').then((res) => { setFieldsOptions( _.map(res, (item) => { return { diff --git a/src/pages/dashboard/Editor/QueryEditor/Elasticsearch.tsx b/src/pages/dashboard/Editor/QueryEditor/Elasticsearch.tsx index e72a5aed9..f910c5a39 100644 --- a/src/pages/dashboard/Editor/QueryEditor/Elasticsearch.tsx +++ b/src/pages/dashboard/Editor/QueryEditor/Elasticsearch.tsx @@ -79,12 +79,7 @@ export default function Prometheus({ chartForm }) { - { - return !_.isEqual(prevValues.datasourceValue, curValues.datasourceValue); - }} - noStyle - > + {({ getFieldValue }) => { const datasourceValue = getFieldValue('datasourceValue'); return ( diff --git a/src/pages/dashboard/Renderer/datasource/elasticsearch/index.ts b/src/pages/dashboard/Renderer/datasource/elasticsearch/index.ts index d4fe43265..f3c390035 100644 --- a/src/pages/dashboard/Renderer/datasource/elasticsearch/index.ts +++ b/src/pages/dashboard/Renderer/datasource/elasticsearch/index.ts @@ -8,6 +8,7 @@ import { IVariable } from '../../../VariableConfig/definition'; import { replaceExpressionVars } from '../../../VariableConfig/constant'; import { getSeriesQuery, getLogsQuery } from './queryBuilder'; import { processResponseToSeries } from './processResponse'; +import { flattenHits } from '@/pages/explorer/Elasticsearch/utils'; interface IOptions { dashboardId: string; @@ -103,11 +104,12 @@ export default async function elasticSearchQuery(options: IOptions) { }); const res = await getDsQuery(datasourceValue, payload); _.forEach(res, (item) => { - _.forEach(item?.hits?.hits, (hit: any) => { + const { docs } = flattenHits(item?.hits?.hits); + _.forEach(docs, (doc: any) => { series.push({ - id: hit._id, - name: hit._index, - metric: hit.fields, + id: doc._id, + name: doc._index, + metric: doc.fields, data: [], }); }); diff --git a/src/pages/dashboard/Renderer/datasource/elasticsearch/processResponse.ts b/src/pages/dashboard/Renderer/datasource/elasticsearch/processResponse.ts index 5d1b4f9c5..959615658 100644 --- a/src/pages/dashboard/Renderer/datasource/elasticsearch/processResponse.ts +++ b/src/pages/dashboard/Renderer/datasource/elasticsearch/processResponse.ts @@ -1,40 +1,85 @@ import _ from 'lodash'; -import { ITarget } from '../../../types'; +import { getSerieName } from '@/pages/dashboard/Renderer/datasource/utils'; function processAggregations(aggregations: any[], seriesList: any[], metric: { [index: string]: string }, hasCountFunc: boolean) { let aggId; + const metricObj = _.cloneDeep(metric); for (aggId in aggregations) { const buckets = aggregations[aggId].buckets; if (aggId === 'date') { const subAggs = _.omit(buckets[0], ['key', 'key_as_string', 'doc_count']); if (hasCountFunc) { seriesList.push({ - name: 'count', - metric: _.cloneDeep(metric), + metric: { + ...metricObj, + __name__: 'count', + }, data: [], }); } _.forEach(subAggs, (_subAgg, subAggId) => { - seriesList.push({ - name: subAggId, - metric: _.cloneDeep(metric), - data: [], - }); + if (subAggId.indexOf('percentiles') === 0) { + const percentilesField = subAggId.split(' ')[1]; + const percentiles = _subAgg.values; + _.forEach(percentiles, (_percentileValue, percentileKey) => { + seriesList.push({ + metric: { + ...metricObj, + __name__: `p${percentileKey} ${percentilesField}`, + }, + data: [], + }); + }); + } else { + seriesList.push({ + metric: { + ...metricObj, + __name__: subAggId, + }, + data: [], + }); + } }); } _.forEach(buckets, (bucket) => { const { key, doc_count } = bucket; const subAggs = _.omit(bucket, ['key', 'key_as_string', 'doc_count']) as any[]; if (aggId === 'date') { - _.forEach(subAggs, (subAgg, subAggId) => { - const { value } = subAgg; - const series = _.find(seriesList, (s) => s.name === subAggId); - if (series) { - series.data.push([key / 1000, value]); + _.forEach(subAggs, (subAgg, subAggId: string) => { + if (subAggId.indexOf('percentiles') === 0) { + const percentilesField = subAggId.split(' ')[1]; + const percentiles = subAgg.values; + _.forEach(percentiles, (percentileValue, percentileKey) => { + const series = _.find(seriesList, (s) => + _.isEqual(s.metric, { + ...metric, + __name__: `p${percentileKey} ${percentilesField}`, + }), + ); + if (series) { + series.data.push([key / 1000, percentileValue]); + } + }); + } else { + const { value } = subAgg; + const series = _.find(seriesList, (s) => + _.isEqual(s.metric, { + ...metric, + __name__: subAggId, + }), + ); + if (series) { + series.data.push([key / 1000, value]); + } } }); if (hasCountFunc) { - const series = _.find(seriesList, (s) => s.name === 'count'); + const series = _.find(seriesList, (s) => + _.isEqual(s.metric, { + ...metric, + __name__: 'count', + }), + ); if (series) { series.data.push([key / 1000, doc_count]); } @@ -57,6 +102,10 @@ export function processResponseToSeries(responses: any[], params: any[]) { const { aggregations } = response; processAggregations(aggregations, seriesList, {}, hasCountFunc(params[idx])); }); - - return seriesList; + return _.map(seriesList, (item) => { + return { + ...item, + name: getSerieName(item.metric), + }; + }); } diff --git a/src/pages/dashboard/Renderer/datasource/elasticsearch/queryBuilder.ts b/src/pages/dashboard/Renderer/datasource/elasticsearch/queryBuilder.ts index cd066d176..183cb1930 100644 --- a/src/pages/dashboard/Renderer/datasource/elasticsearch/queryBuilder.ts +++ b/src/pages/dashboard/Renderer/datasource/elasticsearch/queryBuilder.ts @@ -26,15 +26,9 @@ export function getLogsQuery(target: ElasticsearchQuery) { unmapped_type: 'boolean', }, }, - { - _doc: { - order: 'desc', - }, - }, ], script_fields: {}, - _source: false, - fields: ['*'], + aggs: {}, }; if (target.filter && target.filter !== '') { queryObj.query.bool.filter = [ @@ -108,7 +102,7 @@ export function getSeriesQuery(target: ElasticsearchQuery) { max: target.end, }, format: 'epoch_millis', - fixed_interval: '60s', + interval: target.interval, }; break; } diff --git a/src/pages/explorer/Elasticsearch/index.tsx b/src/pages/explorer/Elasticsearch/index.tsx index 2eaa28a1f..0d80e961b 100644 --- a/src/pages/explorer/Elasticsearch/index.tsx +++ b/src/pages/explorer/Elasticsearch/index.tsx @@ -45,6 +45,7 @@ export default function index(props: IProps) { const [data, setData] = useState([]); const [series, setSeries] = useState([]); const [displayTimes, setDisplayTimes] = useState(''); + const [dateFields, setDateFields] = useState([]); const [fields, setFields] = useState([]); const [selectedFields, setSelectedFields] = useState([]); const [isMore, setIsMore] = useState(true); @@ -160,6 +161,9 @@ export default function index(props: IProps) { (val) => { if (datasourceValue && val) { getFields(datasourceValue, val).then((res) => { + setFields(res); + }); + getFields(datasourceValue, val, 'date').then((res) => { const dateFiled = form.getFieldValue(['query', 'date_field']); if (!_.includes(res, dateFiled)) { if (_.includes(res, '@timestamp')) { @@ -176,7 +180,7 @@ export default function index(props: IProps) { }); } } - setFields(res); + setDateFields(res); }); } }, @@ -274,7 +278,7 @@ export default function index(props: IProps) { ]} >