Skip to content

Commit 25fca57

Browse files
piotrczarnaspiotrczarnas
committed
Containers for labels to aggregate all present labels.
1 parent 7180464 commit 25fca57

29 files changed

+749
-9
lines changed

‎dqops/src/main/java/com/dqops/metadata/defaultchecks/column/TargetColumnPatternFilter.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,10 @@
1818

1919
import com.dqops.connectors.DataTypeCategory;
2020
import com.dqops.metadata.defaultchecks.table.TargetTablePatternFilter;
21-
import com.dqops.metadata.defaultchecks.table.TargetTablePatternSpec;
2221
import com.dqops.metadata.search.pattern.SearchPattern;
2322
import com.dqops.metadata.sources.ColumnSpec;
2423
import com.dqops.metadata.sources.ConnectionSpec;
25-
import com.dqops.metadata.sources.LabelSetSpec;
24+
import com.dqops.metadata.labels.LabelSetSpec;
2625
import com.dqops.metadata.sources.TableSpec;
2726
import com.google.common.base.Strings;
2827

‎dqops/src/main/java/com/dqops/metadata/defaultchecks/table/TargetTablePatternFilter.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
import com.dqops.metadata.search.pattern.SearchPattern;
2020
import com.dqops.metadata.sources.ConnectionSpec;
21-
import com.dqops.metadata.sources.LabelSetSpec;
21+
import com.dqops.metadata.labels.LabelSetSpec;
2222
import com.dqops.metadata.sources.TableSpec;
2323
import com.google.common.base.Strings;
2424

‎dqops/src/main/java/com/dqops/metadata/id/HierarchyNodeResultVisitor.java

+1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import com.dqops.metadata.incidents.IncidentWebhookNotificationsSpec;
6262
import com.dqops.metadata.incidents.TableIncidentGroupingSpec;
6363
import com.dqops.metadata.incidents.defaultnotifications.DefaultIncidentWebhookNotificationsWrapper;
64+
import com.dqops.metadata.labels.LabelSetSpec;
6465
import com.dqops.metadata.scheduling.DefaultSchedulesSpec;
6566
import com.dqops.metadata.scheduling.MonitoringScheduleSpec;
6667
import com.dqops.metadata.scheduling.MonitoringSchedulesWrapper;

‎dqops/src/main/java/com/dqops/metadata/labels/GlobalLabelsContainer.java

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Copyright © 2021 DQOps (support@dqops.com)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.dqops.metadata.labels;
18+
19+
import com.dqops.metadata.sources.PhysicalTableName;
20+
21+
/**
22+
* A global container of all labels, collected from all levels (connection, table, column).
23+
*/
24+
public interface GlobalLabelsContainer {
25+
/**
26+
* Returns the global labels container for labels defined on a connection level (assigned to the connection object).
27+
*
28+
* @return Connection level labels.
29+
*/
30+
LabelCountContainer getConnectionLabels();
31+
32+
/**
33+
* Returns the global labels container for labels defined on a table level (assigned to the table object).
34+
*
35+
* @return Table level labels.
36+
*/
37+
LabelCountContainer getTableLabels();
38+
39+
/**
40+
* Returns the global labels container for labels defined on a column level (assigned to the column object).
41+
*
42+
* @return Column level labels.
43+
*/
44+
LabelCountContainer getColumnLabels();
45+
46+
/**
47+
* Imports new labels from a connection level.
48+
*
49+
* @param connectionName Connection name.
50+
* @param newLabels New labels or null if the connection was removed and the old labels should be unregistered.
51+
*/
52+
void importConnectionLabels(String connectionName, LabelCountContainer newLabels);
53+
54+
/**
55+
* Imports new labels from a table level.
56+
*
57+
* @param connectionName Connection name.
58+
* @param schemaTableName Schema and table name.
59+
* @param newLabels New labels or null if the table was removed and the old labels should be unregistered.
60+
*/
61+
void importTableLabels(String connectionName, PhysicalTableName schemaTableName, LabelCountContainer newLabels);
62+
63+
/**
64+
* Imports new labels from a column level, but aggregated for a whole table.
65+
*
66+
* @param connectionName Connection name.
67+
* @param schemaTableName Schema and table name.
68+
* @param newLabels New labels or null if the table was removed and the old labels should be unregistered.
69+
*/
70+
void importColumnLabels(String connectionName, PhysicalTableName schemaTableName, LabelCountContainer newLabels);
71+
}

‎dqops/src/main/java/com/dqops/metadata/labels/GlobalLabelsContainerImpl.java

+152
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*
2+
* Copyright © 2021 DQOps (support@dqops.com)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.dqops.metadata.labels;
18+
19+
import com.dqops.metadata.sources.PhysicalTableName;
20+
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
21+
import org.springframework.context.annotation.Scope;
22+
import org.springframework.stereotype.Component;
23+
24+
import java.util.HashMap;
25+
import java.util.Objects;
26+
27+
/**
28+
* A global container of all labels, collected from all levels (connection, table, column).
29+
*/
30+
@Component
31+
@Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
32+
public class GlobalLabelsContainerImpl implements GlobalLabelsContainer {
33+
private final LabelCountContainer connectionLabels = new LabelCountContainer();
34+
private final LabelCountContainer tableLabels = new LabelCountContainer();
35+
private final LabelCountContainer columnLabels = new LabelCountContainer();
36+
37+
private final HashMap<String, LabelCountContainer> importedConnectionLabels = new HashMap<>();
38+
private final HashMap<TableLabelsKey, LabelCountContainer> importedTableLabels = new HashMap<>();
39+
private final HashMap<TableLabelsKey, LabelCountContainer> importedColumnLabels = new HashMap<>();
40+
private final Object lock = new Object();
41+
42+
/**
43+
* Returns the global labels container for labels defined on a connection level (assigned to the connection object).
44+
* @return Connection level labels.
45+
*/
46+
@Override
47+
public LabelCountContainer getConnectionLabels() {
48+
return connectionLabels;
49+
}
50+
51+
/**
52+
* Returns the global labels container for labels defined on a table level (assigned to the table object).
53+
* @return Table level labels.
54+
*/
55+
@Override
56+
public LabelCountContainer getTableLabels() {
57+
return tableLabels;
58+
}
59+
60+
/**
61+
* Returns the global labels container for labels defined on a column level (assigned to the column object).
62+
* @return Column level labels.
63+
*/
64+
@Override
65+
public LabelCountContainer getColumnLabels() {
66+
return columnLabels;
67+
}
68+
69+
/**
70+
* Imports new labels from a connection level.
71+
* @param connectionName Connection name.
72+
* @param newLabels New labels or null if the connection was removed and the old labels should be unregistered.
73+
*/
74+
@Override
75+
public void importConnectionLabels(String connectionName, LabelCountContainer newLabels) {
76+
synchronized (this.lock) {
77+
LabelCountContainer oldLabels = this.importedConnectionLabels.get(connectionName);
78+
if (Objects.equals(oldLabels, newLabels)) {
79+
// no change
80+
return;
81+
}
82+
83+
if (oldLabels != null) {
84+
this.connectionLabels.subtractCountsFromContainer(oldLabels);
85+
this.importedConnectionLabels.remove(connectionName);
86+
}
87+
88+
if (newLabels != null && !newLabels.isEmpty()) {
89+
this.connectionLabels.addCountsFromContainer(oldLabels);
90+
this.importedConnectionLabels.put(connectionName, newLabels);
91+
}
92+
}
93+
}
94+
95+
/**
96+
* Imports new labels from a table level.
97+
* @param connectionName Connection name.
98+
* @param schemaTableName Schema and table name.
99+
* @param newLabels New labels or null if the table was removed and the old labels should be unregistered.
100+
*/
101+
@Override
102+
public void importTableLabels(String connectionName, PhysicalTableName schemaTableName, LabelCountContainer newLabels) {
103+
TableLabelsKey tableLabelsKey = new TableLabelsKey(connectionName, schemaTableName);
104+
105+
synchronized (this.lock) {
106+
LabelCountContainer oldLabels = this.importedTableLabels.get(tableLabelsKey);
107+
if (Objects.equals(oldLabels, newLabels)) {
108+
// no change
109+
return;
110+
}
111+
112+
if (oldLabels != null) {
113+
this.tableLabels.subtractCountsFromContainer(oldLabels);
114+
this.importedTableLabels.remove(tableLabelsKey);
115+
}
116+
117+
if (newLabels != null && !newLabels.isEmpty()) {
118+
this.tableLabels.addCountsFromContainer(oldLabels);
119+
this.importedTableLabels.put(tableLabelsKey, newLabels);
120+
}
121+
}
122+
}
123+
124+
/**
125+
* Imports new labels from a column level, but aggregated for a whole table.
126+
* @param connectionName Connection name.
127+
* @param schemaTableName Schema and table name.
128+
* @param newLabels New labels or null if the table was removed and the old labels should be unregistered.
129+
*/
130+
@Override
131+
public void importColumnLabels(String connectionName, PhysicalTableName schemaTableName, LabelCountContainer newLabels) {
132+
TableLabelsKey tableLabelsKey = new TableLabelsKey(connectionName, schemaTableName);
133+
134+
synchronized (this.lock) {
135+
LabelCountContainer oldLabels = this.importedColumnLabels.get(tableLabelsKey);
136+
if (Objects.equals(oldLabels, newLabels)) {
137+
// no change
138+
return;
139+
}
140+
141+
if (oldLabels != null) {
142+
this.columnLabels.subtractCountsFromContainer(oldLabels);
143+
this.importedColumnLabels.remove(tableLabelsKey);
144+
}
145+
146+
if (newLabels != null && !newLabels.isEmpty()) {
147+
this.columnLabels.addCountsFromContainer(oldLabels);
148+
this.importedColumnLabels.put(tableLabelsKey, newLabels);
149+
}
150+
}
151+
}
152+
}

‎dqops/src/main/java/com/dqops/metadata/labels/LabelCountContainer.java

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/*
2+
* Copyright © 2021 DQOps (support@dqops.com)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.dqops.metadata.labels;
18+
19+
import java.util.ArrayList;
20+
import java.util.HashMap;
21+
import java.util.List;
22+
import java.util.Map;
23+
import java.util.stream.Collectors;
24+
25+
/**
26+
* An object that stores all found labels and counts their occurrences.
27+
*/
28+
public class LabelCountContainer {
29+
private final HashMap<String, LabelCounter> labels = new HashMap<>();
30+
private final Object lock = new Object();
31+
32+
/**
33+
* Returns a map of all root level labels in this counter.
34+
* @return Map of all root level labels.
35+
*/
36+
public List<LabelCounter> getLabels() {
37+
synchronized (this.lock) {
38+
return this.labels.values().stream().collect(Collectors.toUnmodifiableList());
39+
}
40+
}
41+
42+
/**
43+
* Checks if the container is empty (has no labels).
44+
* @return Container is empty.
45+
*/
46+
public boolean isEmpty() {
47+
synchronized (this.lock) {
48+
return this.labels.isEmpty();
49+
}
50+
}
51+
52+
/**
53+
* Increments the count of labels for a hierarchical label. If the label is in the form grandparent/parent/child, then increments
54+
* the nested labels counts for the grandparent and parent, and the label count for the child.
55+
* For simple labels, just increments the label count.
56+
* NOTE: This method is not synchronized and is not thread-safe. It should be used only to generate the original object, not the shared object.
57+
* @param label Label to add to the counter.
58+
*/
59+
public void incrementLabelCount(String label) {
60+
if (label == null || label.isEmpty()) {
61+
return;
62+
}
63+
64+
String rootLabelKey = label;
65+
int indexOfSplit = label.indexOf('/');
66+
if (indexOfSplit >= 0) {
67+
// multi-level label
68+
rootLabelKey = label.substring(0, indexOfSplit);
69+
}
70+
71+
LabelCounter rootLabel = this.labels.get(rootLabelKey);
72+
if (rootLabel != null) {
73+
rootLabel.incrementDeepChildLabel(label);
74+
return;
75+
}
76+
77+
LabelCounter newLabelCounter = LabelCounter.createLabel(label);
78+
this.labels.put(rootLabelKey, newLabelCounter);
79+
}
80+
81+
/**
82+
* Adds counts and registers new labels from a different count container.
83+
* @param otherContainer Other count container.
84+
*/
85+
public void addCountsFromContainer(LabelCountContainer otherContainer) {
86+
synchronized (this.lock) {
87+
for (Map.Entry<String, LabelCounter> otherRootLabelKeyValue : otherContainer.labels.entrySet()) {
88+
LabelCounter existingRootLabel = this.labels.get(otherRootLabelKeyValue.getKey());
89+
if (existingRootLabel != null) {
90+
existingRootLabel.addCounts(otherRootLabelKeyValue.getValue());
91+
} else {
92+
this.labels.put(otherRootLabelKeyValue.getKey(), otherRootLabelKeyValue.getValue().clone());
93+
}
94+
}
95+
}
96+
}
97+
98+
/**
99+
* Subtracts (decrements) counts and unregisters empty labels from a different count container.
100+
* @param otherContainer Other count container.
101+
*/
102+
public void subtractCountsFromContainer(LabelCountContainer otherContainer) {
103+
synchronized (this.lock) {
104+
List<String> emptyLabelsToRemove = null;
105+
for (Map.Entry<String, LabelCounter> otherRootLabelKeyValue : otherContainer.labels.entrySet()) {
106+
LabelCounter existingRootLabel = this.labels.get(otherRootLabelKeyValue.getKey());
107+
existingRootLabel.subtractCounts(otherRootLabelKeyValue.getValue());
108+
if (existingRootLabel.isEmpty()) {
109+
if (emptyLabelsToRemove == null) {
110+
emptyLabelsToRemove = new ArrayList<>();
111+
}
112+
113+
emptyLabelsToRemove.add(otherRootLabelKeyValue.getKey());
114+
}
115+
}
116+
117+
if (emptyLabelsToRemove != null) {
118+
for (String keyToRemove : emptyLabelsToRemove) {
119+
this.labels.remove(keyToRemove);
120+
}
121+
}
122+
}
123+
}
124+
125+
@Override
126+
public boolean equals(Object o) {
127+
if (this == o) return true;
128+
if (o == null || getClass() != o.getClass()) return false;
129+
130+
LabelCountContainer that = (LabelCountContainer) o;
131+
132+
return labels.equals(that.labels);
133+
}
134+
135+
@Override
136+
public int hashCode() {
137+
return labels.hashCode();
138+
}
139+
}

0 commit comments

Comments
 (0)