Bump dependencies and fix mock classes checks

RADAR-base · May 9, 2022 · 5897fe9 · 5897fe9
1 parent 628e337
commit 5897fe9
Show file tree

Hide file tree

Showing 15 changed files with 92 additions and 38 deletions.
diff --git a/build.gradle b/build.gradle
@@ -121,5 +121,5 @@ nexusPublishing {
 }
 
 wrapper {
-    gradleVersion '7.4'
+    gradleVersion '7.4.2'
 }
diff --git a/gradle.properties b/gradle.properties
@@ -1,11 +1,12 @@
 slf4jVersion=1.7.36
-confluentVersion=7.0.1
+confluentVersion=7.1.1
 kafkaVersion=3.1.0
 avroVersion=1.11.0
-jacksonVersion=2.13.1
+jacksonVersion=2.13.2.20220328
 okhttpVersion=4.9.3
 junitVersion=4.13.2
-mockitoVersion=4.3.1
+mockitoVersion=4.5.1
 hamcrestVersion=1.3
-radarSchemasVersion=0.7.7
-orgJsonVersion=20211205
+radarSchemasVersion=0.7.9
+orgJsonVersion=20220320
+opencsvVersion=5.6
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/radar-commons-server/build.gradle b/radar-commons-server/build.gradle
@@ -22,12 +22,12 @@ dependencies {
     api project(':radar-commons')
 
     // For POJO classes and ConfigLoader
-    implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion
-    implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion
+    implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion"))
+    implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind'
+    implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml'
 
     api group: 'org.apache.avro', name: 'avro', version: avroVersion
 
-    implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion
     implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion
 
     testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion

diff --git a/radar-commons-testing/build.gradle b/radar-commons-testing/build.gradle
@@ -39,8 +39,9 @@ dependencies {
     api group: 'org.apache.avro', name: 'avro', version: avroVersion
     api group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion
 
-    implementation group: 'com.opencsv', name: 'opencsv', version: '5.5.2'
-    implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion
+    implementation group: 'com.opencsv', name: 'opencsv', version: opencsvVersion
+    implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion"))
+    implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind'
     implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion
     implementation (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) {
         exclude group: 'com.101tec'

diff --git a/radar-commons-testing/mock/light.csv b/radar-commons-testing/mock/light.csv
@@ -1,2 +1,2 @@
-projectId,userId,sourceId,time,timeReceived,light
+key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light
 radar,d90c8b88-5793-438a-b27d-6c87580cc3d9,mock,1613658505.0,1613658505.1,1.0
diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/config/AuthConfig.java b/radar-commons-testing/src/main/java/org/radarbase/mock/config/AuthConfig.java
@@ -37,6 +37,10 @@ public void setTokenUrl(String tokenUrl) {
         this.tokenUrl = tokenUrl;
     }
 
+    /**
+     * Fill in the client ID and client secret from environment variables. The variables are
+     * {@code &lt;prefix&gt;_CLIENT_ID} and {@code &lt;prefix&gt;_CLIENT_SECRET}.
+     */
     public void withEnv(String prefix) {
         String envClientId = System.getenv(prefix + "_CLIENT_ID");
         if (envClientId != null) {

diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.java
@@ -11,25 +11,48 @@ public class HeaderHierarchy {
     private final HeaderHierarchy parent;
     private final String name;
 
+    /** Root node. */
     public HeaderHierarchy() {
         this(null, -1, null);
     }
 
+    /**
+     * Header hierarchy child node. Usually accessed via {@link #add(int, List)}
+     *
+     * @param name name of the node
+     * @param index index in the csv file. -1 if not a leaf node.
+     * @param parent parent node.
+     */
     public HeaderHierarchy(String name, int index, HeaderHierarchy parent) {
         this.name = name;
         this.index = index;
         this.children = new HashMap<>();
         this.parent = parent;
     }
 
+    /**
+     * Add child nodes to this node. Each item in the list will become a new level down, and the
+     * last item will become a leaf node with given index.
+     *
+     * @param i index if the item.
+     * @param item list of item elements, each one level deeper than the previous.
+     */
     public void add(int i, List<String> item) {
         Objects.requireNonNull(item);
-        if (item.size() == 0) return;
+        if (item.isEmpty()) {
+            return;
+        }
         HeaderHierarchy child = this.children.computeIfAbsent(item.get(0),
                 k -> new HeaderHierarchy(k, item.size() == 1 ? i : -1, this));
         child.add(i, item.subList(1, item.size()));
     }
 
+    /**
+     * Get the index of current element.
+     *
+     * @return index
+     * @throws IllegalStateException if current node is not a leaf.
+     */
     public int getIndex() {
         if (index == -1) {
             throw new IllegalStateException("Header does not exist");
@@ -50,6 +73,7 @@ private void appendTo(StringBuilder builder) {
         }
     }
 
+    @Override
     public String toString() {
         StringBuilder builder = new StringBuilder(50);
         appendTo(builder);

diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java
@@ -68,7 +68,8 @@ public class MockCsvParser implements Closeable {
     public MockCsvParser(MockDataConfig config, Path root, Instant startTime,
             SchemaRetriever retriever)
             throws IOException, CsvValidationException {
-        Schema keySchema, valueSchema;
+        Schema keySchema;
+        Schema valueSchema;
         try {
             AvroTopic<SpecificRecord, SpecificRecord> specificTopic = config.parseAvroTopic();
             keySchema = specificTopic.getKeySchema();
@@ -89,7 +90,8 @@ public MockCsvParser(MockDataConfig config, Path root, Instant startTime,
         rowDuration = Duration.ofMillis((long)(1.0 / config.getFrequency()));
         rowTime = this.startTime.toEpochMilli();
 
-        bufferedReader = Files.newBufferedReader(config.getDataFile(root));
+        Path dataFile = config.getDataFile(root);
+        bufferedReader = Files.newBufferedReader(dataFile);
         csvReader = new CSVReader(bufferedReader);
         headers = new HeaderHierarchy();
         String[] header = csvReader.readNext();
@@ -115,15 +117,20 @@ public Record<GenericRecord, GenericRecord> next() throws IOException, CsvValida
             throw new IllegalStateException("No next record available");
         }
 
-        GenericRecord key = parseRecord(currentLine, topic.getKeySchema(), headers.getChildren().get("key"));
-        GenericRecord value = parseRecord(currentLine, topic.getValueSchema(), headers.getChildren().get("value"));
+        GenericRecord key = parseRecord(currentLine, topic.getKeySchema(),
+                headers.getChildren().get("key"));
+        GenericRecord value = parseRecord(currentLine, topic.getValueSchema(),
+                headers.getChildren().get("value"));
+        incrementRow();
+        return new Record<>(key, value);
+    }
 
+    private void incrementRow() throws CsvValidationException, IOException {
         currentLine = csvReader.readNext();
         row++;
         rowTime = startTime
                 .plus(rowDuration.multipliedBy(row))
                 .toEpochMilli();
-        return new Record<>(key, value);
     }
 
     /**
@@ -176,6 +183,9 @@ public Object parseValue(String[] rawValues, Schema schema, HeaderHierarchy head
 
     private Object parseScalar(String[] rawValues, Schema schema, HeaderHierarchy headers) {
         int fieldHeader = headers.getIndex();
+        if (fieldHeader >= rawValues.length) {
+            throw new IllegalArgumentException("Row is missing value for " + headers.getName());
+        }
         String fieldString = rawValues[fieldHeader]
                 .replace("${timeSeconds}", Double.toString(rowTime / 1000.0))
                 .replace("${timeMillis}", Long.toString(rowTime));
@@ -213,7 +223,8 @@ private static Object parseScalar(String fieldString, Schema schema, HeaderHiera
         }
     }
 
-    private Map<String, Object> parseMap(String[] rawValues, Schema schema, HeaderHierarchy headers) {
+    private Map<String, Object> parseMap(String[] rawValues, Schema schema,
+            HeaderHierarchy headers) {
         Map<String, HeaderHierarchy> children = headers.getChildren();
         Map<String, Object> map = new LinkedHashMap<>(children.size() * 4 / 3);
         for (HeaderHierarchy child : children.values()) {

diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java
@@ -60,8 +60,7 @@ public MockRecordValidator(MockDataConfig config, long duration, Path root,
      */
     public void validate() {
         Instant now = Instant.now();
-        try (MockCsvParser parser = new MockCsvParser(config, root, now,
-                retriever)) {
+        try (MockCsvParser parser = new MockCsvParser(config, root, now, retriever)) {
             if (!parser.hasNext()) {
                 throw new IllegalArgumentException("CSV file is empty");
             }

diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java
@@ -105,8 +105,19 @@ public RecordGenerator(MockDataConfig config, Class<K> keyClass) {
         }
     }
 
+    /**
+     * Get the header with correct prefixes.
+     * @return generated header
+     */
     public List<String> getHeader() {
-        return header;
+        List<String> withPrefix = new ArrayList<>(header);
+        for (int i = 0; i < 3; i++) {
+            withPrefix.set(i, "key." + withPrefix.get(i));
+        }
+        for (int i = 3; i < header.size(); i++) {
+            withPrefix.set(i, "value." + withPrefix.get(i));
+        }
+        return withPrefix;
     }
 
     /** Get given schema field, and throw an IllegalArgumentException if it does not exists. */

diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java
@@ -55,7 +55,7 @@ public void generateMockConfig() throws IOException, CsvValidationException {
         Path p = Paths.get(config.getDataFile());
         try (Reader reader = Files.newBufferedReader(p);
                 CSVReader parser = new CSVReader(reader)) {
-            String[] headers = {"projectId", "userId", "sourceId", "time", "timeReceived", "light"};
+            String[] headers = {"key.projectId", "key.userId", "key.sourceId", "value.time", "value.timeReceived", "value.light"};
             assertArrayEquals(headers, parser.readNext());
 
             int n = 0;

diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java
@@ -73,8 +73,8 @@ public void getHeaders() throws Exception {
         RecordGenerator<ObservationKey> generator = new RecordGenerator<>(config,
                 ObservationKey.class);
         assertEquals(
-                Arrays.asList("projectId", "userId", "sourceId",
-                        "time", "timeReceived", "x", "y", "z"),
+                Arrays.asList("key.projectId", "key.userId", "key.sourceId",
+                        "value.time", "value.timeReceived", "value.x", "value.y", "value.z"),
                 generator.getHeader());
     }
-}
+}
diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java
@@ -30,6 +30,7 @@
 import org.radarbase.mock.config.MockDataConfig;
 import org.radarbase.producer.rest.SchemaRetriever;
 import org.radarcns.monitor.application.ApplicationServerStatus;
+import org.radarcns.kafka.ObservationKey;
 import org.radarcns.passive.phone.PhoneAcceleration;
 import org.radarcns.passive.phone.PhoneLight;
 
@@ -52,6 +53,7 @@ public void setUp() {
 
     public static MockDataConfig makeConfig(TemporaryFolder folder) throws IOException {
         MockDataConfig config = new MockDataConfig();
+        config.setKeySchema(ObservationKey.class.getName());
         config.setDataFile(folder.newFile().getAbsolutePath());
         config.setValueSchema(PhoneLight.class.getName());
         config.setValueField("light");
@@ -65,7 +67,7 @@ public void validateEnum() throws IOException {
         config.setValueSchema(ApplicationServerStatus.class.getName());
 
         try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) {
-            writer.append("projectId,userId,sourceId,time,serverStatus,ipAddress\n");
+            writer.append("key.projectId,key.userId,key.sourceId,value.time,value.serverStatus,value.ipAddress\n");
             writer.append("test,a,b,1,UNKNOWN,\n");
             writer.append("test,a,b,2,CONNECTED,\n");
         }
@@ -109,7 +111,7 @@ public void validateCustom() throws Exception {
         MockDataConfig config = makeConfig();
 
         try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) {
-            writer.append("projectId,userId,sourceId,time,timeReceived,light\n");
+            writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n");
             writer.append("test,a,b,1,1,1\n");
             writer.append("test,a,b,1,2,1\n");
         }
@@ -122,7 +124,7 @@ public void validateWrongKey() throws Exception {
         MockDataConfig config = makeConfig();
 
         try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) {
-            writer.append("projectId,userId,sourceId,time,timeReceived,light\n");
+            writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n");
             writer.append("test,a,b,1,1,1\n");
             writer.append("test,a,c,1,2,1\n");
         }
@@ -135,7 +137,7 @@ public void validateWrongTime() throws Exception {
         MockDataConfig config = makeConfig();
 
         try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) {
-            writer.append("projectId,userId,sourceId,time,timeReceived,light\n");
+            writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n");
             writer.append("test,a,b,1,1,1\n");
             writer.append("test,a,b,1,0,1\n");
         }
@@ -149,7 +151,7 @@ public void validateMissingKeyField() throws Exception {
         MockDataConfig config = makeConfig();
 
         try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) {
-            writer.append("projectId,userId,time,timeReceived,light\n");
+            writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n");
             writer.append("test,a,1,1,1\n");
             writer.append("test,a,1,2,1\n");
         }
@@ -162,7 +164,7 @@ public void validateMissingValueField() throws Exception {
         MockDataConfig config = makeConfig();
 
         try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) {
-            writer.append("projectId,userId,sourceId,time,light\n");
+            writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n");
             writer.append("test,a,b,1,1\n");
             writer.append("test,a,b,1,2\n");
         }
@@ -175,20 +177,20 @@ public void validateMissingValue() throws Exception {
         MockDataConfig config = makeConfig();
 
         try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) {
-            writer.append("projectId,userId,sourceId,time,timeReceived,light\n");
+            writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n");
             writer.append("test,a,b,1,1\n");
             writer.append("test,a,b,1,2,1\n");
         }
 
-        assertValidateThrows(ArrayIndexOutOfBoundsException.class, config);
+        assertValidateThrows(IllegalArgumentException.class, config);
     }
 
     @Test
     public void validateWrongValueType() throws Exception {
         MockDataConfig config = makeConfig();
 
         try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) {
-            writer.append("projectId,userId,sourceId,time,timeReceived,light\n");
+            writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n");
             writer.append("test,a,b,1,1,a\n");
             writer.append("test,a,b,1,2,b\n");
         }
@@ -203,7 +205,7 @@ public void validateMultipleFields() throws Exception {
         config.setValueFields(Arrays.asList("x", "y", "z"));
 
         try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) {
-            writer.append("projectId,userId,sourceId,time,timeReceived,x,y,z\n");
+            writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.x,value.y,value.z\n");
             writer.append("test,a,b,1,1,1,1,1\n");
             writer.append("test,a,b,1,2,1,1,1\n");
         }

diff --git a/radar-commons/build.gradle b/radar-commons/build.gradle
@@ -26,7 +26,8 @@ dependencies {
     // The production code uses the SLF4J logging API at compile time
     implementation group: 'org.slf4j', name:'slf4j-api', version: slf4jVersion
 
-    testImplementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion
+    testImplementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion"))
+    testImplementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind'
     testImplementation group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion
     testImplementation group: 'junit', name: 'junit', version: junitVersion
     testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion