diff --git a/.travis.yml b/.travis.yml index 8a032264..1f8c508b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: java jdk: - - oraclejdk8 + - oraclejdk11 sudo: false cache: @@ -12,8 +12,6 @@ cache: - $HOME/.gradle/native - $HOME/.gradle/wrapper -before_install: - - ./gradlew downloadDependencies after_script: - ./gradlew sendCoverageToCodacy diff --git a/README.md b/README.md index dced8fce..512b524f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ repositories { } dependencies { - implementation group: 'org.radarcns', name: 'radar-commons', version: '0.11.3' + implementation group: 'org.radarbase', name: 'radar-commons', version: '0.12.0' } ``` @@ -26,21 +26,21 @@ repositories { } dependencies { - implementation group: 'org.radarcns', name: 'radar-commons-server', version: '0.11.3' + implementation group: 'org.radarbase', name: 'radar-commons-server', version: '0.12.0' } ``` -For mocking clients of the RADAR-CNS infrastructure, use that 'radar-commons-testing' repository: +For mocking clients of the RADAR-base infrastructure, use that 'radar-commons-testing' repository: ```gradle repositories { jcenter() maven { url 'http://packages.confluent.io/maven/' } - maven { url 'http://dl.bintray.com/radar-cns/org.radarcns' } + maven { url 'http://dl.bintray.com/radar-base/org.radarbase' } } dependencies { - testImplementation group: 'org.radarcns', name: 'radar-commons-testing', version: '0.11.3' + testImplementation group: 'org.radarbase', name: 'radar-commons-testing', version: '0.12.0' } ``` @@ -49,11 +49,11 @@ Finally, if the schema registry is losing old schemas and your code is not recov repositories { jcenter() maven { url 'http://packages.confluent.io/maven/' } - maven { url 'http://dl.bintray.com/radar-cns/org.radarcns' } + maven { url 'http://dl.bintray.com/radar-base/org.radarbase' } } dependencies { - runtimeOnly group: 'org.radarcns', name: 'radar-commons-unsafe', version: '0.11.3' + runtimeOnly group: 'org.radarbase', name: 'radar-commons-unsafe', version: '0.12.0' } ``` @@ -78,9 +78,9 @@ configurations.all { } dependencies { - compile group: 'org.radarcns', name: 'radar-commons', version: '0.11.4-SNAPSHOT', changing: true + compile group: 'org.radarbase', name: 'radar-commons', version: '0.12.1-SNAPSHOT' } ``` Code should be formatted using the [Google Java Code Style Guide](https://google.github.io/styleguide/javaguide.html). -If you want to contribute a feature or fix browse our [issues](https://github.com/RADAR-CNS/RADAR-Commons/issues), and please make a pull request. +If you want to contribute a feature or fix browse our [issues](https://github.com/RADAR-base/radar-commons/issues), and please make a pull request. diff --git a/build.gradle b/build.gradle index dcb68b01..d5dadd3e 100644 --- a/build.gradle +++ b/build.gradle @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - plugins { // Get bintray version - id 'com.jfrog.bintray' version '1.8.1' apply false - id 'com.jfrog.artifactory' version '4.7.5' apply false + id 'com.jfrog.bintray' version '1.8.4' apply false + id 'com.jfrog.artifactory' version '4.9.1' apply false + id 'com.commercehub.gradle.plugin.avro' version '0.16.0' } subprojects { @@ -25,37 +25,31 @@ subprojects { apply plugin: 'java' apply plugin: 'java-library' apply plugin: 'idea' - apply plugin: 'checkstyle' - apply plugin: 'pmd' - apply plugin: 'com.jfrog.bintray' - apply plugin: 'maven-publish' - apply plugin: 'jacoco' - apply plugin: 'com.jfrog.artifactory' //---------------------------------------------------------------------------// // Configuration // //---------------------------------------------------------------------------// - version = '0.11.3' - group = 'org.radarcns' - ext.githubRepoName = 'RADAR-CNS/RADAR-Commons' + version = '0.12.0' + group = 'org.radarbase' + ext.githubRepoName = 'RADAR-base/radar-commons' - ext.slf4jVersion = '1.7.25' - ext.kafkaVersion = '2.0.0' + ext.slf4jVersion = '1.7.26' + ext.kafkaVersion = '2.1.1' ext.avroVersion = '1.8.2' - ext.confluentVersion = '5.0.0' - ext.jacksonVersion = '2.9.7' - ext.okhttpVersion = '3.11.0' + ext.confluentVersion = '5.1.2' + ext.jacksonVersion = '2.9.8' + ext.okhttpVersion = '3.13.1' ext.junitVersion = '4.12' - ext.mockitoVersion = '2.13.0' + ext.mockitoVersion = '2.24.5' ext.hamcrestVersion = '1.3' - ext.codacyVersion = '2.0.1' - ext.radarSchemasVersion = '0.4.0' - ext.orgJsonVersion = '20170516' + ext.codacyVersion = '4.0.5' + ext.radarSchemasVersion = '0.4.3' + ext.orgJsonVersion = '20180813' - ext.githubUrl = 'https://github.com/' + githubRepoName + '.git' - ext.issueUrl = 'https://github.com/' + githubRepoName + '/issues' - ext.website = 'http://radar-cns.org' + ext.githubUrl = "https://github.com/$githubRepoName" + ext.issueUrl = "https://github.com/$githubRepoName/issues" + ext.website = 'http://radar-base.org' //---------------------------------------------------------------------------// // Dependencies // @@ -69,48 +63,6 @@ subprojects { } } - ext.pomConfig = { - licenses { - license { - name 'The Apache Software License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - distribution 'repo' - } - } - developers { - developer { - id 'nivemaham' - name 'Nivethika Mahasivam' - email 'nivethika@thehyve.nl' - organization 'The Hyve' - } - developer { - id 'blootsvoets' - name 'Joris Borgdorff' - email 'joris@thehyve.nl' - organization 'The Hyve' - } - developer { - id 'fnobilia' - name 'Francesco Nobilia' - email 'francesco.nobilia@kcl.ac.uk' - organization 'King\'s College London' - } - } - issueManagement { - system 'GitHub' - url githubUrl + '/issues' - } - organization { - name 'RADAR-CNS' - url website - } - scm { - connection 'scm:git:' + githubUrl - url githubUrl - } - } - idea { module { downloadSources = true @@ -121,26 +73,6 @@ subprojects { // Style checking // //---------------------------------------------------------------------------// - checkstyle { - // codacy version - toolVersion '6.16' - ignoreFailures false - - configFile = rootProject.file('config/checkstyle/checkstyle.xml') - } - - pmd { - // codacy version - toolVersion = '5.5.2' - ignoreFailures = false - - consoleOutput = true - - ruleSets = [] - ruleSetFiles = files(rootProject.file("config/pmd/ruleset.xml")) - } - - tasks.matching { it instanceof Test }.all { def stdout = new LinkedList() beforeTest { TestDescriptor td -> @@ -179,5 +111,5 @@ subprojects { } wrapper { - gradleVersion '4.10.2' + gradleVersion '5.2.1' } diff --git a/config/pmd/ruleset.xml b/config/pmd/ruleset.xml index d7b23e4c..ca3f5871 100644 --- a/config/pmd/ruleset.xml +++ b/config/pmd/ruleset.xml @@ -8,82 +8,76 @@ This ruleset was parsed from the Codacy default codestyle. - - - - - - - - - - - - - - - + + + + + + + + + + + - + - + - + - - - - - + - - - - - - - - - - - - - + + + + - - + + + + - - - - - - - - + + + + + + + + - - - - - - - \ No newline at end of file + + + + + + + + + + + + + + + + diff --git a/gradle/codestyle.gradle b/gradle/codestyle.gradle new file mode 100644 index 00000000..f2bd9bd1 --- /dev/null +++ b/gradle/codestyle.gradle @@ -0,0 +1,55 @@ +apply plugin: 'checkstyle' +apply plugin: 'pmd' +apply plugin: 'jacoco' + +configurations { + codacy +} + +dependencies { + codacy group: 'com.github.codacy', name: 'codacy-coverage-reporter', version: codacyVersion +} + +checkstyle { + // codacy version + toolVersion '6.16' + ignoreFailures false + + configFile = rootProject.file('config/checkstyle/checkstyle.xml') +} + +pmd { + // codacy version + toolVersion = '6.12.0' + ignoreFailures = false + + consoleOutput = true + + ruleSets = [] + ruleSetFiles = files(rootProject.file("config/pmd/ruleset.xml")) +} + +checkstyle { + // ignore tests + sourceSets = [sourceSets.main] +} + +pmd { + // ignore tests + sourceSets = [sourceSets.main] +} + +jacocoTestReport { + reports { + xml.enabled = true + csv.enabled = false + html.enabled = false + } +} + +task sendCoverageToCodacy(type: JavaExec, dependsOn: jacocoTestReport) { + main = 'com.codacy.CodacyCoverageReporter' + classpath = configurations.codacy + args = ['report', '-l', 'Java', '-r', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml"] +} + diff --git a/gradle/publishing.gradle b/gradle/publishing.gradle new file mode 100644 index 00000000..bab00a0f --- /dev/null +++ b/gradle/publishing.gradle @@ -0,0 +1,124 @@ +apply plugin: 'maven-publish' +apply plugin: 'com.jfrog.bintray' +apply plugin: 'com.jfrog.artifactory' + +def sharedManifest = manifest { + attributes("Implementation-Title": project.name, + "Implementation-Version": version) +} + +jar { + archiveBaseName.set(project.name) + manifest.from sharedManifest +} + +// custom tasks for creating source/javadoc jars +task sourcesJar(type: Jar, dependsOn: classes) { + archiveBaseName.set(project.name) + archiveClassifier.set('sources') + from sourceSets.main.allSource + manifest.from sharedManifest +} + +task javadocJar(type: Jar, dependsOn: javadoc) { + archiveBaseName.set(project.name) + archiveClassifier.set('javadoc') + from javadoc.destinationDir + manifest.from sharedManifest +} + +// add javadoc/source jar tasks as artifacts +artifacts { + archives sourcesJar, javadocJar +} + +publishing { + publications { + mavenJar(MavenPublication) { + from components.java + artifact sourcesJar + artifact javadocJar + + pom { + name = project.name + description = project.description + url = githubUrl + licenses { + license { + name = 'The Apache Software License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + distribution = 'repo' + } + } + developers { + developer { + id = 'blootsvoets' + name = 'Joris Borgdorff' + email = 'joris@thehyve.nl' + organization = 'The Hyve' + } + developer { + id = 'nivemaham' + name = 'Nivethika Mahasivam' + email = 'nivethika@thehyve.nl' + organization = 'The Hyve' + } + } + issueManagement { + system = 'GitHub' + url = githubUrl + '/issues' + } + organization { + name = 'RADAR-base' + url = website + } + scm { + connection = 'scm:git:' + githubUrl + url = githubUrl + } + } + } + } +} + +bintray { + user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') + key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') + override false + publications 'mavenJar' + pkg { + repo = project.group + name = project.name + userOrg = 'radar-cns' + desc = project.description + licenses = ['Apache-2.0'] + websiteUrl = website + issueTrackerUrl = issueUrl + vcsUrl = githubUrl + githubRepo = githubRepoName + githubReleaseNotesFile = 'README.md' + version { + name = project.version + desc = project.description + vcsTag = System.getenv('TRAVIS_TAG') + released = new Date() + } + } +} + + +artifactory { + contextUrl = 'https://oss.jfrog.org/artifactory' + publish { + repository { + repoKey = 'oss-snapshot-local' + username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') + password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') + maven = true + } + } +} + +artifactoryPublish { + publications('mavenJar') +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 29953ea1..87b738cb 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e0b3fb8d..44e7c4d1 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index cccdd3d5..af6708ff 100755 --- a/gradlew +++ b/gradlew @@ -28,7 +28,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" diff --git a/gradlew.bat b/gradlew.bat index e95643d6..0f8d5937 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome diff --git a/radar-commons-server/build.gradle b/radar-commons-server/build.gradle index 85402bf2..6f1843ff 100644 --- a/radar-commons-server/build.gradle +++ b/radar-commons-server/build.gradle @@ -1,3 +1,5 @@ +import com.commercehub.gradle.plugin.avro.GenerateAvroJavaTask + /* * Copyright 2017 The Hyve and King's College London * @@ -14,11 +16,12 @@ * limitations under the License. */ -ext.moduleName = 'radar-commons-server' -ext.description = 'RADAR Common server library utilities.' +apply plugin: 'com.commercehub.gradle.plugin.avro' + +description = 'RADAR Common server library utilities.' -targetCompatibility = '1.8' -sourceCompatibility = '1.8' +targetCompatibility = '11.0' +sourceCompatibility = '11.0' repositories { maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } @@ -38,116 +41,20 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion // Direct producer uses KafkaAvroSerializer if initialized - testImplementation (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) { - exclude group: 'com.101tec' - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - } + testImplementation group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion testImplementation group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion // Direct producer uses KafkaAvroSerializer if initialized testImplementation group: 'junit', name: 'junit', version: junitVersion testRuntime group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion } -checkstyle { - // ignore tests - sourceSets = [sourceSets.main] - ignoreFailures = false -} - -pmd { - // ignore tests - sourceSets = [sourceSets.main] - ignoreFailures = false -} - -ext.sharedManifest = manifest { - attributes("Implementation-Title": moduleName, - "Implementation-Version": version) -} - -jar { - baseName moduleName - manifest.from sharedManifest -} - -// custom tasks for creating source/javadoc jars -task sourcesJar(type: Jar, dependsOn: classes) { - baseName moduleName - classifier = 'sources' - from sourceSets.main.allSource - manifest.from sharedManifest -} - -task javadocJar(type: Jar, dependsOn: javadoc) { - baseName moduleName - classifier = 'javadoc' - from javadoc.destinationDir - manifest.from sharedManifest -} - -// add javadoc/source jar tasks as artifacts -artifacts { - archives sourcesJar, javadocJar -} - -publishing { - publications { - RadarCommonsServerPublication(MavenPublication) { - from components.java - artifact sourcesJar - artifact javadocJar - groupId project.group - artifactId moduleName - version project.version - pom.withXml { - def root = asNode() - root.appendNode('description', description) - root.appendNode('name', moduleName) - root.appendNode('url', githubUrl) - root.children().last() + pomConfig - } - } - } -} - -bintray { - user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - override false - publications 'RadarCommonsServerPublication' - pkg { - repo = project.group - name = moduleName - userOrg = 'radar-cns' - desc = description - licenses = ['Apache-2.0'] - websiteUrl = website - issueTrackerUrl = issueUrl - vcsUrl = githubUrl - githubRepo = githubRepoName - githubReleaseNotesFile = 'README.md' - version { - name = project.version - desc = description - vcsTag = System.getenv('TRAVIS_TAG') - released = new Date() - } - } -} - +apply from: '../gradle/publishing.gradle' +apply from: '../gradle/codestyle.gradle' -artifactory { - contextUrl = 'https://oss.jfrog.org/artifactory' - publish { - repository { - repoKey = 'oss-snapshot-local' - username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - maven = true - } - } +tasks.withType(Checkstyle) { + exclude '**/org/radarbase/stream/collector/*State.*' } -artifactoryPublish { - publications('RadarCommonsServerPublication') +tasks.withType(Pmd) { + exclude '**/org/radarbase/stream/collector/*State.*' } diff --git a/radar-commons-server/src/main/avro/kafka/aggregate_list_state.avsc b/radar-commons-server/src/main/avro/kafka/aggregate_list_state.avsc new file mode 100644 index 00000000..9e623b94 --- /dev/null +++ b/radar-commons-server/src/main/avro/kafka/aggregate_list_state.avsc @@ -0,0 +1,9 @@ +{ + "namespace": "org.radarbase.stream.collector", + "name": "AggregateListState", + "type": "record", + "doc": "State of the AggregateListCollector", + "fields": [ + { "name": "aggregates", "type": {"type": "array", "items": ["NumericAggregateState"]}, "doc": "Collectors part of this aggregate."} + ] +} diff --git a/radar-commons-server/src/main/avro/kafka/big_decimal_state.avsc b/radar-commons-server/src/main/avro/kafka/big_decimal_state.avsc new file mode 100644 index 00000000..c85f22c5 --- /dev/null +++ b/radar-commons-server/src/main/avro/kafka/big_decimal_state.avsc @@ -0,0 +1,9 @@ +{ + "namespace": "org.radarbase.stream.collector", + "name": "BigDecimalState", + "type": "record", + "fields": [ + {"name": "intVal", "type": "bytes", "doc": "java BigInteger integer value."}, + {"name": "scale", "type": "int", "doc": "java BigDecimal scale."} + ] +} diff --git a/radar-commons-server/src/main/avro/kafka/numeric_aggregate_state.avsc b/radar-commons-server/src/main/avro/kafka/numeric_aggregate_state.avsc new file mode 100644 index 00000000..85f64a05 --- /dev/null +++ b/radar-commons-server/src/main/avro/kafka/numeric_aggregate_state.avsc @@ -0,0 +1,16 @@ +{ + "namespace": "org.radarbase.stream.collector", + "name": "NumericAggregateState", + "type": "record", + "doc": "State of the NumericAggregateCollector", + "fields": [ + { "name": "name", "type": ["null", "string"], "doc": "Name of field or null if not belonging to a specific field.", "default": null }, + { "name": "pos", "type": ["null", "int"], "doc": "Position of field or null if not belonging to a specific field.", "default": null }, + { "name": "fieldType", "type": ["null", "string"], "doc": "Type of field or null if not belonging to a specific field.", "default": null }, + { "name": "count", "type": "long", "doc": "Number of values encountered." }, + { "name": "min", "type": ["null", "double"], "doc": "Minimum value encountered or null if no value has been encountered.", "default": null }, + { "name": "max", "type": ["null", "double"], "doc": "Maximum value encountered or null if no value has been encountered.", "default": null }, + { "name": "sum", "type": ["null", "BigDecimalState"], "doc": "Sum encoded as Java BigDecimal bytes or null if no value has been encountered.", "default": null }, + { "name": "reservoir", "type": ["null", "SamplingReservoirState"], "doc": "Uniform sampling reservoir of values encountered. Null if reservoir sampling is disabled or no value has been encountered.", "default": null} + ] +} diff --git a/radar-commons-server/src/main/avro/kafka/sampling_reservoir_state.avsc b/radar-commons-server/src/main/avro/kafka/sampling_reservoir_state.avsc new file mode 100644 index 00000000..1f51e16b --- /dev/null +++ b/radar-commons-server/src/main/avro/kafka/sampling_reservoir_state.avsc @@ -0,0 +1,10 @@ +{ + "namespace": "org.radarbase.stream.collector", + "name": "SamplingReservoirState", + "type": "record", + "fields": [ + {"name": "samples", "type": {"type": "array", "items": "double"}, "doc": "java BigInteger integer value." }, + {"name": "count", "type": "long", "doc": "Number of values processed for samples." }, + {"name": "maxSize", "type": "int", "doc": "Maximum size of the reservoir." } + ] +} diff --git a/radar-commons-server/src/main/java/org/radarcns/config/AvroTopicConfig.java b/radar-commons-server/src/main/java/org/radarbase/config/AvroTopicConfig.java similarity index 97% rename from radar-commons-server/src/main/java/org/radarcns/config/AvroTopicConfig.java rename to radar-commons-server/src/main/java/org/radarbase/config/AvroTopicConfig.java index b0f6a7b8..6e916f6d 100644 --- a/radar-commons-server/src/main/java/org/radarcns/config/AvroTopicConfig.java +++ b/radar-commons-server/src/main/java/org/radarbase/config/AvroTopicConfig.java @@ -14,12 +14,12 @@ * limitations under the License. */ -package org.radarcns.config; +package org.radarbase.config; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; import org.apache.avro.specific.SpecificRecord; -import org.radarcns.topic.AvroTopic; +import org.radarbase.topic.AvroTopic; /** * Specifies an Avro topic. diff --git a/radar-commons-server/src/main/java/org/radarcns/config/ServerConfigMixin.java b/radar-commons-server/src/main/java/org/radarbase/config/ServerConfigMixin.java similarity index 82% rename from radar-commons-server/src/main/java/org/radarcns/config/ServerConfigMixin.java rename to radar-commons-server/src/main/java/org/radarbase/config/ServerConfigMixin.java index baf80340..fff80291 100644 --- a/radar-commons-server/src/main/java/org/radarcns/config/ServerConfigMixin.java +++ b/radar-commons-server/src/main/java/org/radarbase/config/ServerConfigMixin.java @@ -1,14 +1,10 @@ -package org.radarcns.config; +package org.radarbase.config; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSetter; /** Mixin class to load ServerConfig files with. */ public abstract class ServerConfigMixin { - public ServerConfigMixin() { - // POJO initializer - } - /** Proxy host name. Null if not set. */ @JsonProperty("proxy_host") public abstract String getProxyHost(); diff --git a/radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java b/radar-commons-server/src/main/java/org/radarbase/config/YamlConfigLoader.java similarity index 62% rename from radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java rename to radar-commons-server/src/main/java/org/radarbase/config/YamlConfigLoader.java index 349dc0bc..1b8ae639 100644 --- a/radar-commons-server/src/main/java/org/radarcns/config/YamlConfigLoader.java +++ b/radar-commons-server/src/main/java/org/radarbase/config/YamlConfigLoader.java @@ -14,38 +14,39 @@ * limitations under the License. */ -package org.radarcns.config; +package org.radarbase.config; import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.io.File; import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; /** * A YAML Config file loader, to load YAML files into equivalent POJO Objects. */ public class YamlConfigLoader { + public static final JsonFactory YAML_FACTORY = new YAMLFactory(); + private static final ObjectMapper YAML_MAPPER = new ObjectMapper(YAML_FACTORY); - private final ObjectMapper mapper; - - /** - * Default loader. - */ - public YamlConfigLoader() { - mapper = new ObjectMapper(new YAMLFactory()); - mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); + static { + YAML_MAPPER.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); // only serialize fields, not getters, etc. - mapper.setVisibility(mapper.getSerializationConfig().getDefaultVisibilityChecker() + YAML_MAPPER.setVisibility(YAML_MAPPER.getSerializationConfig().getDefaultVisibilityChecker() .withFieldVisibility(JsonAutoDetect.Visibility.ANY) .withGetterVisibility(JsonAutoDetect.Visibility.NONE) .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE) .withSetterVisibility(JsonAutoDetect.Visibility.NONE) .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)); - mapper.addMixIn(ServerConfig.class, ServerConfigMixin.class); + YAML_MAPPER.addMixIn(ServerConfig.class, ServerConfigMixin.class); } /** @@ -56,31 +57,33 @@ public YamlConfigLoader() { * @return loaded file. * @throws IOException if the file cannot be opened or parsed. */ - public T load(File file, Class configClass) throws IOException { - return mapper.readValue(file, configClass); + public T load(Path file, Class configClass) throws IOException { + try (Reader reader = Files.newBufferedReader(file, StandardCharsets.UTF_8)) { + return YAML_MAPPER.readValue(reader, configClass); + } } /** Store config into given YAML file. */ - public void store(File file, Object config) throws IOException { - mapper.writeValue(file, config); + public void store(Path file, Object config) throws IOException { + try (Writer writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8)) { + YAML_MAPPER.writeValue(writer, config); + } } /** * Pretty-print the given object as a YAML string. */ public String prettyString(Object config) { + ObjectMapper prettyPrintMapper = YAML_MAPPER.copy(); // pretty print - mapper.enable(SerializationFeature.INDENT_OUTPUT); + prettyPrintMapper.enable(SerializationFeature.INDENT_OUTPUT); // make ConfigRadar the root element - mapper.enable(SerializationFeature.WRAP_ROOT_VALUE); + prettyPrintMapper.enable(SerializationFeature.WRAP_ROOT_VALUE); try { - return mapper.writeValueAsString(config); + return prettyPrintMapper.writeValueAsString(config); } catch (JsonProcessingException ex) { throw new UnsupportedOperationException("Cannot serialize config", ex); - } finally { - mapper.disable(SerializationFeature.INDENT_OUTPUT); - mapper.disable(SerializationFeature.WRAP_ROOT_VALUE); } } } diff --git a/radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java b/radar-commons-server/src/main/java/org/radarbase/producer/direct/DirectSender.java similarity index 91% rename from radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java rename to radar-commons-server/src/main/java/org/radarbase/producer/direct/DirectSender.java index ca1b6ad7..4617c574 100644 --- a/radar-commons-server/src/main/java/org/radarcns/producer/direct/DirectSender.java +++ b/radar-commons-server/src/main/java/org/radarbase/producer/direct/DirectSender.java @@ -14,15 +14,15 @@ * limitations under the License. */ -package org.radarcns.producer.direct; +package org.radarbase.producer.direct; import java.util.Properties; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; -import org.radarcns.data.RecordData; -import org.radarcns.producer.KafkaSender; -import org.radarcns.producer.KafkaTopicSender; -import org.radarcns.topic.AvroTopic; +import org.radarbase.data.RecordData; +import org.radarbase.producer.KafkaSender; +import org.radarbase.producer.KafkaTopicSender; +import org.radarbase.topic.AvroTopic; /** * Directly sends a message to Kafka using a KafkaProducer. diff --git a/radar-commons-server/src/main/java/org/radarbase/stream/collector/.gitignore b/radar-commons-server/src/main/java/org/radarbase/stream/collector/.gitignore new file mode 100644 index 00000000..a731935a --- /dev/null +++ b/radar-commons-server/src/main/java/org/radarbase/stream/collector/.gitignore @@ -0,0 +1,4 @@ +AggregateListState.java +BigDecimalState.java +NumericAggregateState.java +SamplingReservoirState.java diff --git a/radar-commons-server/src/main/java/org/radarcns/stream/collector/AggregateListCollector.java b/radar-commons-server/src/main/java/org/radarbase/stream/collector/AggregateListCollector.java similarity index 55% rename from radar-commons-server/src/main/java/org/radarcns/stream/collector/AggregateListCollector.java rename to radar-commons-server/src/main/java/org/radarbase/stream/collector/AggregateListCollector.java index db22ce80..c4576223 100644 --- a/radar-commons-server/src/main/java/org/radarcns/stream/collector/AggregateListCollector.java +++ b/radar-commons-server/src/main/java/org/radarbase/stream/collector/AggregateListCollector.java @@ -14,41 +14,40 @@ * limitations under the License. */ -package org.radarcns.stream.collector; +package org.radarbase.stream.collector; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificRecord; +import org.radarbase.util.SpecificAvroConvertible; /** * Java class to aggregate multiple fields of data using Kafka Streams. It can be used for example * on acceleration or gyroscope observations that have multiple axes. */ -public class AggregateListCollector implements RecordCollector { - private final NumericAggregateCollector[] collectors; +public class AggregateListCollector implements RecordCollector, SpecificAvroConvertible { + private NumericAggregateCollector[] collectors; - @JsonCreator - public AggregateListCollector( - @JsonProperty("collectors") NumericAggregateCollector[] collectors) { - this.collectors = collectors; + public AggregateListCollector() { + this.collectors = null; } /** Array collector without schema. Double entries can be added, but entire records cannot. */ - public AggregateListCollector(String[] fieldNames) { - this(fieldNames, null); + public AggregateListCollector(String[] fieldNames, boolean useReservoir) { + this(fieldNames, null, useReservoir); } /** * Aggregate list collector with single record schema. Double entries or records can be added. * This assumes that all fields in the aggregate list are extracted from a single record schema. */ - public AggregateListCollector(String[] fieldNames, Schema schema) { + public AggregateListCollector(String[] fieldNames, Schema schema, boolean useReservoir) { collectors = new NumericAggregateCollector[fieldNames.length]; for (int i = 0; i < collectors.length; i++) { - collectors[i] = new NumericAggregateCollector(fieldNames[i], schema); + collectors[i] = new NumericAggregateCollector(fieldNames[i], schema, useReservoir); } } @@ -64,7 +63,7 @@ public AggregateListCollector add(SpecificRecord record) { * Add a sample to the collection. * @param value new sample that has to be analysed */ - public AggregateListCollector add(double[] value) { + public AggregateListCollector add(double... value) { if (collectors.length != value.length) { throw new IllegalArgumentException( "The length of current input differs from the length of the value used to " @@ -85,4 +84,41 @@ public String toString() { public List getCollectors() { return Arrays.asList(collectors); } + + @Override + public AggregateListState toAvro() { + AggregateListState state = new AggregateListState(); + if (collectors == null) { + state.setAggregates(Collections.emptyList()); + } else { + List aggregates = new ArrayList<>(collectors.length); + for (NumericAggregateCollector collector : collectors) { + aggregates.add(collector.toAvro()); + } + state.setAggregates(aggregates); + } + return state; + } + + @Override + public void fromAvro(SpecificRecord record) { + if (!(record instanceof AggregateListState)) { + throw new IllegalArgumentException("Cannot convert incompatible Avro record"); + } + + AggregateListState aggregateList = (AggregateListState) record; + List aggregates = aggregateList.getAggregates(); + int len = aggregates.size(); + + collectors = new NumericAggregateCollector[len]; + for (int i = 0; i < len; i++) { + Object aggregate = aggregates.get(i); + if (aggregate instanceof NumericAggregateState) { + collectors[i] = new NumericAggregateCollector(); + } else { + throw new IllegalArgumentException("Cannot convert type " + record.getClass()); + } + collectors[i].fromAvro((SpecificRecord) aggregate); + } + } } diff --git a/radar-commons-server/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java b/radar-commons-server/src/main/java/org/radarbase/stream/collector/NumericAggregateCollector.java similarity index 61% rename from radar-commons-server/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java rename to radar-commons-server/src/main/java/org/radarbase/stream/collector/NumericAggregateCollector.java index 741bc1b4..52ab548b 100644 --- a/radar-commons-server/src/main/java/org/radarcns/stream/collector/NumericAggregateCollector.java +++ b/radar-commons-server/src/main/java/org/radarbase/stream/collector/NumericAggregateCollector.java @@ -14,59 +14,52 @@ * limitations under the License. */ -package org.radarcns.stream.collector; +package org.radarbase.stream.collector; -import static org.radarcns.util.Serialization.floatToDouble; +import static org.radarbase.util.Serialization.floatToDouble; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; import java.util.List; import java.util.Objects; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; import org.apache.avro.Schema.Type; import org.apache.avro.specific.SpecificRecord; +import org.radarbase.util.SpecificAvroConvertible; /** * Java class to aggregate data using Kafka Streams. Double is the base type. * Only the sum and sorted history are collected, other getSamples are calculated on request. */ -@JsonDeserialize(builder = NumericAggregateCollector.Builder.class) -public class NumericAggregateCollector implements RecordCollector { - private final String name; - private final int pos; - private final Type fieldType; +public class NumericAggregateCollector implements RecordCollector, SpecificAvroConvertible { + private String name; + private int pos; + private Type fieldType; private double min; private double max; + private long count; private BigDecimal sum; - private final UniformSamplingReservoir reservoir; - - /** Aggregate collector from builder. */ - private NumericAggregateCollector(Builder builder) { - this.name = builder.nameValue; - this.pos = builder.posValue; - this.fieldType = builder.fieldTypeValue; - this.min = builder.minValue; - this.max = builder.maxValue; - this.sum = builder.sumValue; - this.reservoir = builder.reservoirValue; + private UniformSamplingReservoir reservoir; + + /** Aggregate collector with only a field name. */ + public NumericAggregateCollector() { + this(null, null, false); } /** Aggregate collector with only a field name. */ - public NumericAggregateCollector(String fieldName) { - this(fieldName, null); + public NumericAggregateCollector(String fieldName, boolean useReservoir) { + this(fieldName, null, useReservoir); } /** Aggregate collector with a field name and accompanying schema containing the name. */ - public NumericAggregateCollector(String fieldName, Schema schema) { + public NumericAggregateCollector(String fieldName, Schema schema, boolean useReservoir) { sum = BigDecimal.ZERO; min = Double.POSITIVE_INFINITY; max = Double.NEGATIVE_INFINITY; - reservoir = new UniformSamplingReservoir(); + count = 0; + reservoir = useReservoir ? new UniformSamplingReservoir() : null; name = fieldName; if (schema == null) { @@ -139,13 +132,16 @@ public NumericAggregateCollector add(float value) { */ public NumericAggregateCollector add(double value) { sum = sum.add(BigDecimal.valueOf(value)); - reservoir.add(value); + if (reservoir != null) { + reservoir.add(value); + } if (value > max) { max = value; } if (value < min) { min = value; } + count++; return this; } @@ -159,6 +155,7 @@ public String toString() { + ", sum=" + getSum() + ", mean=" + getMean() + ", quartile=" + getQuartile() + + ", count=" + getCount() + ", reservoir=" + reservoir + '}'; } @@ -174,19 +171,36 @@ public double getSum() { return sum.doubleValue(); } - public int getCount() { - return reservoir.getCount(); + public long getCount() { + return count; } public double getMean() { return sum.doubleValue() / getCount(); } + /** Has a sampling reservoir. */ + public boolean hasReservoir() { + return reservoir != null; + } + + /** + * Get the quartiles as estimated from a uniform sampling reservoir. + * @throws IllegalStateException if the collector does not keep a sampling reservoir, as + * indicated by {@link #hasReservoir()}. + */ public List getQuartile() { + if (!hasReservoir()) { + throw new IllegalStateException("Cannot query quartiles without reservoir"); + } return reservoir.getQuartiles(); } - /** Difference between the first quartile and third quartile (IQR). */ + /** + * Difference between the first quartile and third quartile (IQR). + * @throws IllegalStateException if the collector does not keep a sampling reservoir, as + * indicated by {@link #hasReservoir()}. + */ public double getInterQuartileRange() { List quartiles = getQuartile(); return BigDecimal.valueOf(quartiles.get(2)) @@ -211,6 +225,7 @@ public boolean equals(Object o) { } NumericAggregateCollector that = (NumericAggregateCollector) o; return pos == that.pos + && count == that.count && Double.compare(that.min, min) == 0 && Double.compare(that.max, max) == 0 && Objects.equals(name, that.name) @@ -224,81 +239,67 @@ public int hashCode() { return Objects.hash(name, pos, fieldType, min, max, sum, reservoir); } - @JsonPOJOBuilder(withPrefix = "") - public static class Builder { - private double maxValue = Double.NEGATIVE_INFINITY; - private double minValue = Double.POSITIVE_INFINITY; - private final String nameValue; - private int posValue = -1; - private Type fieldTypeValue = null; - private BigDecimal sumValue = BigDecimal.ZERO; - private UniformSamplingReservoir reservoirValue = new UniformSamplingReservoir(); - - @JsonCreator - public Builder(@JsonProperty("name") String name) { - this.nameValue = Objects.requireNonNull(name); - } - - @JsonSetter - public Builder pos(int pos) { - posValue = pos; - return this; - } - - @JsonSetter - public Builder fieldType(Type fieldType) { - fieldTypeValue = fieldType; - return this; - } - - /** Set minimum value. */ - @JsonSetter - public Builder min(double min) { - if (min < minValue) { - minValue = min; - } - return this; + @Override + public SpecificRecord toAvro() { + NumericAggregateState state = new NumericAggregateState(); + state.setCount(count); + if (count > 0) { + state.setMin(min); + state.setMax(max); + state.setSum(new BigDecimalState( + ByteBuffer.wrap(sum.unscaledValue().toByteArray()), + sum.scale())); + } else { + state.setMin(null); + state.setMax(null); + state.setSum(null); } - - /** Set maximum value. */ - @JsonSetter - public Builder max(double max) { - if (max > maxValue) { - maxValue = max; - } - return this; + if (pos != -1) { + state.setPos(pos); + state.setFieldType(fieldType.name()); + } else { + state.setPos(null); + state.setFieldType(null); } + state.setName(name); + state.setReservoir(reservoir != null ? reservoir.toAvro() : null); + return state; + } - @JsonSetter - public Builder sum(BigDecimal sum) { - sumValue = sum; - return this; + @Override + public void fromAvro(SpecificRecord record) { + if (!(record instanceof NumericAggregateState)) { + throw new IllegalArgumentException("Cannot deserialize from non NumericAggregateState"); } + NumericAggregateState state = (NumericAggregateState) record; - @JsonSetter - public Builder reservoir(UniformSamplingReservoir reservoir) { - this.reservoirValue = reservoir; - return this; + name = state.getName(); + if (state.getPos() != null) { + pos = state.getPos(); + fieldType = Schema.Type.valueOf(state.getFieldType()); + } else { + pos = -1; + fieldType = null; } - - /** - * For backwards compatibility purposes, convert a full history to a reservoir. - * @param history stored history. - * @return the current builder. - * @deprecated use reservoir instead. - */ - @Deprecated - @JsonSetter - public Builder history(double[] history) { - min(history[0]); - max(history[history.length - 1]); - reservoir(new UniformSamplingReservoir(history)); - return this; + count = state.getCount(); + + if (count > 0) { + min = state.getMin(); + max = state.getMax(); + sum = new BigDecimal( + new BigInteger(state.getSum().getIntVal().array()), + state.getSum().getScale()); + } else { + min = Double.MAX_VALUE; + max = Double.MIN_VALUE; + sum = BigDecimal.ZERO; } - /** Build the aggregator. */ - public NumericAggregateCollector build() { - return new NumericAggregateCollector(this); + if (state.getReservoir() == null) { + reservoir = null; + } else { + reservoir = new UniformSamplingReservoir(new double[0], 0, 1); + reservoir.fromAvro(state.getReservoir()); } } } diff --git a/radar-commons-server/src/main/java/org/radarcns/stream/collector/RecordCollector.java b/radar-commons-server/src/main/java/org/radarbase/stream/collector/RecordCollector.java similarity index 89% rename from radar-commons-server/src/main/java/org/radarcns/stream/collector/RecordCollector.java rename to radar-commons-server/src/main/java/org/radarbase/stream/collector/RecordCollector.java index d57ca934..885817d9 100644 --- a/radar-commons-server/src/main/java/org/radarcns/stream/collector/RecordCollector.java +++ b/radar-commons-server/src/main/java/org/radarbase/stream/collector/RecordCollector.java @@ -1,4 +1,4 @@ -package org.radarcns.stream.collector; +package org.radarbase.stream.collector; import org.apache.avro.specific.SpecificRecord; diff --git a/radar-commons-server/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java b/radar-commons-server/src/main/java/org/radarbase/stream/collector/UniformSamplingReservoir.java similarity index 75% rename from radar-commons-server/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java rename to radar-commons-server/src/main/java/org/radarbase/stream/collector/UniformSamplingReservoir.java index 257cde14..2e3623e1 100644 --- a/radar-commons-server/src/main/java/org/radarcns/stream/collector/UniformSamplingReservoir.java +++ b/radar-commons-server/src/main/java/org/radarbase/stream/collector/UniformSamplingReservoir.java @@ -1,8 +1,6 @@ -package org.radarcns.stream.collector; +package org.radarbase.stream.collector; -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonGetter; -import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -11,6 +9,8 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; +import org.apache.avro.specific.SpecificRecord; +import org.radarbase.util.SpecificAvroConvertible; /** * Uniform sampling reservoir for streaming. This should capture the input distribution in order @@ -20,10 +20,10 @@ * As long as the number of samples is lower than the maximum size of the reservoir, the quartiles * are computed exactly. */ -public class UniformSamplingReservoir { - private final double[] samples; - private final int maxSize; - private int count; +public class UniformSamplingReservoir implements SpecificAvroConvertible { + private double[] samples; + private int maxSize; + private long count; private transient int currentLength; private static final int MAX_SIZE_DEFAULT = 999; @@ -37,7 +37,7 @@ public UniformSamplingReservoir() { * @param allValues list of values to sample from. * @throws NullPointerException if given allValues are {@code null}. */ - public UniformSamplingReservoir(double[] allValues) { + public UniformSamplingReservoir(double... allValues) { this(allValues, allValues.length, MAX_SIZE_DEFAULT); } @@ -48,35 +48,38 @@ public UniformSamplingReservoir(double[] allValues) { * @param maxSize maximum reservoir size. * @throws NullPointerException if given allValues are {@code null} */ - @JsonCreator - public UniformSamplingReservoir( - @JsonProperty("samples") double[] samples, - @JsonProperty("count") int count, - @JsonProperty("maxSize") int maxSize) { - if (samples == null) { + public UniformSamplingReservoir(double[] samples, long count, int maxSize) { + initializeReservoir(samples, count, maxSize); + } + + /** Sample from given list of samples to initialize the reservoir. */ + private void initializeReservoir(double[] initSamples, long initCount, int initMaxSize) { + if (initSamples == null) { throw new IllegalArgumentException("Samples may not be null"); } - if (maxSize <= 0) { + if (initMaxSize <= 0) { throw new IllegalArgumentException("Reservoir maximum size must be strictly positive"); } - if (count < 0) { - throw new IllegalArgumentException("Reservoir size must be positive"); + if (initCount < initSamples.length) { + throw new IllegalArgumentException( + "Reservoir count must be larger or equal than number of samples."); } - this.samples = new double[maxSize]; - this.maxSize = maxSize; - this.count = count; - initializeReservoir(samples); - } + this.maxSize = initMaxSize; + this.samples = new double[initMaxSize]; + this.count = initCount; - /** Sample from given list of samples to initialize the reservoir. */ - private void initializeReservoir(double[] initSamples) { - int length = Math.min(initSamples.length, count); + int length = (int)Math.min(initSamples.length, count); if (length == 0) { currentLength = 0; return; } + subsample(initSamples, length); + Arrays.sort(this.samples, 0, currentLength); + } + + private void subsample(double[] initSamples, int length) { ThreadLocalRandom random = ThreadLocalRandom.current(); // There are much more samples than the size permits. Random sample from the @@ -113,15 +116,14 @@ private void initializeReservoir(double[] initSamples) { System.arraycopy(initSamples, 0, this.samples, 0, length); currentLength = length; } - Arrays.sort(this.samples, 0, currentLength); } /** Add a sample to the reservoir. */ public void add(double value) { if (currentLength == maxSize) { - int removeIndex = ThreadLocalRandom.current().nextInt(count); + long removeIndex = ThreadLocalRandom.current().nextLong(count); if (removeIndex < maxSize) { - removeAndAdd(removeIndex, value); + removeAndAdd((int)removeIndex, value); } } else { removeAndAdd(currentLength, value); @@ -214,7 +216,7 @@ public int getMaxSize() { } /** Get the number of samples that are being represented by the reservoir. */ - public int getCount() { + public long getCount() { return count; } @@ -245,4 +247,38 @@ public String toString() { + ", count=" + count + '}'; } + + @Override + public SamplingReservoirState toAvro() { + SamplingReservoirState state = new SamplingReservoirState(); + state.setCount(count); + state.setMaxSize(maxSize); + + int length = (int) Math.min(maxSize, count); + List sampleList = new ArrayList<>(length); + for (int i = 0; i < length; i++) { + sampleList.add(samples[i]); + } + state.setSamples(sampleList); + + return state; + } + + @Override + public void fromAvro(SpecificRecord record) { + if (!(record instanceof SamplingReservoirState)) { + throw new IllegalArgumentException("Cannot initialize from non-samplingreservoirstate"); + } + SamplingReservoirState state = (SamplingReservoirState) record; + List stateSamples = state.getSamples(); + + if (stateSamples == null) { + throw new IllegalArgumentException("Samples may not be null"); + } + double[] newSamples = new double[stateSamples.size()]; + for (int i = 0; i < newSamples.length; i++) { + newSamples[i] = stateSamples.get(i); + } + initializeReservoir(newSamples, state.getCount(), state.getMaxSize()); + } } diff --git a/radar-commons-server/src/main/java/org/radarcns/util/RollingTimeAverage.java b/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.java similarity index 99% rename from radar-commons-server/src/main/java/org/radarcns/util/RollingTimeAverage.java rename to radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.java index 26d2b1e3..a4515c96 100644 --- a/radar-commons-server/src/main/java/org/radarcns/util/RollingTimeAverage.java +++ b/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.util; +package org.radarbase.util; import java.util.Deque; import java.util.LinkedList; diff --git a/radar-commons-server/src/main/java/org/radarbase/util/SpecificAvroConvertible.java b/radar-commons-server/src/main/java/org/radarbase/util/SpecificAvroConvertible.java new file mode 100644 index 00000000..5dada5b1 --- /dev/null +++ b/radar-commons-server/src/main/java/org/radarbase/util/SpecificAvroConvertible.java @@ -0,0 +1,15 @@ +package org.radarbase.util; + +import org.apache.avro.specific.SpecificRecord; + +/** + * Convertible from Avro. This assumes that the entire state of an object can be replaced + * with the data in the record, using e.g. an empty constructor. + */ +public interface SpecificAvroConvertible { + /** Converts the convertible to a specificrecord. */ + SpecificRecord toAvro(); + + /** Reads the convertible from a specificrecord. */ + void fromAvro(SpecificRecord record); +} diff --git a/radar-commons-server/src/main/java/org/radarcns/util/serde/AbstractKafkaAvroSerde.java b/radar-commons-server/src/main/java/org/radarcns/util/serde/AbstractKafkaAvroSerde.java deleted file mode 100644 index dfff31cc..00000000 --- a/radar-commons-server/src/main/java/org/radarcns/util/serde/AbstractKafkaAvroSerde.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import java.util.Map; -import org.apache.avro.io.EncoderFactory; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.serialization.Serializer; -import org.radarcns.config.ServerConfig; -import org.radarcns.producer.rest.SchemaRetriever; - -/** - * Abstract class for KafkaAvro(De)serializer. - */ -public abstract class AbstractKafkaAvroSerde implements Serializer { - public static final String SCHEMA_REGISTRY_CONFIG = "schema.registry"; - - protected boolean ofValue; - protected final EncoderFactory encoderFactory = EncoderFactory.get(); - protected SchemaRetriever schemaRetriever; - - public AbstractKafkaAvroSerde() { - // Bean constructor - } - - public AbstractKafkaAvroSerde(SchemaRetriever retriever) { - this.schemaRetriever = retriever; - } - - @Override - public void configure(Map configs, boolean isKey) { - this.ofValue = !isKey; - if (schemaRetriever != null) { - return; - } - Object schemaConfig = configs.get(SCHEMA_REGISTRY_CONFIG); - if (schemaConfig instanceof SchemaRetriever) { - schemaRetriever = (SchemaRetriever) schemaConfig; - } else if (schemaConfig instanceof ServerConfig) { - schemaRetriever = new SchemaRetriever((ServerConfig) schemaConfig, 30); - } else { - throw new ConfigException("Config " + SCHEMA_REGISTRY_CONFIG - + " is not correctly configured. " - + "Pass a " + SchemaRetriever.class + " or a " + ServerConfig.class); - } - } - - @SuppressWarnings("PMD.EmptyMethodInAbstractClassShouldBeAbstract") - @Override - public void close() { - // noop - } -} diff --git a/radar-commons-server/src/main/java/org/radarcns/util/serde/KafkaAvroSerializer.java b/radar-commons-server/src/main/java/org/radarcns/util/serde/KafkaAvroSerializer.java deleted file mode 100644 index 6f110a78..00000000 --- a/radar-commons-server/src/main/java/org/radarcns/util/serde/KafkaAvroSerializer.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.avro.specific.SpecificRecord; -import org.apache.kafka.common.errors.SerializationException; -import org.radarcns.producer.rest.ParsedSchemaMetadata; -import org.radarcns.producer.rest.SchemaRetriever; -import org.radarcns.util.Serialization; - -/** - * Serialize Kafka producer data with Avro. - */ -public class KafkaAvroSerializer extends AbstractKafkaAvroSerde { - private final byte[] header = new byte[5]; - private BinaryEncoder encoder; - - public KafkaAvroSerializer() { - super(); - } - - public KafkaAvroSerializer(SchemaRetriever retriever) { - super(retriever); - } - - @Override - public byte[] serialize(String topic, Object data) { - if (data == null) { - return null; - } - Schema initialSchema = SchemaRetriever.getSchema(data); - - try { - ParsedSchemaMetadata schema = schemaRetriever - .getOrSetSchemaMetadata(topic, ofValue, initialSchema, -1); - - if (data instanceof byte[]) { - byte[] byteData = (byte[])data; - byte[] result = new byte[byteData.length + 5]; - storeHeader(result, schema); - System.arraycopy(byteData, 0, result, 5, byteData.length); - return result; - } else { - storeHeader(header, schema); - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - out.write(header); - - encoder = this.encoderFactory.directBinaryEncoder(out, encoder); - Object writer; - if (data instanceof SpecificRecord) { - writer = new SpecificDatumWriter(schema.getSchema()); - } else { - writer = new GenericDatumWriter(schema.getSchema()); - } - - //noinspection unchecked - ((DatumWriter) writer).write(data, encoder); - encoder.flush(); - return out.toByteArray(); - } - } - } catch (IOException ex) { - throw new SerializationException("Cannot match data with schema registry", ex); - } - } - - private static void storeHeader(byte[] buffer, ParsedSchemaMetadata schema) { - buffer[0] = 0; - Serialization.intToBytes(schema.getId(), buffer, 1); - } -} diff --git a/radar-commons-server/src/test/java/org/radarcns/config/ServerConfigTest.java b/radar-commons-server/src/test/java/org/radarbase/config/ServerConfigTest.java similarity index 99% rename from radar-commons-server/src/test/java/org/radarcns/config/ServerConfigTest.java rename to radar-commons-server/src/test/java/org/radarbase/config/ServerConfigTest.java index fc5f3c0f..11cc14bc 100644 --- a/radar-commons-server/src/test/java/org/radarcns/config/ServerConfigTest.java +++ b/radar-commons-server/src/test/java/org/radarbase/config/ServerConfigTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.config; +package org.radarbase.config; import static org.junit.Assert.assertEquals; diff --git a/radar-commons-server/src/test/java/org/radarcns/stream/collector/AggregateListCollectorTest.java b/radar-commons-server/src/test/java/org/radarbase/stream/collector/AggregateListCollectorTest.java similarity index 92% rename from radar-commons-server/src/test/java/org/radarcns/stream/collector/AggregateListCollectorTest.java rename to radar-commons-server/src/test/java/org/radarbase/stream/collector/AggregateListCollectorTest.java index 1f7199bc..3645a24d 100644 --- a/radar-commons-server/src/test/java/org/radarcns/stream/collector/AggregateListCollectorTest.java +++ b/radar-commons-server/src/test/java/org/radarbase/stream/collector/AggregateListCollectorTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.stream.collector; +package org.radarbase.stream.collector; import static org.junit.Assert.assertEquals; @@ -28,7 +28,7 @@ public class AggregateListCollectorTest { @Test public void add() { AggregateListCollector arrayCollector = new AggregateListCollector( - new String[]{"a", "b", "c", "d"}); + new String[]{"a", "b", "c", "d"}, false); double[] arrayvalues = {0.15d, 1.0d, 2.0d, 3.0d}; arrayCollector.add(arrayvalues); assertEquals(4, arrayCollector.getCollectors().size()); @@ -41,7 +41,7 @@ public void add() { @Test public void addRecord() { AggregateListCollector arrayCollector = new AggregateListCollector(new String[] {"x", "y", "z"}, - EmpaticaE4Acceleration.getClassSchema()); + EmpaticaE4Acceleration.getClassSchema(), false); arrayCollector.add(new EmpaticaE4Acceleration(0d, 0d, 0.15f, 1.0f, 2.0f)); assertEquals(3, arrayCollector.getCollectors().size()); diff --git a/radar-commons-server/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java b/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java similarity index 72% rename from radar-commons-server/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java rename to radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java index 2d104354..a7879c69 100644 --- a/radar-commons-server/src/test/java/org/radarcns/stream/collector/NumericAggregateCollectorTest.java +++ b/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java @@ -14,15 +14,14 @@ * limitations under the License. */ -package org.radarcns.stream.collector; +package org.radarbase.stream.collector; import static org.junit.Assert.assertEquals; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; -import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.util.Arrays; import org.junit.Before; import org.junit.Test; import org.radarcns.kafka.AggregateKey; @@ -39,8 +38,7 @@ public class NumericAggregateCollectorTest { @Before public void setUp() { - this.valueCollector = new NumericAggregateCollector.Builder("test") - .build(); + this.valueCollector = new NumericAggregateCollector("test", true); } @Test @@ -116,32 +114,32 @@ public void testAddRecordWithoutSchema() { @Test(expected = IllegalArgumentException.class) public void testWrongRecordType() { this.valueCollector = new NumericAggregateCollector("isPlugged", - PhoneBatteryLevel.getClassSchema()); + PhoneBatteryLevel.getClassSchema(), false); } @Test(expected = IllegalArgumentException.class) public void testWrongFieldName() { this.valueCollector = new NumericAggregateCollector("doesNotExist", - PhoneBatteryLevel.getClassSchema()); + PhoneBatteryLevel.getClassSchema(), false); } @Test public void testRecordType() { this.valueCollector = new NumericAggregateCollector("batteryLevel", - PhoneBatteryLevel.getClassSchema()); + PhoneBatteryLevel.getClassSchema(), false); this.valueCollector = new NumericAggregateCollector("time", - PhoneBatteryLevel.getClassSchema()); + PhoneBatteryLevel.getClassSchema(), false); this.valueCollector = new NumericAggregateCollector("recordsSent", - ApplicationRecordCounts.getClassSchema()); + ApplicationRecordCounts.getClassSchema(), false); this.valueCollector = new NumericAggregateCollector("recordsCached", - ApplicationRecordCounts.getClassSchema()); + ApplicationRecordCounts.getClassSchema(), false); this.valueCollector = new NumericAggregateCollector("timeStart", - AggregateKey.getClassSchema()); + AggregateKey.getClassSchema(), false); } @Test public void testAddRecord() { - this.valueCollector = new NumericAggregateCollector("bloodVolumePulse", EmpaticaE4BloodVolumePulse.getClassSchema()); + this.valueCollector = new NumericAggregateCollector("bloodVolumePulse", EmpaticaE4BloodVolumePulse.getClassSchema(), false); valueCollector.add(new EmpaticaE4BloodVolumePulse(0d, 0d, 0f)); assertEquals(1, valueCollector.getCount()); assertEquals(0d, valueCollector.getMean(), 0d); @@ -149,7 +147,7 @@ public void testAddRecord() { @Test public void testAddRecordWithNull() { - this.valueCollector = new NumericAggregateCollector("recordsCached", ApplicationRecordCounts.getClassSchema()); + this.valueCollector = new NumericAggregateCollector("recordsCached", ApplicationRecordCounts.getClassSchema(), false); valueCollector.add(new ApplicationRecordCounts(0d, 1, 0, 1)); assertEquals(1, valueCollector.getCount()); assertEquals(1d, valueCollector.getMean(), 1e-5d); @@ -163,36 +161,47 @@ public void testAddRecordWithNull() { @Test public void testSerialization() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - - mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); - mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); - - valueCollector = new NumericAggregateCollector.Builder("test") - .history(new double[] {-1d, 15d}) - .sum(BigDecimal.valueOf(14)) - .build(); - - String valueString = mapper.writeValueAsString(valueCollector); - System.out.println(valueString); - assertEquals(valueCollector, mapper.readValue(valueString, NumericAggregateCollector.class)); + valueCollector = new NumericAggregateCollector(); + NumericAggregateState state = new NumericAggregateState(); + state.setName("test"); + state.setCount(2L); + state.setMin(-1d); + state.setMax(15d); + state.setSum(new BigDecimalState(ByteBuffer.wrap(BigInteger.valueOf(14).toByteArray()), 0)); + state.setReservoir(new SamplingReservoirState(Arrays.asList(-1d, 15d), 2L, 999)); + valueCollector.fromAvro(state); + assertEquals(state, valueCollector.toAvro()); } @Test public void testReservoirBuilder() { - valueCollector = new NumericAggregateCollector.Builder("test") - .reservoir(new UniformSamplingReservoir(new double[] {-1d, 15d}, 2, 1)) - .build(); + valueCollector = new NumericAggregateCollector(); + NumericAggregateState state = new NumericAggregateState(); + state.setName("test"); + state.setCount(2L); + state.setMin(-1d); + state.setMax(15d); + state.setSum(new BigDecimalState(ByteBuffer.wrap(BigInteger.valueOf(14).toByteArray()), 0)); + state.setReservoir(new SamplingReservoirState(Arrays.asList(-1d, 15d), 2L, 1)); + valueCollector.fromAvro(state); assertEquals(2, valueCollector.getCount()); assertEquals(1, valueCollector.getReservoir().getSamples().size()); + assertEquals(2, valueCollector.getReservoir().getCount()); + assertEquals(1, valueCollector.getReservoir().getMaxSize()); } @Test public void testReservoirBuilderUnlimited() { - valueCollector = new NumericAggregateCollector.Builder("name") - .reservoir(new UniformSamplingReservoir(new double[] {-1d, 15d}, 2, 1000)) - .build(); + valueCollector = new NumericAggregateCollector(); + NumericAggregateState state = new NumericAggregateState(); + state.setName("test"); + state.setCount(2L); + state.setMin(-1d); + state.setMax(15d); + state.setSum(new BigDecimalState(ByteBuffer.wrap(BigInteger.valueOf(14).toByteArray()), 0)); + state.setReservoir(new SamplingReservoirState(Arrays.asList(-1d, 15d), 2L, 1000)); + valueCollector.fromAvro(state); assertEquals(2, valueCollector.getCount()); assertEquals(2, valueCollector.getReservoir().getSamples().size()); diff --git a/radar-commons-server/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java b/radar-commons-server/src/test/java/org/radarbase/stream/collector/UniformSamplingReservoirTest.java similarity index 98% rename from radar-commons-server/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java rename to radar-commons-server/src/test/java/org/radarbase/stream/collector/UniformSamplingReservoirTest.java index 733e0dbf..359289b8 100644 --- a/radar-commons-server/src/test/java/org/radarcns/stream/collector/UniformSamplingReservoirTest.java +++ b/radar-commons-server/src/test/java/org/radarbase/stream/collector/UniformSamplingReservoirTest.java @@ -1,4 +1,4 @@ -package org.radarcns.stream.collector; +package org.radarbase.stream.collector; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/radar-commons-server/src/test/java/org/radarcns/util/serde/KafkaAvroSerializerTest.java b/radar-commons-server/src/test/java/org/radarcns/util/serde/KafkaAvroSerializerTest.java deleted file mode 100644 index 4042038e..00000000 --- a/radar-commons-server/src/test/java/org/radarcns/util/serde/KafkaAvroSerializerTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import static org.junit.Assert.assertArrayEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; -import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.apache.avro.Schema.Type; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.junit.Test; -import org.radarcns.kafka.ObservationKey; -import org.radarcns.producer.rest.ParsedSchemaMetadata; -import org.radarcns.producer.rest.SchemaRetriever; - -public class KafkaAvroSerializerTest { - - @Test - public void serialize() throws Exception { - testSerialization("this", Schema.create(Type.STRING)); - testSerialization(10, Schema.create(Type.INT)); - - ObservationKey key = new ObservationKey("test", "a", "b"); - testSerialization(key, key.getSchema()); - - Schema genericSchema = Schema.createRecord(Arrays.asList( - new Field("a", Schema.create(Type.STRING), "aaa", "salt"), - new Field("b", Schema.create(Type.INT), "bb", (Object)null) - )); - GenericRecord record = new GenericData.Record(genericSchema); - record.put("a", "nodefault"); - record.put("b", 10); - testSerialization(record, genericSchema); - } - - private void testSerialization(Object data, Schema schema) - throws IOException, RestClientException { - SchemaRetriever retriever = mock(SchemaRetriever.class); - KafkaAvroSerializer serializer = new KafkaAvroSerializer(retriever); - serializer.configure(Collections.emptyMap(), true); - when(retriever.getOrSetSchemaMetadata("bla", false, schema, -1)).thenReturn(new ParsedSchemaMetadata(10, 2, schema)); - - byte[] result = serializer.serialize("bla", data); - - verify(retriever, times(1)).getOrSetSchemaMetadata("bla", false, schema, -1); - - SchemaRegistryClient registryClient = mock(SchemaRegistryClient.class); - io.confluent.kafka.serializers.KafkaAvroSerializer altSerializer = new io.confluent.kafka.serializers.KafkaAvroSerializer( - registryClient); - altSerializer.configure(Collections.singletonMap("schema.registry.url", "http://example.com:8081"), true); - - when(registryClient.register("bla-key", schema)).thenReturn(10); - - byte[] altResult = altSerializer.serialize("bla", data); - - verify(registryClient, times(1)).register("bla-key", schema); - - assertArrayEquals(altResult, result); - } -} \ No newline at end of file diff --git a/radar-commons-testing/build.gradle b/radar-commons-testing/build.gradle index 9a8267bb..8fb78f6f 100644 --- a/radar-commons-testing/build.gradle +++ b/radar-commons-testing/build.gradle @@ -16,7 +16,7 @@ apply plugin: 'application' -mainClassName = 'org.radarcns.mock.MockProducer' +mainClassName = 'org.radarbase.mock.MockProducer' run { if (project.hasProperty("mockConfig")) { @@ -26,11 +26,10 @@ run { } } -ext.testingName = 'radar-commons-testing' -ext.description = 'RADAR Common testing library mocking code and utilities.' +description = 'RADAR Common testing library mocking code and utilities.' -targetCompatibility = '1.8' -sourceCompatibility = '1.8' +targetCompatibility = '11.0' +sourceCompatibility = '11.0' repositories { maven { url 'https://oss.jfrog.org/artifactory/oss-snapshot-local' } @@ -42,8 +41,13 @@ dependencies { api group: 'org.apache.avro', name: 'avro', version: avroVersion api group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion + implementation group: 'com.opencsv', name: 'opencsv', version: '4.5' implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' , version: jacksonVersion implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion + implementation (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) { + exclude group: 'com.101tec' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + } runtimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion @@ -52,111 +56,5 @@ dependencies { testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: hamcrestVersion } -checkstyle { - // ignore tests - sourceSets = [sourceSets.main] - ignoreFailures = false -} - -pmd { - // ignore tests - sourceSets = [sourceSets.main] - ignoreFailures = false -} - -ext.sharedManifest = manifest { - attributes("Implementation-Title": testingName, - "Implementation-Version": version) -} - -jar { - baseName testingName - manifest.from sharedManifest -} - -// custom tasks for creating source/javadoc jars -task sourcesJar(type: Jar, dependsOn: classes) { - baseName testingName - classifier = 'sources' - from sourceSets.main.allSource - manifest.from sharedManifest -} - -task javadocJar(type: Jar, dependsOn: javadoc) { - baseName testingName - classifier = 'javadoc' - from javadoc.destinationDir - manifest.from sharedManifest -} - -// add javadoc/source jar tasks as artifacts -artifacts { - archives sourcesJar, javadocJar -} - -publishing { - publications { - RadarCommonsTestingPublication(MavenPublication) { - from components.java - artifact sourcesJar - artifact javadocJar - groupId project.group - artifactId testingName - version project.version - pom.withXml { - def root = asNode() - root.appendNode('description', description) - root.appendNode('name', testingName) - root.appendNode('url', githubUrl) - root.dependencies.'*'.findAll() { - it.artifactId.text() == 'slf4j-simple' - }.each() { - it.parent().remove(it) - } - root.children().last() + pomConfig - } - } - } -} - -bintray { - user = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - key = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - override = false - publications = ['RadarCommonsTestingPublication'] - pkg { - repo = project.group - name = testingName - userOrg = 'radar-cns' - desc = description - licenses = ['Apache-2.0'] - websiteUrl = website - issueTrackerUrl = issueUrl - vcsUrl = githubUrl - githubRepo = githubRepoName - githubReleaseNotesFile = 'README.md' - version { - name = project.version - desc = description - vcsTag = System.getenv('TRAVIS_TAG') - released = new Date() - } - } -} - - -artifactory { - contextUrl = 'https://oss.jfrog.org/artifactory' - publish { - repository { - repoKey = 'oss-snapshot-local' - username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - maven = true - } - } -} - -artifactoryPublish { - publications('RadarCommonsTestingPublication') -} +apply from: '../gradle/publishing.gradle' +apply from: '../gradle/codestyle.gradle' diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/MockDevice.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java similarity index 95% rename from radar-commons-testing/src/main/java/org/radarcns/mock/MockDevice.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java index b0503257..16e547d5 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/MockDevice.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.mock; +package org.radarbase.mock; import java.io.IOException; import java.math.BigInteger; @@ -24,11 +24,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.avro.SchemaValidationException; import org.apache.avro.specific.SpecificRecord; -import org.radarcns.data.Record; -import org.radarcns.mock.data.RecordGenerator; -import org.radarcns.producer.KafkaSender; -import org.radarcns.producer.KafkaTopicSender; -import org.radarcns.util.Oscilloscope; +import org.radarbase.data.Record; +import org.radarbase.mock.data.RecordGenerator; +import org.radarbase.producer.KafkaSender; +import org.radarbase.producer.KafkaTopicSender; +import org.radarbase.util.Oscilloscope; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/MockFileSender.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java similarity index 89% rename from radar-commons-testing/src/main/java/org/radarcns/mock/MockFileSender.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java index 934b6e41..f1638213 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/MockFileSender.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java @@ -14,14 +14,14 @@ * limitations under the License. */ -package org.radarcns.mock; +package org.radarbase.mock; import java.io.IOException; import org.apache.avro.SchemaValidationException; -import org.radarcns.data.Record; -import org.radarcns.mock.data.MockCsvParser; -import org.radarcns.producer.KafkaSender; -import org.radarcns.producer.KafkaTopicSender; +import org.radarbase.data.Record; +import org.radarbase.mock.data.MockCsvParser; +import org.radarbase.producer.KafkaSender; +import org.radarbase.producer.KafkaTopicSender; /** * Send mock data from a CSV file. diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/MockProducer.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java similarity index 82% rename from radar-commons-testing/src/main/java/org/radarcns/mock/MockProducer.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java index 7c52ba26..d70772cf 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/MockProducer.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java @@ -14,16 +14,17 @@ * limitations under the License. */ -package org.radarcns.mock; +package org.radarbase.mock; +import static io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; -import static org.radarcns.util.serde.AbstractKafkaAvroSerde.SCHEMA_REGISTRY_CONFIG; -import java.io.File; +import io.confluent.kafka.serializers.KafkaAvroSerializer; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -31,27 +32,26 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.avro.SchemaValidationException; -import org.radarcns.config.ServerConfig; -import org.radarcns.config.YamlConfigLoader; +import org.radarbase.config.ServerConfig; +import org.radarbase.config.YamlConfigLoader; +import org.radarbase.mock.config.MockDataConfig; +import org.radarbase.mock.data.MockCsvParser; +import org.radarbase.mock.data.RecordGenerator; +import org.radarbase.producer.BatchedKafkaSender; +import org.radarbase.producer.KafkaSender; +import org.radarbase.producer.direct.DirectSender; +import org.radarbase.producer.rest.ConnectionState; +import org.radarbase.producer.rest.RestClient; +import org.radarbase.producer.rest.RestSender; +import org.radarbase.producer.rest.SchemaRetriever; import org.radarcns.kafka.ObservationKey; -import org.radarcns.mock.config.BasicMockConfig; -import org.radarcns.mock.config.MockDataConfig; -import org.radarcns.mock.data.MockCsvParser; -import org.radarcns.mock.data.RecordGenerator; +import org.radarbase.mock.config.BasicMockConfig; import org.radarcns.passive.empatica.EmpaticaE4Acceleration; import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; import org.radarcns.passive.empatica.EmpaticaE4ElectroDermalActivity; import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; import org.radarcns.passive.empatica.EmpaticaE4Temperature; -import org.radarcns.producer.BatchedKafkaSender; -import org.radarcns.producer.KafkaSender; -import org.radarcns.producer.direct.DirectSender; -import org.radarcns.producer.rest.ConnectionState; -import org.radarcns.producer.rest.RestClient; -import org.radarcns.producer.rest.RestSender; -import org.radarcns.producer.rest.SchemaRetriever; -import org.radarcns.util.serde.KafkaAvroSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +59,7 @@ * A Mock Producer class that can be used to stream data. It can use MockFileSender and MockDevice * for testing purposes, with direct or indirect streaming. */ +@SuppressWarnings("PMD.DoNotCallSystemExit") public class MockProducer { private static final Logger logger = LoggerFactory.getLogger(MockProducer.class); @@ -84,7 +85,7 @@ public MockProducer(BasicMockConfig mockConfig) throws IOException { * @param root root directory of where mock files are located * @throws IOException if data could not be sent */ - public MockProducer(BasicMockConfig mockConfig, File root) throws IOException { + public MockProducer(BasicMockConfig mockConfig, Path root) throws IOException { int numDevices = mockConfig.getNumberOfDevices(); retriever = new SchemaRetriever(mockConfig.getSchemaRegistry(), 10); @@ -101,13 +102,8 @@ public MockProducer(BasicMockConfig mockConfig, File root) throws IOException { List> generators; List> mockFiles; - try { - generators = createGenerators(dataConfigs); - mockFiles = createMockFiles(dataConfigs, root); - } catch (NoSuchMethodException | IllegalAccessException | ClassNotFoundException - | InvocationTargetException ex) { - throw new IllegalStateException("Configuration invalid", ex); - } + generators = createGenerators(dataConfigs); + mockFiles = createMockFiles(dataConfigs, root); tmpSenders = createSenders(mockConfig, numDevices + mockFiles.size()); @@ -140,7 +136,8 @@ private List createSenders( BasicMockConfig mockConfig, int numDevices) { if (mockConfig.isDirectProducer()) { - return createDirectSenders(numDevices, retriever, mockConfig.getBrokerPaths()); + return createDirectSenders(numDevices, mockConfig.getSchemaRegistry().getUrlString(), + mockConfig.getBrokerPaths()); } else { return createRestSenders(numDevices, retriever, mockConfig.getRestProxy(), mockConfig.hasCompression()); @@ -149,13 +146,13 @@ private List createSenders( /** Create senders that directly produce data to Kafka. */ private List createDirectSenders(int numDevices, - SchemaRetriever retriever, String brokerPaths) { + String retrieverUrl, String brokerPaths) { List result = new ArrayList<>(numDevices); for (int i = 0; i < numDevices; i++) { Properties properties = new Properties(); properties.put(KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class); properties.put(VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class); - properties.put(SCHEMA_REGISTRY_CONFIG, retriever); + properties.put(SCHEMA_REGISTRY_URL_CONFIG, retrieverUrl); properties.put(BOOTSTRAP_SERVERS_CONFIG, brokerPaths); result.add(new DirectSender(properties)); @@ -227,7 +224,7 @@ public static void main(String[] args) { System.exit(1); } - File mockFile = new File(args[0]).getAbsoluteFile(); + Path mockFile = Paths.get(args[0]).toAbsolutePath(); BasicMockConfig config = null; try { config = new YamlConfigLoader().load(mockFile, BasicMockConfig.class); @@ -237,7 +234,7 @@ public static void main(String[] args) { } try { - MockProducer producer = new MockProducer(config, mockFile.getParentFile()); + MockProducer producer = new MockProducer(config, mockFile.getParent()); producer.start(); waitForProducer(producer, config.getDuration()); } catch (IllegalArgumentException ex) { @@ -256,19 +253,17 @@ private static void waitForProducer(final MockProducer producer, long duration) throws IOException, InterruptedException, SchemaValidationException { final AtomicBoolean isShutdown = new AtomicBoolean(false); - Runtime.getRuntime().addShutdownHook(new Thread() { - public void run() { - try { - if (!isShutdown.get()) { - producer.shutdown(); - } - } catch (InterruptedException ex) { - logger.warn("Shutdown interrupted", ex); - } catch (Exception ex) { - logger.warn("Failed to shutdown producer", ex); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + if (!isShutdown.get()) { + producer.shutdown(); } + } catch (InterruptedException ex) { + logger.warn("Shutdown interrupted", ex); + } catch (Exception ex) { + logger.warn("Failed to shutdown producer", ex); } - }); + })); if (duration <= 0L) { try { @@ -336,9 +331,7 @@ private List defaultDataConfig() { return Arrays.asList(acceleration, battery, bvp, eda, ibi, temperature); } - private List> createGenerators(List configs) - throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, - InvocationTargetException { + private List> createGenerators(List configs) { List> result = new ArrayList<>(configs.size()); @@ -352,20 +345,18 @@ private List> createGenerators(List> createMockFiles(List configs, - File dataRoot) - throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, - InvocationTargetException, IOException { + Path dataRoot) throws IOException { List> result = new ArrayList<>(configs.size()); - File parent = dataRoot; + Path parent = dataRoot; if (parent == null) { - parent = new File(".").getAbsoluteFile(); + parent = Paths.get(".").toAbsolutePath(); } for (MockDataConfig config : configs) { if (config.getDataFile() != null) { - result.add(new MockCsvParser(config, parent)); + result.add(new MockCsvParser<>(config, parent)); } } diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/config/BasicMockConfig.java b/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java similarity index 97% rename from radar-commons-testing/src/main/java/org/radarcns/mock/config/BasicMockConfig.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java index c28998b2..753df59c 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/config/BasicMockConfig.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java @@ -14,12 +14,12 @@ * limitations under the License. */ -package org.radarcns.mock.config; +package org.radarbase.mock.config; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; import java.util.Objects; -import org.radarcns.config.ServerConfig; +import org.radarbase.config.ServerConfig; /** * A Minimal Mock Config to talk to Kafka and stream data using a mock set-up. diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/config/MockDataConfig.java b/radar-commons-testing/src/main/java/org/radarbase/mock/config/MockDataConfig.java similarity index 90% rename from radar-commons-testing/src/main/java/org/radarcns/mock/config/MockDataConfig.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/config/MockDataConfig.java index a6769bf3..93a2fd9b 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/config/MockDataConfig.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/config/MockDataConfig.java @@ -14,16 +14,17 @@ * limitations under the License. */ -package org.radarcns.mock.config; +package org.radarbase.mock.config; import com.fasterxml.jackson.annotation.JsonProperty; -import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.List; import org.apache.avro.specific.SpecificRecord; -import org.radarcns.config.AvroTopicConfig; +import org.radarbase.config.AvroTopicConfig; +import org.radarbase.topic.AvroTopic; import org.radarcns.kafka.ObservationKey; -import org.radarcns.topic.AvroTopic; public class MockDataConfig extends AvroTopicConfig { @JsonProperty("file") @@ -64,13 +65,13 @@ public AvroTopic pars * @return absolute path to the data file * @throws NullPointerException if root is null */ - public File getDataFile(File root) { - File directDataFile = new File(dataFile); + public Path getDataFile(Path root) { + Path directDataFile = Paths.get(dataFile); if (directDataFile.isAbsolute()) { return directDataFile; } else { - File absoluteFile = new File(root, dataFile); - this.absolutePath = absoluteFile.getAbsolutePath(); + Path absoluteFile = root.resolve(dataFile).toAbsolutePath(); + this.absolutePath = absoluteFile.toString(); return absoluteFile; } } diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/data/CsvGenerator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/CsvGenerator.java similarity index 77% rename from radar-commons-testing/src/main/java/org/radarcns/mock/data/CsvGenerator.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/data/CsvGenerator.java index 4ceda9d4..0ce4c0ab 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/data/CsvGenerator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/CsvGenerator.java @@ -14,13 +14,15 @@ * limitations under the License. */ -package org.radarcns.mock.data; +package org.radarbase.mock.data; -import java.io.File; +import com.opencsv.CSVWriter; import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import org.radarbase.mock.config.MockDataConfig; import org.radarcns.kafka.ObservationKey; -import org.radarcns.mock.config.MockDataConfig; -import org.radarcns.util.CsvWriter; /** * It generates a CVS file that can be used to stream data and @@ -47,9 +49,9 @@ public CsvGenerator(ObservationKey key) { * @param root directory relative to which the output csv file is generated * @throws IOException if the CSV file cannot be written to */ - public void generate(MockDataConfig config, long duration, File root) + public void generate(MockDataConfig config, long duration, Path root) throws IOException { - File file = config.getDataFile(root); + Path file = config.getDataFile(root); generate(new RecordGenerator<>(config, ObservationKey.class), duration, file); } @@ -62,10 +64,12 @@ public void generate(MockDataConfig config, long duration, File root) * @param csvFile CSV file to write data to * @throws IOException if the CSV file cannot be written to */ - public void generate(RecordGenerator generator, long duration, File csvFile) + public void generate(RecordGenerator generator, long duration, Path csvFile) throws IOException { - try (CsvWriter writer = new CsvWriter(csvFile, generator.getHeader())) { - writer.writeRows(generator.iterateRawValues(key, duration)); + try (Writer writer = Files.newBufferedWriter(csvFile); + CSVWriter csvWriter = new CSVWriter(writer)) { + csvWriter.writeNext(generator.getHeader().toArray(new String[0])); + csvWriter.writeAll(generator.iteratableRawValues(key, duration)); } } } diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/data/MockCsvParser.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java similarity index 84% rename from radar-commons-testing/src/main/java/org/radarcns/mock/data/MockCsvParser.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java index 5c9c0ce1..0c3e4dcd 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/data/MockCsvParser.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java @@ -14,14 +14,14 @@ * limitations under the License. */ -package org.radarcns.mock.data; +package org.radarbase.mock.data; +import com.opencsv.CSVReader; import java.io.BufferedReader; import java.io.Closeable; -import java.io.File; -import java.io.FileReader; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -30,10 +30,9 @@ import org.apache.avro.Schema.Field; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificRecord; -import org.radarcns.data.Record; -import org.radarcns.mock.config.MockDataConfig; -import org.radarcns.topic.AvroTopic; -import org.radarcns.util.CsvParser; +import org.radarbase.data.Record; +import org.radarbase.mock.config.MockDataConfig; +import org.radarbase.topic.AvroTopic; /** * Parse mock data from a CSV file @@ -46,10 +45,9 @@ public class MockCsvParser implements Closeable { private final AvroTopic topic; private final Map headerMap; - private final CsvParser csvReader; + private final CSVReader csvReader; private final BufferedReader bufferedReader; - private final FileReader fileReader; - private List currentLine; + private String[] currentLine; /** * Base constructor. @@ -57,21 +55,18 @@ public class MockCsvParser implements Closeable { * @param root parent directory of the data file. * @throws IllegalArgumentException if the second row has the wrong number of columns */ - public MockCsvParser(MockDataConfig config, File root) - throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, - IllegalAccessException, IOException { + public MockCsvParser(MockDataConfig config, Path root) throws IOException { //noinspection unchecked topic = config.parseAvroTopic(); - fileReader = new FileReader(config.getDataFile(root)); - bufferedReader = new BufferedReader(fileReader); - csvReader = new CsvParser(bufferedReader); - List header = csvReader.parseLine(); + bufferedReader = Files.newBufferedReader(config.getDataFile(root)); + csvReader = new CSVReader(bufferedReader); + String[] header = csvReader.readNext(); headerMap = new HashMap<>(); - for (int i = 0; i < header.size(); i++) { - headerMap.put(header.get(i), i); + for (int i = 0; i < header.length; i++) { + headerMap.put(header[i], i); } - currentLine = csvReader.parseLine(); + currentLine = csvReader.readNext(); } public AvroTopic getTopic() { @@ -95,7 +90,7 @@ public Record next() throws IOException { SpecificRecord value = parseRecord(currentLine, headerMap, topic.getValueClass(), topic.getValueSchema()); - currentLine = csvReader.parseLine(); + currentLine = csvReader.readNext(); return new Record<>(key, value); } @@ -107,13 +102,13 @@ public boolean hasNext() { return currentLine != null; } - private V parseRecord(List rawValues, + private V parseRecord(String[] rawValues, Map header, Class recordClass, Schema schema) { @SuppressWarnings("unchecked") V record = (V) SpecificData.newInstance(recordClass, schema); for (Field field : schema.getFields()) { - String fieldString = rawValues.get(header.get(field.name())); + String fieldString = rawValues[header.get(field.name())]; Object fieldValue = parseValue(field.schema(), fieldString); record.put(field.pos(), fieldValue); } @@ -208,6 +203,5 @@ private static List parseArray(Schema schema, String fieldString) { public void close() throws IOException { csvReader.close(); bufferedReader.close(); - fileReader.close(); } } diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/data/MockRecordValidator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java similarity index 81% rename from radar-commons-testing/src/main/java/org/radarcns/mock/data/MockRecordValidator.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java index 0c09160a..607fc459 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/data/MockRecordValidator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java @@ -14,15 +14,14 @@ * limitations under the License. */ -package org.radarcns.mock.data; +package org.radarbase.mock.data; -import java.io.File; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; +import java.nio.file.Path; import org.apache.avro.specific.SpecificRecord; -import org.radarcns.data.Record; +import org.radarbase.data.Record; +import org.radarbase.mock.config.MockDataConfig; import org.radarcns.kafka.ObservationKey; -import org.radarcns.mock.config.MockDataConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,13 +33,13 @@ public class MockRecordValidator { private static final Logger logger = LoggerFactory.getLogger(MockRecordValidator.class); private final MockDataConfig config; private final long duration; - private final File root; + private final Path root; private int timePos; private double time; private double startTime; /** Create a new validator for given configuration. */ - public MockRecordValidator(MockDataConfig config, long duration, File root) { + public MockRecordValidator(MockDataConfig config, long duration, Path root) { this.config = config; this.duration = duration; this.root = root; @@ -74,14 +73,6 @@ public void validate() { checkFrequency(line); } catch (IOException e) { error("Cannot open file", -1, e); - } catch (InvocationTargetException e) { - error("Cannot instantiate mock type", -1, e); - } catch (NoSuchMethodException e) { - error("Mock type is not a SpecificRecord", -1, e); - } catch (IllegalAccessException e) { - error("Mock type is not accessible", -1, e); - } catch (ClassNotFoundException e) { - error("Mock type class does not exist", -1, e); } } @@ -107,13 +98,23 @@ private void checkRecord(Record record, } private void error(String message, long line, Exception ex) { - String mex = config.getDataFile() + " with topic " + config.getTopic() + " is invalid"; + StringBuilder messageBuilder = new StringBuilder(150); + messageBuilder + .append(config.getDataFile()) + .append(" with topic ") + .append(config.getTopic()) + .append(" is invalid"); if (line > 0L) { - mex += " on line " + line; + messageBuilder + .append(" on line ") + .append(line); } - mex += ". " + message; - logger.error(mex); - throw new IllegalArgumentException(mex, ex); + String fullMessage = messageBuilder + .append(". ") + .append(message) + .toString(); + logger.error(fullMessage); + throw new IllegalArgumentException(fullMessage, ex); } private void checkDuration() { diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/data/RecordGenerator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java similarity index 90% rename from radar-commons-testing/src/main/java/org/radarcns/mock/data/RecordGenerator.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java index aebaaa12..e2f17346 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/data/RecordGenerator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.mock.data; +package org.radarbase.mock.data; import java.util.ArrayList; import java.util.Arrays; @@ -28,10 +28,10 @@ import org.apache.avro.Schema.Field; import org.apache.avro.Schema.Type; import org.apache.avro.specific.SpecificRecord; -import org.radarcns.data.Record; -import org.radarcns.mock.config.MockDataConfig; -import org.radarcns.topic.AvroTopic; -import org.radarcns.util.Metronome; +import org.radarbase.data.Record; +import org.radarbase.mock.config.MockDataConfig; +import org.radarbase.topic.AvroTopic; +import org.radarbase.util.Metronome; /** * Generates records according to the specification in a {@link MockDataConfig}. @@ -125,10 +125,11 @@ private Field forceGetField(Schema schema, String name) { * @param key key to generate data with * @return list containing simulated values */ - public Iterator> iterateRawValues(K key, long duration) { - final Iterator> baseIterator = iterateValues(key, - duration); - return new RecordListIterator<>(baseIterator); + public Iterable iteratableRawValues(K key, long duration) { + return () -> { + final Iterator> baseIterator = iterateValues(key, duration); + return new RecordArrayIterator<>(baseIterator); + }; } /** @@ -159,12 +160,12 @@ private long getTimeReceived(long time) { return time + ThreadLocalRandom.current().nextLong(1 , 10); } - private static class RecordListIterator - implements Iterator> { + private static class RecordArrayIterator + implements Iterator { private final Iterator> baseIterator; - private RecordListIterator(Iterator> baseIterator) { + private RecordArrayIterator(Iterator> baseIterator) { this.baseIterator = baseIterator; } @@ -174,18 +175,18 @@ public boolean hasNext() { } @Override - public List next() { + public String[] next() { Record record = baseIterator.next(); int keyFieldsSize = record.key.getSchema().getFields().size(); int valueFieldsSize = record.value.getSchema().getFields().size(); - List result = new ArrayList<>(keyFieldsSize + valueFieldsSize); + String[] result = new String[keyFieldsSize + valueFieldsSize]; for (int i = 0; i < keyFieldsSize; i++) { - result.add(record.key.get(i).toString()); + result[i] = record.key.get(i).toString(); } for (int i = 0; i < valueFieldsSize; i++) { - result.add(record.value.get(i).toString()); + result[i + keyFieldsSize] = record.value.get(i).toString(); } return result; } diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/model/ExpectedArrayValue.java b/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedArrayValue.java similarity index 86% rename from radar-commons-testing/src/main/java/org/radarcns/mock/model/ExpectedArrayValue.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedArrayValue.java index af8e3e20..83c96d71 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/model/ExpectedArrayValue.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedArrayValue.java @@ -14,11 +14,11 @@ * limitations under the License. */ -package org.radarcns.mock.model; +package org.radarbase.mock.model; import java.util.List; import org.apache.avro.Schema; -import org.radarcns.stream.collector.AggregateListCollector; +import org.radarbase.stream.collector.AggregateListCollector; /** * {@code ExpectedValue} represented as {@code Double[]}. @@ -35,6 +35,6 @@ public ExpectedArrayValue(Schema valueSchema, List valueFields) { @Override protected AggregateListCollector createCollector() { - return new AggregateListCollector(fieldNames, schema); + return new AggregateListCollector(fieldNames, schema, true); } } diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/model/ExpectedDoubleValue.java b/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedDoubleValue.java similarity index 91% rename from radar-commons-testing/src/main/java/org/radarcns/mock/model/ExpectedDoubleValue.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedDoubleValue.java index 7ba50691..9ac6beb8 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/model/ExpectedDoubleValue.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedDoubleValue.java @@ -14,11 +14,11 @@ * limitations under the License. */ -package org.radarcns.mock.model; +package org.radarbase.mock.model; import java.util.List; import org.apache.avro.Schema; -import org.radarcns.stream.collector.NumericAggregateCollector; +import org.radarbase.stream.collector.NumericAggregateCollector; /** * {@code ExpectedValue} represented as {@code Double}. @@ -35,6 +35,6 @@ public ExpectedDoubleValue(Schema valueSchema, List valueFields) { @Override protected NumericAggregateCollector createCollector() { - return new NumericAggregateCollector(fieldNames[0], schema); + return new NumericAggregateCollector(fieldNames[0], schema, true); } } diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/model/ExpectedValue.java b/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedValue.java similarity index 92% rename from radar-commons-testing/src/main/java/org/radarcns/mock/model/ExpectedValue.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedValue.java index 81c74b15..c75251f1 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/model/ExpectedValue.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedValue.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.mock.model; +package org.radarbase.mock.model; import java.util.HashMap; import java.util.List; @@ -22,9 +22,9 @@ import java.util.concurrent.TimeUnit; import org.apache.avro.Schema; import org.apache.avro.specific.SpecificRecord; -import org.radarcns.data.Record; +import org.radarbase.data.Record; +import org.radarbase.stream.collector.RecordCollector; import org.radarcns.kafka.ObservationKey; -import org.radarcns.stream.collector.RecordCollector; /** * It computes the expected value for a test case. @@ -45,7 +45,7 @@ public abstract class ExpectedValue { **/ public ExpectedValue(Schema valueSchema, List fieldNames) { this.schema = valueSchema; - this.fieldNames = fieldNames.toArray(new String[fieldNames.size()]); + this.fieldNames = fieldNames.toArray(new String[0]); timeReceivedPos = valueSchema.getField("timeReceived").pos(); series = new HashMap<>(); diff --git a/radar-commons-testing/src/main/java/org/radarcns/mock/model/MockAggregator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/model/MockAggregator.java similarity index 82% rename from radar-commons-testing/src/main/java/org/radarcns/mock/model/MockAggregator.java rename to radar-commons-testing/src/main/java/org/radarbase/mock/model/MockAggregator.java index 4d303372..099ada0d 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/mock/model/MockAggregator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/model/MockAggregator.java @@ -14,17 +14,17 @@ * limitations under the License. */ -package org.radarcns.mock.model; +package org.radarbase.mock.model; -import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.avro.Schema; +import org.radarbase.mock.config.MockDataConfig; import org.radarcns.kafka.ObservationKey; -import org.radarcns.mock.config.MockDataConfig; -import org.radarcns.mock.data.MockCsvParser; +import org.radarbase.mock.data.MockCsvParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,14 +35,17 @@ *
  • singleton {@code Double} * */ -public final class MockAggregator { +public class MockAggregator { private static final Logger logger = LoggerFactory.getLogger(MockAggregator.class); + private final List mockDataConfigs; + private final Path root; /** * Default constructor. */ - private MockAggregator() { - // utility class + public MockAggregator(List mockDataConfigs, Path root) { + this.mockDataConfigs = mockDataConfigs; + this.root = root; } /** @@ -51,8 +54,8 @@ private MockAggregator() { * @return {@code Map} of key {@code MockDataConfig} and value {@code ExpectedValue}. {@link * ExpectedDoubleValue}. **/ - public static Map getSimulations( - List mockDataConfigs, File root) throws IOException { + @SuppressWarnings("unused") + public Map simulate() throws IOException { Map expectedValue = new HashMap<>(); @@ -78,8 +81,6 @@ public static Map getSimulations( } expectedValue.put(config, value); - } catch (ReflectiveOperationException ex) { - throw new IllegalArgumentException("Could not read topic", ex); } } diff --git a/radar-commons-testing/src/main/java/org/radarcns/util/Metronome.java b/radar-commons-testing/src/main/java/org/radarbase/util/Metronome.java similarity index 98% rename from radar-commons-testing/src/main/java/org/radarcns/util/Metronome.java rename to radar-commons-testing/src/main/java/org/radarbase/util/Metronome.java index a1d34ab1..b26b2d41 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/util/Metronome.java +++ b/radar-commons-testing/src/main/java/org/radarbase/util/Metronome.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.util; +package org.radarbase.util; import java.util.concurrent.TimeUnit; diff --git a/radar-commons-testing/src/main/java/org/radarcns/util/Oscilloscope.java b/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.java similarity index 98% rename from radar-commons-testing/src/main/java/org/radarcns/util/Oscilloscope.java rename to radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.java index d5d7ac6b..28492cd2 100644 --- a/radar-commons-testing/src/main/java/org/radarcns/util/Oscilloscope.java +++ b/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.util; +package org.radarbase.util; /** * Oscilloscope gives out a regular beat, at a given frequency per second. The intended way to use diff --git a/radar-commons-testing/src/main/java/org/radarcns/util/CsvParser.java b/radar-commons-testing/src/main/java/org/radarcns/util/CsvParser.java deleted file mode 100644 index f02270c6..00000000 --- a/radar-commons-testing/src/main/java/org/radarcns/util/CsvParser.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util; - -import java.io.Closeable; -import java.io.IOException; -import java.io.Reader; -import java.util.ArrayList; -import java.util.List; - -/** - * Parses a CSV file. - */ -public class CsvParser implements Closeable { - private static final char DEFAULT_SEPARATOR = ','; - private static final char DEFAULT_QUOTE = '"'; - - private final char separator; - private final char quote; - private final Reader reader; - private int length; - private int row; - - /** CsvParser from reader with double quotes '"' as quotes and comma ',' as separator. */ - public CsvParser(Reader r) { - this(r, DEFAULT_SEPARATOR, DEFAULT_QUOTE); - } - - /** CsvParser from reader. */ - public CsvParser(Reader r, char separator, char quote) { - this.reader = r; - this.separator = separator; - this.quote = quote; - this.length = -1; - this.row = 0; - } - - /** - * Returns a null when the input stream is empty. - */ - public List parseLine() throws IOException { - int ch = reader.read(); - while (ch == '\r') { - ch = reader.read(); - } - if (ch < 0) { - return null; - } - this.row++; - - List store = new ArrayList<>(length >= 0 ? length : 10); - ParsingState state = new ParsingState(); - StringBuilder builder = new StringBuilder(); - - while (ch >= 0) { - if (state.inQuotes) { - processQuoted(ch, state, builder); - } else if (!processUnquoted(ch, state, builder, store)) { - break; - } - ch = reader.read(); - } - store.add(builder.toString()); - - verifyLength(store); - - return store; - } - - private void verifyLength(List store) { - if (length == -1) { - length = store.size(); - } else if (length != store.size()) { - throw new IllegalArgumentException("CSV file does not have a fixed number of columns: " - + "row " + row + " contains " + store.size() - + " columns, instead of " + length + ". Row:\n\t" + store); - } - } - - private void processQuoted(int ch, ParsingState state, StringBuilder builder) { - state.inValue = true; - if (ch == quote) { - state.inQuotes = false; - } else { - builder.append((char) ch); - } - } - - private boolean processUnquoted(int ch, ParsingState state, StringBuilder builder, - List store) { - if (ch == quote) { - state.inQuotes = true; - if (state.inValue) { - // this is for the double quote in the middle of a value - builder.append(quote); - } - } else if (ch == separator) { - store.add(builder.toString()); - builder.setLength(0); - state.reset(); - } else if (ch == '\n') { - //end of a line, break out - return false; - } else if (ch != '\r') { //ignore LF characters - builder.append((char) ch); - } - return true; - } - - @Override - public void close() throws IOException { - reader.close(); - } - - private static class ParsingState { - private boolean inQuotes; - private boolean inValue; - - private void reset() { - inValue = false; - inQuotes = false; - } - } -} diff --git a/radar-commons-testing/src/main/java/org/radarcns/util/CsvWriter.java b/radar-commons-testing/src/main/java/org/radarcns/util/CsvWriter.java deleted file mode 100644 index 4f1dfe80..00000000 --- a/radar-commons-testing/src/main/java/org/radarcns/util/CsvWriter.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util; - -import java.io.BufferedWriter; -import java.io.Closeable; -import java.io.File; -import java.io.FileWriter; -import java.io.Flushable; -import java.io.IOException; -import java.io.Writer; -import java.util.Iterator; -import java.util.List; - -/** - * CSV writer. - */ -public class CsvWriter implements Flushable, Closeable { - public static final char DEFAULT_SEPARATOR = ','; - private final int separator; - private final boolean shouldClose; - private final Writer writer; - private final int headerLength; - - /** - * Constructs a CSV writer to write to a file. The header is written immediately. - * @param file file to write to, it will be created or emptied. - * @param header header to write - * @throws IOException if writing to the file fails - */ - public CsvWriter(File file, List header) throws IOException { - this(new BufferedWriter(new FileWriter(file, false)), header, true, DEFAULT_SEPARATOR); - } - - /** - * Constructs a CSV writer to write to a writer. The header is written immediately. - * @param writer writer to write to, it will not be closed when this CsvWriter is closed. - * @param header header to write - * @throws IOException if writing to the writer fails - */ - public CsvWriter(Writer writer, List header) throws IOException { - this(writer, header, false, DEFAULT_SEPARATOR); - } - - /** - * Constructs a CSV writer to write to a writer. The header is written immediately. - * @param writer writer to write to - * @param header header to write - * @param shouldClose if and only if true, given writer will be closed when this writer is - * closed - * @param separator separator to use in the CSV file, comma ',' by default. - * @throws IOException if writing to the writer fails - */ - public CsvWriter(Writer writer, List header, boolean shouldClose, char separator) - throws IOException { - this.separator = separator; - this.writer = writer; - this.shouldClose = shouldClose; - headerLength = header.size(); - writeRow(header); - } - - /** Write all rows in given iterator. */ - public void writeRows(Iterator> rows) throws IOException { - while (rows.hasNext()) { - writeRow(rows.next()); - } - } - - /** Write a single row, with each list entry as a column. */ - public void writeRow(List strings) throws IOException { - if (strings.size() != headerLength) { - throw new IllegalArgumentException("Row size does not match header size"); - } - boolean first = true; - for (String v : strings) { - if (first) { - first = false; - } else { - writer.write(separator); - } - writer.write(v); - } - writer.write('\n'); - } - - @Override - public void flush() throws IOException { - writer.flush(); - } - - @Override - public void close() throws IOException { - flush(); - if (shouldClose) { - try (Writer localWriter = writer) { - localWriter.flush(); - } - } - } -} diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java new file mode 100644 index 00000000..e713976c --- /dev/null +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.radarbase.mock; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import com.opencsv.CSVReader; +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.radarbase.mock.data.MockRecordValidatorTest; +import org.radarcns.kafka.ObservationKey; +import org.radarbase.mock.config.MockDataConfig; +import org.radarbase.mock.data.CsvGenerator; +import org.radarbase.mock.data.RecordGenerator; + +public class CsvGeneratorTest { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + private MockDataConfig makeConfig() throws IOException { + return MockRecordValidatorTest.makeConfig(folder); + } + + @Test + public void generateMockConfig() throws IOException { + CsvGenerator generator = new CsvGenerator(); + + MockDataConfig config = makeConfig(); + generator.generate(config, 100_000L, folder.getRoot().toPath()); + + Path p = Paths.get(config.getDataFile()); + try (Reader reader = Files.newBufferedReader(p); + CSVReader parser = new CSVReader(reader)) { + String[] headers = {"projectId", "userId", "sourceId", "time", "timeReceived", "light"}; + assertArrayEquals(headers, parser.readNext()); + + int n = 0; + String[] line; + while ((line = parser.readNext()) != null) { + String value = line[5]; + assertNotEquals("NaN", value); + assertNotEquals("Infinity", value); + assertNotEquals("-Infinity", value); + // no decimals lost or appended + assertEquals(value, Float.valueOf(value).toString()); + n++; + } + assertEquals(100, n); + } + } + + @Test + public void generateGenerator() throws IOException { + CsvGenerator generator = new CsvGenerator(); + + MockDataConfig config = makeConfig(); + + final String time = Double.toString(System.currentTimeMillis() / 1000d); + + RecordGenerator recordGenerator = new RecordGenerator( + config, ObservationKey.class) { + + @Override + public Iterable iteratableRawValues(ObservationKey key, long duration) { + return List.of(new String[] { + "test", "UserID_0", "SourceID_0", time, time, + Float.valueOf((float)0.123112412410423518).toString() + }); + } + }; + + generator.generate(recordGenerator, 1000L, Paths.get(config.getDataFile())); + + Path p = Paths.get(config.getDataFile()); + + try (Reader reader = Files.newBufferedReader(p); + CSVReader parser = new CSVReader(reader)) { + assertArrayEquals( + recordGenerator.getHeader().toArray(new String[0]), + parser.readNext()); + // float will cut off a lot of decimals + assertArrayEquals( + new String[] { "test", "UserID_0", "SourceID_0", time, time, "0.12311241" }, + parser.readNext()); + } + } +} \ No newline at end of file diff --git a/radar-commons-testing/src/test/java/org/radarcns/mock/RecordGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java similarity index 94% rename from radar-commons-testing/src/test/java/org/radarcns/mock/RecordGeneratorTest.java rename to radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java index e55d3929..19eadee9 100644 --- a/radar-commons-testing/src/test/java/org/radarcns/mock/RecordGeneratorTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.mock; +package org.radarbase.mock; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -23,10 +23,10 @@ import java.util.Iterator; import org.apache.avro.specific.SpecificRecord; import org.junit.Test; -import org.radarcns.data.Record; +import org.radarbase.data.Record; import org.radarcns.kafka.ObservationKey; -import org.radarcns.mock.config.MockDataConfig; -import org.radarcns.mock.data.RecordGenerator; +import org.radarbase.mock.config.MockDataConfig; +import org.radarbase.mock.data.RecordGenerator; import org.radarcns.passive.empatica.EmpaticaE4Acceleration; /** diff --git a/radar-commons-testing/src/test/java/org/radarcns/mock/data/MockRecordValidatorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java similarity index 72% rename from radar-commons-testing/src/test/java/org/radarcns/mock/data/MockRecordValidatorTest.java rename to radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java index ae0d15a3..1204cf64 100644 --- a/radar-commons-testing/src/test/java/org/radarcns/mock/data/MockRecordValidatorTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java @@ -14,16 +14,19 @@ * limitations under the License. */ -package org.radarcns.mock.data; +package org.radarbase.mock.data; -import java.io.FileWriter; import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; -import org.radarcns.mock.config.MockDataConfig; +import org.radarbase.mock.config.MockDataConfig; import org.radarcns.passive.phone.PhoneAcceleration; import org.radarcns.passive.phone.PhoneLight; @@ -33,11 +36,17 @@ public class MockRecordValidatorTest { @Rule public ExpectedException exception = ExpectedException.none(); + private Path root; private MockDataConfig makeConfig() throws IOException { return makeConfig(folder); } + @Before + public void setUp() { + root = folder.getRoot().toPath(); + } + public static MockDataConfig makeConfig(TemporaryFolder folder) throws IOException { MockDataConfig config = new MockDataConfig(); config.setDataFile(folder.newFile().getAbsolutePath()); @@ -52,10 +61,10 @@ public void validate() throws Exception { CsvGenerator generator = new CsvGenerator(); MockDataConfig config = makeConfig(); - generator.generate(config, 100_000L, folder.getRoot()); + generator.generate(config, 100_000L, root); // doesn't throw - new MockRecordValidator(config, 100_000L, folder.getRoot()).validate(); + new MockRecordValidator(config, 100_000L, root).validate(); } @Test @@ -63,51 +72,51 @@ public void validateWrongDuration() throws Exception { CsvGenerator generator = new CsvGenerator(); MockDataConfig config = makeConfig(); - generator.generate(config, 100_000L, folder.getRoot()); + generator.generate(config, 100_000L, root); exception.expect(IllegalArgumentException.class); - new MockRecordValidator(config, 10_000L, folder.getRoot()).validate(); + new MockRecordValidator(config, 10_000L, root).validate(); } @Test public void validateCustom() throws Exception { MockDataConfig config = makeConfig(); - try (FileWriter writer = new FileWriter(config.getDataFile(folder.getRoot()))) { + try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { writer.append("projectId,userId,sourceId,time,timeReceived,light\n"); writer.append("test,a,b,1,1,1\n"); writer.append("test,a,b,1,2,1\n"); } - new MockRecordValidator(config, 2_000L, folder.getRoot()).validate(); + new MockRecordValidator(config, 2_000L, root).validate(); } @Test public void validateWrongKey() throws Exception { MockDataConfig config = makeConfig(); - try (FileWriter writer = new FileWriter(config.getDataFile(folder.getRoot()))) { + try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { writer.append("projectId,userId,sourceId,time,timeReceived,light\n"); writer.append("test,a,b,1,1,1\n"); writer.append("test,a,c,1,2,1\n"); } exception.expect(IllegalArgumentException.class); - new MockRecordValidator(config, 2_000L, folder.getRoot()).validate(); + new MockRecordValidator(config, 2_000L, root).validate(); } @Test public void validateWrongTime() throws Exception { MockDataConfig config = makeConfig(); - try (FileWriter writer = new FileWriter(config.getDataFile(folder.getRoot()))) { + try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { writer.append("projectId,userId,sourceId,time,timeReceived,light\n"); writer.append("test,a,b,1,1,1\n"); writer.append("test,a,b,1,0,1\n"); } exception.expect(IllegalArgumentException.class); - new MockRecordValidator(config, 2_000L, folder.getRoot()).validate(); + new MockRecordValidator(config, 2_000L, root).validate(); } @@ -115,56 +124,56 @@ public void validateWrongTime() throws Exception { public void validateMissingKeyField() throws Exception { MockDataConfig config = makeConfig(); - try (FileWriter writer = new FileWriter(config.getDataFile(folder.getRoot()))) { + try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { writer.append("projectId,userId,time,timeReceived,light\n"); writer.append("test,a,1,1,1\n"); writer.append("test,a,1,2,1\n"); } exception.expect(NullPointerException.class); - new MockRecordValidator(config, 2_000L, folder.getRoot()).validate(); + new MockRecordValidator(config, 2_000L, root).validate(); } @Test public void validateMissingValueField() throws Exception { MockDataConfig config = makeConfig(); - try (FileWriter writer = new FileWriter(config.getDataFile(folder.getRoot()))) { + try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { writer.append("projectId,userId,sourceId,time,light\n"); writer.append("test,a,b,1,1\n"); writer.append("test,a,b,1,2\n"); } exception.expect(NullPointerException.class); - new MockRecordValidator(config, 2_000L, folder.getRoot()).validate(); + new MockRecordValidator(config, 2_000L, root).validate(); } @Test public void validateMissingValue() throws Exception { MockDataConfig config = makeConfig(); - try (FileWriter writer = new FileWriter(config.getDataFile(folder.getRoot()))) { + try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { writer.append("projectId,userId,sourceId,time,timeReceived,light\n"); writer.append("test,a,b,1,1\n"); writer.append("test,a,b,1,2,1\n"); } - exception.expect(IllegalArgumentException.class); - new MockRecordValidator(config, 2_000L, folder.getRoot()).validate(); + exception.expect(ArrayIndexOutOfBoundsException.class); + new MockRecordValidator(config, 2_000L, root).validate(); } @Test public void validateWrongValueType() throws Exception { MockDataConfig config = makeConfig(); - try (FileWriter writer = new FileWriter(config.getDataFile(folder.getRoot()))) { + try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { writer.append("projectId,userId,sourceId,time,timeReceived,light\n"); writer.append("test,a,b,1,1,a\n"); writer.append("test,a,b,1,2,b\n"); } exception.expect(NumberFormatException.class); - new MockRecordValidator(config, 2_000L, folder.getRoot()).validate(); + new MockRecordValidator(config, 2_000L, root).validate(); } @Test @@ -173,12 +182,12 @@ public void validateMultipleFields() throws Exception { config.setValueSchema(PhoneAcceleration.class.getName()); config.setValueFields(Arrays.asList("x", "y", "z")); - try (FileWriter writer = new FileWriter(config.getDataFile(folder.getRoot()))) { + try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { writer.append("projectId,userId,sourceId,time,timeReceived,x,y,z\n"); writer.append("test,a,b,1,1,1,1,1\n"); writer.append("test,a,b,1,2,1,1,1\n"); } - new MockRecordValidator(config, 2_000L, folder.getRoot()).validate(); + new MockRecordValidator(config, 2_000L, root).validate(); } } \ No newline at end of file diff --git a/radar-commons-testing/src/test/java/org/radarcns/util/MetronomeTest.java b/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java similarity index 98% rename from radar-commons-testing/src/test/java/org/radarcns/util/MetronomeTest.java rename to radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java index f995082f..86330aa8 100644 --- a/radar-commons-testing/src/test/java/org/radarcns/util/MetronomeTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.util; +package org.radarbase.util; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; diff --git a/radar-commons-testing/src/test/java/org/radarcns/util/OscilloscopeTest.java b/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java similarity index 98% rename from radar-commons-testing/src/test/java/org/radarcns/util/OscilloscopeTest.java rename to radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java index ed6ab3f1..75a2e4db 100644 --- a/radar-commons-testing/src/test/java/org/radarcns/util/OscilloscopeTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.util; +package org.radarbase.util; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; diff --git a/radar-commons-testing/src/test/java/org/radarcns/mock/CsvGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarcns/mock/CsvGeneratorTest.java deleted file mode 100644 index 71879ffa..00000000 --- a/radar-commons-testing/src/test/java/org/radarcns/mock/CsvGeneratorTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.mock; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.radarcns.kafka.ObservationKey; -import org.radarcns.mock.config.MockDataConfig; -import org.radarcns.mock.data.CsvGenerator; -import org.radarcns.mock.data.MockRecordValidatorTest; -import org.radarcns.mock.data.RecordGenerator; -import org.radarcns.util.CsvParser; - -public class CsvGeneratorTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - private MockDataConfig makeConfig() throws IOException { - return MockRecordValidatorTest.makeConfig(folder); - } - - @Test - public void generateMockConfig() throws IOException { - CsvGenerator generator = new CsvGenerator(); - - MockDataConfig config = makeConfig(); - generator.generate(config, 100_000L, folder.getRoot()); - - CsvParser parser = new CsvParser(new BufferedReader(new FileReader(config.getDataFile()))); - List headers = Arrays.asList( - "projectId", "userId", "sourceId", "time", "timeReceived", "light"); - assertEquals(headers, parser.parseLine()); - - int n = 0; - List line; - while ((line = parser.parseLine()) != null) { - String value = line.get(5); - assertNotEquals("NaN", value); - assertNotEquals("Infinity", value); - assertNotEquals("-Infinity", value); - // no decimals lost or appended - assertEquals(value, Float.valueOf(value).toString()); - n++; - } - assertEquals(100, n); - } - - @Test - public void generateGenerator() - throws IOException, ClassNotFoundException, NoSuchMethodException, - IllegalAccessException, InvocationTargetException { - CsvGenerator generator = new CsvGenerator(); - - MockDataConfig config = makeConfig(); - - final String time = Double.toString(System.currentTimeMillis() / 1000d); - - RecordGenerator recordGenerator = new RecordGenerator( - config, ObservationKey.class) { - @Override - public Iterator> iterateRawValues(ObservationKey key, long duration) { - return Collections.singletonList( - Arrays.asList("test", "UserID_0", "SourceID_0", time, time, - Float.valueOf((float)0.123112412410423518).toString())) - .iterator(); - } - }; - - generator.generate(recordGenerator, 1000L, new File(config.getDataFile())); - - CsvParser parser = new CsvParser(new BufferedReader(new FileReader(config.getDataFile()))); - assertEquals(recordGenerator.getHeader(), parser.parseLine()); - // float will cut off a lot of decimals - assertEquals(Arrays.asList("test", "UserID_0", "SourceID_0", time, time, "0.12311241"), - parser.parseLine()); - } -} \ No newline at end of file diff --git a/radar-commons-testing/src/test/java/org/radarcns/util/CsvParserTest.java b/radar-commons-testing/src/test/java/org/radarcns/util/CsvParserTest.java deleted file mode 100644 index 0062e4c0..00000000 --- a/radar-commons-testing/src/test/java/org/radarcns/util/CsvParserTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; - -import java.io.StringReader; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -public class CsvParserTest { - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Test - public void parseLine() throws Exception { - try (StringReader input = new StringReader("a,b,c\n0,1,\"1,1\""); - CsvParser parser = new CsvParser(input)) { - assertThat(parser.parseLine(), contains("a", "b", "c")); - assertThat(parser.parseLine(), contains("0", "1", "1,1")); - } - } - - @Test - public void parseLineException() throws Exception { - try (StringReader input = new StringReader("a,b,c\n0,1,1,1"); - CsvParser parser = new CsvParser(input)) { - assertThat(parser.parseLine(), contains("a", "b", "c")); - exception.expect(IllegalArgumentException.class); - parser.parseLine(); - } - } -} \ No newline at end of file diff --git a/radar-commons-testing/src/test/java/org/radarcns/util/CsvWriterTest.java b/radar-commons-testing/src/test/java/org/radarcns/util/CsvWriterTest.java deleted file mode 100644 index f46a02dd..00000000 --- a/radar-commons-testing/src/test/java/org/radarcns/util/CsvWriterTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.nio.file.Files; -import java.util.Arrays; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - - -public class CsvWriterTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Test - public void write() throws Exception { - File f = folder.newFile(); - try (CsvWriter writer = new CsvWriter(f, Arrays.asList("a", "b"))) { - writer.writeRows(Arrays.asList( - Arrays.asList("1", "2"), - Arrays.asList("3", "4")).iterator()); - } - - String result = new String(Files.readAllBytes(f.toPath())); - assertThat(result, is(equalTo("a,b\n1,2\n3,4\n"))); - - try (FileReader fr = new FileReader(f); - BufferedReader reader = new BufferedReader(fr)) { - CsvParser parser = new CsvParser(reader); - assertThat(parser.parseLine(), contains("a", "b")); - assertThat(parser.parseLine(), contains("1", "2")); - assertThat(parser.parseLine(), contains("3", "4")); - assertThat(parser.parseLine(), is(nullValue())); - } - } -} \ No newline at end of file diff --git a/radar-commons-unsafe/build.gradle b/radar-commons-unsafe/build.gradle index 198560ab..ceacc03d 100644 --- a/radar-commons-unsafe/build.gradle +++ b/radar-commons-unsafe/build.gradle @@ -1,5 +1,4 @@ -ext.moduleName = 'radar-commons-unsafe' -ext.description = 'RADAR Common unsafe libraries overriding stock Kafka and Confluent code.' +description = 'RADAR Common unsafe libraries overriding stock Kafka and Confluent code.' targetCompatibility = '1.8' sourceCompatibility = '1.8' @@ -8,10 +7,6 @@ sourceCompatibility = '1.8' // Sources and classpath configurations // //---------------------------------------------------------------------------// -configurations { - codacy -} - configurations.compile { resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' } @@ -30,114 +25,4 @@ dependencies { } -//---------------------------------------------------------------------------// -// Testing // -//---------------------------------------------------------------------------// -checkstyle { - // ignore everything - sourceSets = [] -} - -pmd { - // ignore everything - sourceSets = [] -} - -task downloadDependencies(type: Exec) { - configurations.testRuntime.files - configurations.codacy.files - configurations.jacocoAnt.files - commandLine 'echo', 'Downloaded all dependencies' -} - -//---------------------------------------------------------------------------// -// Build system metadata // -//---------------------------------------------------------------------------// - -ext.sharedManifest = manifest { - attributes("Implementation-Title": moduleName, - "Implementation-Version": version) -} - -jar { - manifest.from sharedManifest -} -// custom tasks for creating source/javadoc jars -task sourcesJar(type: Jar, dependsOn: classes) { - classifier = 'sources' - from sourceSets.main.allSource - manifest.from sharedManifest -} - -task javadocJar(type: Jar, dependsOn: javadoc) { - classifier = 'javadoc' - from javadoc.destinationDir - manifest.from sharedManifest -} - -// add javadoc/source jar tasks as artifacts -artifacts { - archives javadocJar - archives sourcesJar -} - -publishing { - publications { - RadarCommonsUnsafePublication(MavenPublication) { - from components.java - artifact sourcesJar - artifact javadocJar - groupId project.group - artifactId moduleName - version project.version - pom.withXml { - def root = asNode() - root.appendNode('description', description) - root.appendNode('name', moduleName) - root.appendNode('url', githubUrl) - root.children().last() + pomConfig - } - } - } -} - -bintray { - user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - override false - publications 'RadarCommonsUnsafePublication' - pkg { - repo = project.group - name = moduleName - userOrg = 'radar-cns' - desc = description - licenses = ['Apache-2.0'] - websiteUrl = website - issueTrackerUrl = issueUrl - vcsUrl = githubUrl - githubRepo = githubRepoName - githubReleaseNotesFile = 'README.md' - version { - name = project.version - desc = description - vcsTag = System.getenv('TRAVIS_TAG') - released = new Date() - } - } -} - -artifactory { - contextUrl = 'https://oss.jfrog.org/artifactory' - publish { - repository { - repoKey = 'oss-snapshot-local' - username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - maven = true - } - } -} - -artifactoryPublish { - publications('RadarCommonsUnsafePublication') -} \ No newline at end of file +apply from: '../gradle/publishing.gradle' diff --git a/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java b/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java index 6165d75e..4d8b2f44 100644 --- a/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java +++ b/radar-commons-unsafe/src/main/java/io/confluent/kafka/serializers/AbstractKafkaAvroDeserializer.java @@ -1,5 +1,5 @@ /* - * Copyright 2014 Confluent Inc. + * Copyright 2018 Confluent Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,6 @@ package io.confluent.kafka.serializers; -import io.confluent.kafka.schemaregistry.client.SchemaMetadata; -import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.IdentityHashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import kafka.utils.VerifiableProperties; import org.apache.avro.Schema; import org.apache.avro.generic.GenericContainer; import org.apache.avro.generic.GenericDatumReader; @@ -35,17 +26,23 @@ import org.apache.avro.specific.SpecificRecord; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.SerializationException; -import org.codehaus.jackson.node.JsonNodeFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; +import io.confluent.kafka.schemaregistry.client.SchemaMetadata; +import kafka.utils.VerifiableProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public abstract class AbstractKafkaAvroDeserializer extends AbstractKafkaAvroSerDe { - public static final String SCHEMA_REGISTRY_SCHEMA_VERSION_PROP = - "schema.registry.schema.version"; - - private final Map> oldToNewIdMap = new IdentityHashMap<>(); - private final Map> oldToNewVersionMap = new IdentityHashMap<>(); + private final ConcurrentMap> oldToNewIdMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> oldToNewVersionMap = new ConcurrentHashMap<>(); private static final Logger logger = LoggerFactory.getLogger(AbstractKafkaAvroDeserializer.class); @@ -53,7 +50,6 @@ public abstract class AbstractKafkaAvroDeserializer extends AbstractKafkaAvroSer protected boolean useSpecificAvroReader = false; private final Map readerSchemaCache = new ConcurrentHashMap(); - /** * Sets properties for this deserializer without overriding the schema registry client itself. * Useful for testing, where a mock client is injected. @@ -126,18 +122,16 @@ protected Object deserialize(boolean includeSchemaAndVersion, String topic, Bool try { ByteBuffer buffer = getByteBuffer(payload); id = buffer.getInt(); - String subject = includeSchemaAndVersion ? getSubjectName(topic, isKey, null) : null; Schema schema; + String subject = null; + if (includeSchemaAndVersion) { + subject = subjectName(topic, isKey, null); + } SchemaMetadata schemaMetadata = null; - Map subjectIdMap = oldToNewIdMap.get(subject); + ConcurrentMap subjectIdMap = oldToNewIdMap.computeIfAbsent(subject, sub -> new ConcurrentHashMap<>()); try { - if (subjectIdMap != null) { - Integer newId = subjectIdMap.get(id); - if (newId != null) { - id = newId; - } - } - schema = getBySubjectAndId(subject, id); + id = subjectIdMap.getOrDefault(id, id); + schema = schemaForDeserialize(id, null, subject, isKey); } catch (RestClientException ex) { if (ex.getErrorCode() == 40403) { logger.debug("Trying to get id from Latest SchemaMetadata from schemaRegistry for subject {}", subject); @@ -145,20 +139,14 @@ protected Object deserialize(boolean includeSchemaAndVersion, String topic, Bool schemaMetadata = schemaRegistry.getLatestSchemaMetadata(subject); id = schemaMetadata.getId(); schema = new Schema.Parser().parse(schemaMetadata.getSchema()); - - if (subjectIdMap == null) { - subjectIdMap = new HashMap<>(); - oldToNewIdMap.put(subject, subjectIdMap); - } // keep a track of a subject's ids map so that subsequent records don't query the wrong schema id subjectIdMap.put(oldId, id); - logger.debug("success -> schemaMetadata.getId({}) for subject {}", id, subject); - } else { throw ex; } } + int length = buffer.limit() - 1 - idSize; final Object result; if (schema.getType().equals(Schema.Type.BYTES)) { @@ -188,56 +176,35 @@ protected Object deserialize(boolean includeSchemaAndVersion, String topic, Bool // Converter to let a version provided by a Kafka Connect source take priority over the // schema registry's ordering (which is implicit by auto-registration time rather than // explicit from the Connector). - Integer version = null; + Integer version; if (schemaMetadata != null) { version = schemaMetadata.getVersion(); } else { - Map schemaVersionMap = oldToNewVersionMap.get(subject); - if (schemaVersionMap != null) { - version = schemaVersionMap.get(schema); - } - try { - if (version == null) { - version = schemaRegistry.getVersion(subject, schema); - } - } catch (RestClientException e) { - if (e.getErrorCode() == 40403) { - logger.debug("Trying to get version from Latest SchemaMetadata from schemaRegistry for subject {}", subject); - schemaMetadata = schemaRegistry.getLatestSchemaMetadata(subject); - version = schemaMetadata.getVersion(); + Map schemaVersionMap = oldToNewVersionMap.computeIfAbsent(subject, sub -> new ConcurrentHashMap<>()); + version = schemaVersionMap.get(schema); - if (schemaVersionMap == null) { - schemaVersionMap = new IdentityHashMap<>(); - oldToNewVersionMap.put(subject, schemaVersionMap); - } + if (version == null) { + try { + version = schemaRegistry.getVersion(subject, schema); + } catch (RestClientException e) { + if (e.getErrorCode() == 40403) { + logger.debug("Trying to get version from Latest SchemaMetadata from schemaRegistry for subject {}", subject); + schemaMetadata = schemaRegistry.getLatestSchemaMetadata(subject); + version = schemaMetadata.getVersion(); - schemaVersionMap.put(schema, version); - logger.debug("success -> schemaMetadata.getVersion({}) for subject {}", version, subject); - } else { - throw e; + schemaVersionMap.putIfAbsent(schema, version); + logger.debug("success -> schemaMetadata.getVersion({}) for subject {}", version, subject); + } else { + throw e; + } } } } - - if (schema.getType() == Schema.Type.UNION) { - // Can't set additional properties on a union schema since it's just a list, so set it - // on the first non-null entry - for (Schema memberSchema : schema.getTypes()) { - if (memberSchema.getType() != Schema.Type.NULL) { - memberSchema.addProp(SCHEMA_REGISTRY_SCHEMA_VERSION_PROP, - JsonNodeFactory.instance.numberNode(version)); - break; - } - } - } else { - schema.addProp(SCHEMA_REGISTRY_SCHEMA_VERSION_PROP, - JsonNodeFactory.instance.numberNode(version)); - } if (schema.getType().equals(Schema.Type.RECORD)) { - return result; + return new GenericContainerWithVersion((GenericContainer) result, version); } else { - return new NonRecordContainer(schema, result); + return new GenericContainerWithVersion(new NonRecordContainer(schema, result), version); } } else { return result; @@ -250,6 +217,39 @@ protected Object deserialize(boolean includeSchemaAndVersion, String topic, Bool } } + private Integer schemaVersion(String topic, + Boolean isKey, + int id, + String subject, + Schema schema, + Object result) throws IOException, RestClientException { + Integer version; + if (isDeprecatedSubjectNameStrategy(isKey)) { + subject = getSubjectName(topic, isKey, result, schema); + Schema subjectSchema = schemaRegistry.getBySubjectAndId(subject, id); + version = schemaRegistry.getVersion(subject, subjectSchema); + } else { + //we already got the subject name + version = schemaRegistry.getVersion(subject, schema); + } + return version; + } + + private String subjectName(String topic, Boolean isKey, Schema schemaFromRegistry) { + return isDeprecatedSubjectNameStrategy(isKey) + ? null + : getSubjectName(topic, isKey, null, schemaFromRegistry); + } + + private Schema schemaForDeserialize(int id, + Schema schemaFromRegistry, + String subject, + Boolean isKey) throws IOException, RestClientException { + return isDeprecatedSubjectNameStrategy(isKey) + ? AvroSchemaUtils.copyOf(schemaFromRegistry) + : schemaRegistry.getBySubjectAndId(subject, id); + } + /** * Deserializes the payload and includes schema information, with version information from the * schema registry embedded in the schema. @@ -258,14 +258,15 @@ protected Object deserialize(boolean includeSchemaAndVersion, String topic, Bool * @return a GenericContainer with the schema and data, either as a {@link NonRecordContainer}, * {@link org.apache.avro.generic.GenericRecord}, or {@link SpecificRecord} */ - protected GenericContainer deserializeWithSchemaAndVersion(String topic, boolean isKey, + protected GenericContainerWithVersion deserializeWithSchemaAndVersion(String topic, boolean isKey, byte[] payload) throws SerializationException { - return (GenericContainer) deserialize(true, topic, isKey, payload, null); + return (GenericContainerWithVersion) deserialize(true, topic, isKey, payload, null); } private DatumReader getDatumReader(Schema writerSchema, Schema readerSchema) { - boolean writerSchemaIsPrimitive = getPrimitiveSchemas().values().contains(writerSchema); + boolean writerSchemaIsPrimitive = + AvroSchemaUtils.getPrimitiveSchemas().values().contains(writerSchema); // do not use SpecificDatumReader if writerSchema is a primitive if (useSpecificAvroReader && !writerSchemaIsPrimitive) { if (readerSchema == null) { @@ -309,5 +310,4 @@ private Schema getReaderSchema(Schema writerSchema) { } return readerSchema; } - } diff --git a/radar-commons/build.gradle b/radar-commons/build.gradle index 8bfbc9a5..c45e0791 100644 --- a/radar-commons/build.gradle +++ b/radar-commons/build.gradle @@ -1,5 +1,4 @@ -ext.moduleName = 'radar-commons' -ext.description = 'RADAR Common utilities library.' +description = 'RADAR Common utilities library.' targetCompatibility = '1.7' sourceCompatibility = '1.7' @@ -9,11 +8,9 @@ sourceCompatibility = '1.7' //---------------------------------------------------------------------------// configurations { - codacy -} - -configurations.compile { - resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' + implementation { + resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' + } } // In this section you declare where to find the dependencies of your project @@ -33,8 +30,7 @@ dependencies { // to implement producers and consumers api group: 'com.squareup.okhttp3', name: 'okhttp', version: okhttpVersion - - implementation group: 'org.json', name: 'json', version: orgJsonVersion + api group: 'org.json', name: 'json', version: orgJsonVersion // The production code uses the SLF4J logging API at compile time implementation group: 'org.slf4j', name:'slf4j-api', version: slf4jVersion @@ -45,132 +41,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: okhttpVersion testRuntime group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion - - codacy group: 'com.github.codacy', name: 'codacy-coverage-reporter', version: codacyVersion -} - -//---------------------------------------------------------------------------// -// Testing // -//---------------------------------------------------------------------------// -checkstyle { - // ignore tests - sourceSets = [sourceSets.main] -} - -pmd { - // ignore tests - sourceSets = [sourceSets.main] -} - -jacocoTestReport { - reports { - xml.enabled true - csv.enabled false - html.enabled false - } -} - -task downloadDependencies(type: Exec) { - configurations.testRuntime.files - configurations.codacy.files - configurations.jacocoAnt.files - commandLine 'echo', 'Downloaded all dependencies' -} - -task sendCoverageToCodacy(type: JavaExec, dependsOn: jacocoTestReport) { - main = 'com.codacy.CodacyCoverageReporter' - classpath = configurations.codacy - args = ['-l', 'Java', '-r', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml"] -} - -//---------------------------------------------------------------------------// -// Build system metadata // -//---------------------------------------------------------------------------// - -ext.sharedManifest = manifest { - attributes("Implementation-Title": moduleName, - "Implementation-Version": version) -} - -jar { - manifest.from sharedManifest -} -// custom tasks for creating source/javadoc jars -task sourcesJar(type: Jar, dependsOn: classes) { - classifier = 'sources' - from sourceSets.main.allSource - manifest.from sharedManifest -} - -task javadocJar(type: Jar, dependsOn: javadoc) { - classifier = 'javadoc' - from javadoc.destinationDir - manifest.from sharedManifest -} - -// add javadoc/source jar tasks as artifacts -artifacts { - archives javadocJar - archives sourcesJar -} - -publishing { - publications { - RadarCommonsPublication(MavenPublication) { - from components.java - artifact sourcesJar - artifact javadocJar - groupId project.group - artifactId moduleName - version project.version - pom.withXml { - def root = asNode() - root.appendNode('description', description) - root.appendNode('name', moduleName) - root.appendNode('url', githubUrl) - root.children().last() + pomConfig - } - } - } -} - -bintray { - user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - override false - publications 'RadarCommonsPublication' - pkg { - repo = project.group - name = moduleName - userOrg = 'radar-cns' - desc = description - licenses = ['Apache-2.0'] - websiteUrl = website - issueTrackerUrl = issueUrl - vcsUrl = githubUrl - githubRepo = githubRepoName - githubReleaseNotesFile = 'README.md' - version { - name = project.version - desc = description - vcsTag = System.getenv('TRAVIS_TAG') - released = new Date() - } - } -} - -artifactory { - contextUrl = 'https://oss.jfrog.org/artifactory' - publish { - repository { - repoKey = 'oss-snapshot-local' - username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - maven = true - } - } } -artifactoryPublish { - publications('RadarCommonsPublication') -} \ No newline at end of file +apply from: '../gradle/publishing.gradle' +apply from: '../gradle/codestyle.gradle' diff --git a/radar-commons/src/main/java/org/radarcns/config/ServerConfig.java b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.java similarity index 99% rename from radar-commons/src/main/java/org/radarcns/config/ServerConfig.java rename to radar-commons/src/main/java/org/radarbase/config/ServerConfig.java index 36196d02..5ad6b177 100644 --- a/radar-commons/src/main/java/org/radarcns/config/ServerConfig.java +++ b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.config; +package org.radarbase.config; import java.net.InetSocketAddress; import java.net.MalformedURLException; diff --git a/radar-commons/src/main/java/org/radarcns/data/AvroDatumDecoder.java b/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.java similarity index 99% rename from radar-commons/src/main/java/org/radarcns/data/AvroDatumDecoder.java rename to radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.java index 84fcba4f..21417ec1 100644 --- a/radar-commons/src/main/java/org/radarcns/data/AvroDatumDecoder.java +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/radar-commons/src/main/java/org/radarcns/data/AvroDatumEncoder.java b/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.java similarity index 98% rename from radar-commons/src/main/java/org/radarcns/data/AvroDatumEncoder.java rename to radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.java index 22b154e7..fc364a05 100644 --- a/radar-commons/src/main/java/org/radarcns/data/AvroDatumEncoder.java +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import java.io.IOException; import org.apache.avro.Schema; diff --git a/radar-commons/src/main/java/org/radarcns/data/AvroDecoder.java b/radar-commons/src/main/java/org/radarbase/data/AvroDecoder.java similarity index 98% rename from radar-commons/src/main/java/org/radarcns/data/AvroDecoder.java rename to radar-commons/src/main/java/org/radarbase/data/AvroDecoder.java index f2027fa7..bf411098 100644 --- a/radar-commons/src/main/java/org/radarcns/data/AvroDecoder.java +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDecoder.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import java.io.IOException; import org.apache.avro.Schema; diff --git a/radar-commons/src/main/java/org/radarcns/data/AvroEncoder.java b/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.java similarity index 96% rename from radar-commons/src/main/java/org/radarcns/data/AvroEncoder.java rename to radar-commons/src/main/java/org/radarbase/data/AvroEncoder.java index 513fa1a4..fcfc9fab 100644 --- a/radar-commons/src/main/java/org/radarcns/data/AvroEncoder.java +++ b/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.java @@ -14,12 +14,12 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.SchemaValidationException; -import org.radarcns.producer.rest.ParsedSchemaMetadata; +import org.radarbase.producer.rest.ParsedSchemaMetadata; /** Encode Avro values with a given encoder. The encoder may take into account the schema * that the schema registry has listed for a given topic. */ diff --git a/radar-commons/src/main/java/org/radarcns/data/AvroRecordData.java b/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.java similarity index 91% rename from radar-commons/src/main/java/org/radarcns/data/AvroRecordData.java rename to radar-commons/src/main/java/org/radarbase/data/AvroRecordData.java index 95e9a517..2451f4be 100644 --- a/radar-commons/src/main/java/org/radarcns/data/AvroRecordData.java +++ b/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.java @@ -1,9 +1,9 @@ -package org.radarcns.data; +package org.radarbase.data; import java.util.Iterator; import java.util.List; import java.util.Objects; -import org.radarcns.topic.AvroTopic; +import org.radarbase.topic.AvroTopic; /** * Avro record data belonging to a single key. @@ -32,9 +32,7 @@ public AvroRecordData(AvroTopic topic, K key, List values) { } } - /** - * Get the topic that the keys belong to. - */ + @Override public AvroTopic getTopic() { return topic; } diff --git a/radar-commons/src/main/java/org/radarcns/data/AvroRecordWriter.java b/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.java similarity index 94% rename from radar-commons/src/main/java/org/radarcns/data/AvroRecordWriter.java rename to radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.java index 77accd14..bec2b3ba 100644 --- a/radar-commons/src/main/java/org/radarcns/data/AvroRecordWriter.java +++ b/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -22,8 +22,8 @@ import org.apache.avro.io.DatumWriter; import org.apache.avro.io.Encoder; import org.apache.avro.io.EncoderFactory; -import org.radarcns.data.AvroEncoder.AvroWriter; -import org.radarcns.producer.rest.ParsedSchemaMetadata; +import org.radarbase.producer.rest.ParsedSchemaMetadata; +import org.radarbase.data.AvroEncoder.AvroWriter; /** * Encodes Avro records to bytes. diff --git a/radar-commons/src/main/java/org/radarcns/data/Record.java b/radar-commons/src/main/java/org/radarbase/data/Record.java similarity index 97% rename from radar-commons/src/main/java/org/radarcns/data/Record.java rename to radar-commons/src/main/java/org/radarbase/data/Record.java index d6cc4739..91f526a5 100644 --- a/radar-commons/src/main/java/org/radarcns/data/Record.java +++ b/radar-commons/src/main/java/org/radarbase/data/Record.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; /** * A single data record. diff --git a/radar-commons/src/main/java/org/radarcns/data/RecordData.java b/radar-commons/src/main/java/org/radarbase/data/RecordData.java similarity index 90% rename from radar-commons/src/main/java/org/radarcns/data/RecordData.java rename to radar-commons/src/main/java/org/radarbase/data/RecordData.java index 7b1cf6d0..54b0cb5c 100644 --- a/radar-commons/src/main/java/org/radarcns/data/RecordData.java +++ b/radar-commons/src/main/java/org/radarbase/data/RecordData.java @@ -1,6 +1,6 @@ -package org.radarcns.data; +package org.radarbase.data; -import org.radarcns.topic.AvroTopic; +import org.radarbase.topic.AvroTopic; /** * Record data belonging to a single key. diff --git a/radar-commons/src/main/java/org/radarcns/data/RemoteSchemaEncoder.java b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java similarity index 93% rename from radar-commons/src/main/java/org/radarcns/data/RemoteSchemaEncoder.java rename to radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java index e9934279..c8f40ba0 100644 --- a/radar-commons/src/main/java/org/radarcns/data/RemoteSchemaEncoder.java +++ b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java @@ -1,4 +1,4 @@ -package org.radarcns.data; +package org.radarbase.data; import java.io.IOException; import org.apache.avro.Schema; @@ -6,9 +6,9 @@ import org.apache.avro.generic.GenericData; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificRecord; -import org.radarcns.producer.rest.AvroDataMapper; -import org.radarcns.producer.rest.AvroDataMapperFactory; -import org.radarcns.producer.rest.ParsedSchemaMetadata; +import org.radarbase.producer.rest.AvroDataMapper; +import org.radarbase.producer.rest.AvroDataMapperFactory; +import org.radarbase.producer.rest.ParsedSchemaMetadata; /** * Encodes data according to an Avro schema to the format and schema of the server. diff --git a/radar-commons/src/main/java/org/radarcns/data/StringEncoder.java b/radar-commons/src/main/java/org/radarbase/data/StringEncoder.java similarity index 96% rename from radar-commons/src/main/java/org/radarcns/data/StringEncoder.java rename to radar-commons/src/main/java/org/radarbase/data/StringEncoder.java index 09e24e3b..0ebc4cd2 100644 --- a/radar-commons/src/main/java/org/radarcns/data/StringEncoder.java +++ b/radar-commons/src/main/java/org/radarbase/data/StringEncoder.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import java.io.IOException; import org.apache.avro.Schema; @@ -22,7 +22,7 @@ import org.apache.avro.SchemaValidationException; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.ObjectWriter; -import org.radarcns.producer.rest.ParsedSchemaMetadata; +import org.radarbase.producer.rest.ParsedSchemaMetadata; /** Encodes a String as Avro. */ public class StringEncoder implements AvroEncoder, AvroEncoder.AvroWriter { diff --git a/radar-commons/src/main/java/org/radarcns/data/TimedInt.java b/radar-commons/src/main/java/org/radarbase/data/TimedInt.java similarity index 98% rename from radar-commons/src/main/java/org/radarcns/data/TimedInt.java rename to radar-commons/src/main/java/org/radarbase/data/TimedInt.java index 98322158..7c065bba 100644 --- a/radar-commons/src/main/java/org/radarcns/data/TimedInt.java +++ b/radar-commons/src/main/java/org/radarbase/data/TimedInt.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; diff --git a/radar-commons/src/main/java/org/radarcns/producer/AuthenticationException.java b/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.java similarity index 92% rename from radar-commons/src/main/java/org/radarcns/producer/AuthenticationException.java rename to radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.java index 858cccf9..dcc81fab 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/AuthenticationException.java +++ b/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer; +package org.radarbase.producer; import java.io.IOException; @@ -22,6 +22,8 @@ * Failed to authenticate to server. */ public class AuthenticationException extends IOException { + private static final long serialVersionUID = 1; + public AuthenticationException() { super(); } diff --git a/radar-commons/src/main/java/org/radarcns/producer/BatchedKafkaSender.java b/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java similarity index 97% rename from radar-commons/src/main/java/org/radarcns/producer/BatchedKafkaSender.java rename to radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java index 4d1ed47a..6e2fedce 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/BatchedKafkaSender.java +++ b/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer; +package org.radarbase.producer; import java.io.IOException; import java.util.ArrayList; @@ -22,9 +22,9 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import org.apache.avro.SchemaValidationException; -import org.radarcns.data.AvroRecordData; -import org.radarcns.data.RecordData; -import org.radarcns.topic.AvroTopic; +import org.radarbase.data.AvroRecordData; +import org.radarbase.data.RecordData; +import org.radarbase.topic.AvroTopic; /** * A Kafka REST Proxy sender that batches up records. It will send data once the batch size is diff --git a/radar-commons/src/main/java/org/radarcns/producer/KafkaSender.java b/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.java similarity index 96% rename from radar-commons/src/main/java/org/radarcns/producer/KafkaSender.java rename to radar-commons/src/main/java/org/radarbase/producer/KafkaSender.java index b8ca6ed6..f5e3ac70 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/KafkaSender.java +++ b/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.java @@ -14,12 +14,12 @@ * limitations under the License. */ -package org.radarcns.producer; +package org.radarbase.producer; import java.io.Closeable; import java.io.IOException; import org.apache.avro.SchemaValidationException; -import org.radarcns.topic.AvroTopic; +import org.radarbase.topic.AvroTopic; /** * Thread-safe sender. Calling {@link #close()} must be done after all {@link KafkaTopicSender} diff --git a/radar-commons/src/main/java/org/radarcns/producer/KafkaTopicSender.java b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.java similarity index 95% rename from radar-commons/src/main/java/org/radarcns/producer/KafkaTopicSender.java rename to radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.java index 7c7ebd63..21e26a32 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/KafkaTopicSender.java +++ b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.java @@ -1,9 +1,9 @@ -package org.radarcns.producer; +package org.radarbase.producer; import java.io.Closeable; import java.io.IOException; import org.apache.avro.SchemaValidationException; -import org.radarcns.data.RecordData; +import org.radarbase.data.RecordData; /** * Sender for a single topic. Should be created through a {@link KafkaSender}. diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/AvroDataMapper.java b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.java similarity index 91% rename from radar-commons/src/main/java/org/radarcns/producer/rest/AvroDataMapper.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.java index 710aecc8..f2027223 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/AvroDataMapper.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.java @@ -1,4 +1,4 @@ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import org.apache.avro.Schema; diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/AvroDataMapperFactory.java b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java similarity index 99% rename from radar-commons/src/main/java/org/radarcns/producer/rest/AvroDataMapperFactory.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java index c3d7c75e..5f8e0a09 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/AvroDataMapperFactory.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java @@ -1,4 +1,4 @@ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import static org.apache.avro.JsonProperties.NULL_VALUE; @@ -18,8 +18,8 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecordBuilder; import org.apache.avro.generic.IndexedRecord; -import org.radarcns.util.Base64; -import org.radarcns.util.Base64.Encoder; +import org.radarbase.util.Base64; +import org.radarbase.util.Base64.Encoder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java similarity index 92% rename from radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java index 8b1df3ae..89b53ccc 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/BinaryRecordRequest.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java @@ -14,9 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; - -import static org.radarcns.util.Strings.bytesToHex; +package org.radarbase.producer.rest; import java.io.IOException; import okio.Buffer; @@ -26,10 +24,11 @@ import org.apache.avro.generic.IndexedRecord; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.EncoderFactory; -import org.radarcns.data.AvroEncoder.AvroWriter; -import org.radarcns.data.RecordData; -import org.radarcns.data.RemoteSchemaEncoder; -import org.radarcns.topic.AvroTopic; +import org.radarbase.data.AvroEncoder.AvroWriter; +import org.radarbase.data.RecordData; +import org.radarbase.data.RemoteSchemaEncoder; +import org.radarbase.topic.AvroTopic; +import org.radarbase.util.Strings; /** * Encodes a record request as binary data, in the form of a RecordSet. @@ -127,6 +126,7 @@ public void prepare(ParsedSchemaMetadata keySchema, ParsedSchemaMetadata valueSc public String content(int maxLength) throws IOException { Buffer buffer = new Buffer(); writeToSink(buffer, maxLength / 2 - 2); - return "0x" + bytesToHex(buffer.readByteArray(Math.min(buffer.size(), maxLength - 2))); + return "0x" + Strings.bytesToHex( + buffer.readByteArray(Math.min(buffer.size(), maxLength - 2))); } } diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/ConnectionState.java b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.java similarity index 98% rename from radar-commons/src/main/java/org/radarcns/producer/rest/ConnectionState.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.java index 6a5bdc0c..cb911f07 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/ConnectionState.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import java.util.concurrent.TimeUnit; diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/GzipRequestInterceptor.java b/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java similarity index 98% rename from radar-commons/src/main/java/org/radarcns/producer/rest/GzipRequestInterceptor.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java index cd5b2e08..a12c9011 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/GzipRequestInterceptor.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import java.io.IOException; import okhttp3.Interceptor; diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/JsonRecordRequest.java b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java similarity index 81% rename from radar-commons/src/main/java/org/radarcns/producer/rest/JsonRecordRequest.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java index fbd13b3f..1d47ad99 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/JsonRecordRequest.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java @@ -14,9 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; - -import static org.radarcns.util.Strings.utf8; +package org.radarbase.producer.rest; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -24,21 +22,22 @@ import okio.BufferedSink; import org.apache.avro.SchemaValidationException; import org.json.JSONObject; -import org.radarcns.data.AvroEncoder.AvroWriter; -import org.radarcns.data.RecordData; -import org.radarcns.data.RemoteSchemaEncoder; -import org.radarcns.topic.AvroTopic; +import org.radarbase.data.AvroEncoder.AvroWriter; +import org.radarbase.data.RecordData; +import org.radarbase.data.RemoteSchemaEncoder; +import org.radarbase.topic.AvroTopic; +import org.radarbase.util.Strings; /** * Request data to submit records to the Kafka REST proxy. */ public class JsonRecordRequest implements RecordRequest { - public static final byte[] KEY_SCHEMA_ID = utf8("\"key_schema_id\":"); - public static final byte[] VALUE_SCHEMA_ID = utf8(",\"value_schema_id\":"); - public static final byte[] RECORDS = utf8(",\"records\":["); - public static final byte[] KEY = utf8("{\"key\":"); - public static final byte[] VALUE = utf8(",\"value\":"); - public static final byte[] END = utf8("]}"); + public static final byte[] KEY_SCHEMA_ID = Strings.utf8("\"key_schema_id\":"); + public static final byte[] VALUE_SCHEMA_ID = Strings.utf8(",\"value_schema_id\":"); + public static final byte[] RECORDS = Strings.utf8(",\"records\":["); + public static final byte[] KEY = Strings.utf8("{\"key\":"); + public static final byte[] VALUE = Strings.utf8(",\"value\":"); + public static final byte[] END = Strings.utf8("]}"); private final AvroWriter keyEncoder; private final AvroWriter valueEncoder; @@ -72,9 +71,9 @@ public void writeToSink(BufferedSink sink) throws IOException { private void writeToSink(BufferedSink sink, int maxLength) throws IOException { sink.writeByte('{'); sink.write(KEY_SCHEMA_ID); - sink.write(utf8(String.valueOf(keyEncoder.getReaderSchema().getId()))); + sink.write(Strings.utf8(String.valueOf(keyEncoder.getReaderSchema().getId()))); sink.write(VALUE_SCHEMA_ID); - sink.write(utf8(String.valueOf(valueEncoder.getReaderSchema().getId()))); + sink.write(Strings.utf8(String.valueOf(valueEncoder.getReaderSchema().getId()))); sink.write(RECORDS); diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/ParsedSchemaMetadata.java b/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.java similarity index 97% rename from radar-commons/src/main/java/org/radarcns/producer/rest/ParsedSchemaMetadata.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.java index 18e498c0..d0009e72 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/ParsedSchemaMetadata.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import org.apache.avro.Schema; diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/RecordRequest.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.java similarity index 96% rename from radar-commons/src/main/java/org/radarcns/producer/rest/RecordRequest.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.java index 0095a51f..5c737f35 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/RecordRequest.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.java @@ -14,12 +14,12 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import java.io.IOException; import okio.BufferedSink; import org.apache.avro.SchemaValidationException; -import org.radarcns.data.RecordData; +import org.radarbase.data.RecordData; /** * Record request contents. Before {@link #writeToSink(BufferedSink)} is called, first diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/RestClient.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java similarity index 91% rename from radar-commons/src/main/java/org/radarcns/producer/rest/RestClient.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java index 37219358..24cd6d4b 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/RestClient.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java @@ -14,14 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; - -import static org.radarcns.util.RestUtils.DEFAULT_HOSTNAME_VERIFIER; -import static org.radarcns.util.RestUtils.UNSAFE_HOSTNAME_VERIFIER; -import static org.radarcns.util.RestUtils.UNSAFE_SSL_FACTORY; -import static org.radarcns.util.RestUtils.UNSAFE_TRUST_MANAGER; -import static org.radarcns.util.RestUtils.systemDefaultSslSocketFactory; -import static org.radarcns.util.RestUtils.systemDefaultTrustManager; +package org.radarbase.producer.rest; import java.io.IOException; import java.lang.ref.WeakReference; @@ -39,7 +32,8 @@ import okhttp3.Request; import okhttp3.Response; import okhttp3.ResponseBody; -import org.radarcns.config.ServerConfig; +import org.radarbase.config.ServerConfig; +import org.radarbase.util.RestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -221,14 +215,15 @@ public Builder server(ServerConfig config) { this.serverConfig = Objects.requireNonNull(config); if (config.isUnsafe()) { - this.client.sslSocketFactory(UNSAFE_SSL_FACTORY, - (X509TrustManager) UNSAFE_TRUST_MANAGER[0]); - this.client.hostnameVerifier(UNSAFE_HOSTNAME_VERIFIER); + this.client.sslSocketFactory(RestUtils.UNSAFE_SSL_FACTORY, + (X509TrustManager) RestUtils.UNSAFE_TRUST_MANAGER[0]); + this.client.hostnameVerifier(RestUtils.UNSAFE_HOSTNAME_VERIFIER); } else { - X509TrustManager trustManager = systemDefaultTrustManager(); - SSLSocketFactory socketFactory = systemDefaultSslSocketFactory(trustManager); + X509TrustManager trustManager = RestUtils.systemDefaultTrustManager(); + SSLSocketFactory socketFactory = RestUtils.systemDefaultSslSocketFactory( + trustManager); this.client.sslSocketFactory(socketFactory, trustManager); - this.client.hostnameVerifier(DEFAULT_HOSTNAME_VERIFIER); + this.client.hostnameVerifier(RestUtils.DEFAULT_HOSTNAME_VERIFIER); } return this; } diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/RestException.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.java similarity index 94% rename from radar-commons/src/main/java/org/radarcns/producer/rest/RestException.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/RestException.java index f3d373d2..0c8429f6 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/RestException.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import java.io.IOException; @@ -22,6 +22,8 @@ * Exception when a HTTP REST request fails. */ public class RestException extends IOException { + private static final long serialVersionUID = 1; + private final int statusCode; private final String body; diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/RestSender.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.java similarity index 86% rename from radar-commons/src/main/java/org/radarcns/producer/rest/RestSender.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.java index 4c01bfb0..59556f90 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/RestSender.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.java @@ -14,10 +14,10 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; -import static org.radarcns.producer.rest.RestClient.DEFAULT_TIMEOUT; -import static org.radarcns.producer.rest.RestClient.responseBody; +import static org.radarbase.producer.rest.RestClient.DEFAULT_TIMEOUT; +import static org.radarbase.producer.rest.RestClient.responseBody; import java.io.IOException; import java.net.MalformedURLException; @@ -30,12 +30,12 @@ import okhttp3.Request; import okhttp3.Response; import org.apache.avro.SchemaValidationException; -import org.radarcns.config.ServerConfig; -import org.radarcns.producer.AuthenticationException; -import org.radarcns.producer.KafkaSender; -import org.radarcns.producer.KafkaTopicSender; -import org.radarcns.producer.rest.ConnectionState.State; -import org.radarcns.topic.AvroTopic; +import org.radarbase.config.ServerConfig; +import org.radarbase.producer.AuthenticationException; +import org.radarbase.producer.KafkaSender; +import org.radarbase.producer.KafkaTopicSender; +import org.radarbase.producer.rest.ConnectionState.State; +import org.radarbase.topic.AvroTopic; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,7 +59,7 @@ public class RestSender implements KafkaSender { MediaType.parse("application/vnd.kafka.avro.v1+json; charset=utf-8"); private RequestProperties requestProperties; - private Request.Builder isConnectedRequest; + private Request.Builder connectionTestRequest; private SchemaRetriever schemaRetriever; private RestClient httpClient; private final ConnectionState state; @@ -110,7 +110,7 @@ public synchronized void setKafkaConfig(ServerConfig kafkaConfig) { */ private void setRestClient(RestClient newClient) { try { - isConnectedRequest = newClient.requestBuilder("").head(); + connectionTestRequest = newClient.requestBuilder("").head(); } catch (MalformedURLException ex) { throw new IllegalArgumentException("Schemaless topics do not have a valid URL", ex); } @@ -134,8 +134,8 @@ public synchronized SchemaRetriever getSchemaRetriever() { } /** Get a request to check the connection status. */ - private synchronized Request getIsConnectedRequest() { - return isConnectedRequest.headers(requestProperties.headers).build(); + private synchronized Request getConnectionTestRequest() { + return connectionTestRequest.headers(requestProperties.headers).build(); } /** Set the compression of the REST client. */ @@ -181,7 +181,7 @@ public boolean resetConnection() throws AuthenticationException { if (state.getState() == State.CONNECTED) { return true; } - try (Response response = httpClient.request(getIsConnectedRequest())) { + try (Response response = httpClient.request(getConnectionTestRequest())) { if (response.isSuccessful()) { state.didConnect(); } else if (response.code() == 401) { @@ -253,6 +253,24 @@ public Builder schemaRetriever(SchemaRetriever schemaRetriever) { return this; } + /** + * Whether to try to send binary content. This only works if the server supports it. If not, + * there may be an additional round-trip. + * @param binary true if attempt to send binary content, false otherwise + */ + public Builder useBinaryContent(boolean binary) { + this.binary = binary; + return this; + } + + /** + * Whether to try to send binary content. This only works if the server supports it. If not, + * there may be an additional round-trip. + * @param binary true if attempt to send binary content, false otherwise + * @deprecated use {@link #useBinaryContent(boolean)} instead + */ + @Deprecated + @SuppressWarnings("PMD.LinguisticNaming") public Builder hasBinaryContent(boolean binary) { this.binary = binary; return this; @@ -274,7 +292,7 @@ public Builder headers(Headers headers) { } public Builder addHeader(String header, String value) { - additionalHeaders.add(header, value); + additionalHeaders.add(header + ": " + value); return this; } diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.java similarity index 84% rename from radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.java index 7e14b244..93656210 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/RestTopicSender.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.java @@ -14,15 +14,10 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; -import static org.radarcns.producer.rest.RestClient.responseBody; -import static org.radarcns.producer.rest.RestSender.KAFKA_REST_ACCEPT_ENCODING; -import static org.radarcns.producer.rest.RestSender.KAFKA_REST_ACCEPT_LEGACY_ENCODING; -import static org.radarcns.producer.rest.RestSender.KAFKA_REST_AVRO_ENCODING; -import static org.radarcns.producer.rest.RestSender.KAFKA_REST_AVRO_LEGACY_ENCODING; -import static org.radarcns.producer.rest.RestSender.KAFKA_REST_BINARY_ENCODING; -import static org.radarcns.producer.rest.UncheckedRequestException.fail; +import static org.radarbase.producer.rest.RestClient.responseBody; +import static org.radarbase.producer.rest.UncheckedRequestException.fail; import java.io.IOException; import java.util.Collections; @@ -33,11 +28,12 @@ import okhttp3.RequestBody; import okhttp3.Response; import org.apache.avro.SchemaValidationException; -import org.radarcns.data.AvroRecordData; -import org.radarcns.data.RecordData; -import org.radarcns.producer.AuthenticationException; -import org.radarcns.producer.KafkaTopicSender; -import org.radarcns.topic.AvroTopic; +import org.json.JSONException; +import org.radarbase.data.AvroRecordData; +import org.radarbase.data.RecordData; +import org.radarbase.producer.AuthenticationException; +import org.radarbase.producer.KafkaTopicSender; +import org.radarbase.topic.AvroTopic; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,7 +134,7 @@ private void updateRecords(RestSender.RequestContext context, RecordData r sendTopic, false, topic.getKeySchema(), -1); valueMetadata = retriever.getOrSetSchemaMetadata( sendTopic, true, topic.getValueSchema(), -1); - } catch (IOException ex) { + } catch (JSONException | IOException ex) { throw new IOException("Failed to get schemas for topic " + topic, ex); } @@ -150,21 +146,24 @@ private void downgradeConnection(Request request, Response response) throws IOEx state.didConnect(); logger.warn("Binary Avro encoding is not supported." + " Switching to JSON encoding."); - sender.useLegacyEncoding(KAFKA_REST_ACCEPT_ENCODING, KAFKA_REST_AVRO_ENCODING, + sender.useLegacyEncoding( + RestSender.KAFKA_REST_ACCEPT_ENCODING, RestSender.KAFKA_REST_AVRO_ENCODING, false); requestData = new JsonRecordRequest<>(topic); } else if (Objects.equals(request.header("Accept"), - KAFKA_REST_ACCEPT_ENCODING)) { + RestSender.KAFKA_REST_ACCEPT_ENCODING)) { state.didConnect(); logger.warn("Latest Avro encoding is not supported. Switching to legacy " + "encoding."); sender.useLegacyEncoding( - KAFKA_REST_ACCEPT_LEGACY_ENCODING, KAFKA_REST_AVRO_LEGACY_ENCODING, + RestSender.KAFKA_REST_ACCEPT_LEGACY_ENCODING, + RestSender.KAFKA_REST_AVRO_LEGACY_ENCODING, false); } else { RequestBody body = request.body(); MediaType contentType = body != null ? body.contentType() : null; - if (contentType == null || contentType.equals(KAFKA_REST_AVRO_LEGACY_ENCODING)) { + if (contentType == null + || contentType.equals(RestSender.KAFKA_REST_AVRO_LEGACY_ENCODING)) { throw fail(request, response, new IOException("Content-Type " + contentType + " not accepted by server.")); } else { @@ -188,9 +187,9 @@ private Request buildRequest(RestSender.RequestContext context, RecordData .header("Accept", context.properties.acceptType); MediaType contentType = context.properties.contentType; - if (contentType.equals(KAFKA_REST_BINARY_ENCODING) + if (contentType.equals(RestSender.KAFKA_REST_BINARY_ENCODING) && !(requestData instanceof BinaryRecordRequest)) { - contentType = KAFKA_REST_AVRO_ENCODING; + contentType = RestSender.KAFKA_REST_AVRO_ENCODING; } requestBody = new TopicRequestBody(requestData, contentType); diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java similarity index 92% rename from radar-commons/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java index 1a4b421b..7c9f83e4 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/SchemaRetriever.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java @@ -14,9 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; - -import static org.radarcns.util.Strings.utf8; +package org.radarbase.producer.rest; import java.io.IOException; import java.util.HashMap; @@ -32,8 +30,10 @@ import org.apache.avro.Schema; import org.apache.avro.Schema.Type; import org.apache.avro.generic.GenericContainer; +import org.json.JSONException; import org.json.JSONObject; -import org.radarcns.config.ServerConfig; +import org.radarbase.config.ServerConfig; +import org.radarbase.util.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +47,7 @@ public class SchemaRetriever { "application/vnd.schemaregistry.v1+json; charset=utf-8"); private static final Schema NULL_SCHEMA = Schema.create(Type.NULL); private static final Map, Schema> PRIMITIVE_SCHEMAS = new HashMap<>(); - private static final byte[] SCHEMA = utf8("{\"schema\":"); + private static final byte[] SCHEMA = Strings.utf8("{\"schema\":"); private static final long MAX_VALIDITY = 86400L; static { @@ -99,15 +99,18 @@ protected static String subject(String topic, boolean ofValue) { /** Retrieve schema metadata from server. */ protected ParsedSchemaMetadata retrieveSchemaMetadata(String subject, int version) - throws IOException { - String path = "/subjects/" + subject + "/versions/"; + throws JSONException, IOException { + StringBuilder pathBuilder = new StringBuilder(50) + .append("/subjects/") + .append(subject) + .append("/versions/"); if (version > 0) { - path += version; + pathBuilder.append(version); } else { - path += "latest"; + pathBuilder.append("latest"); } RestClient restClient = getRestClient(); - Request request = restClient.requestBuilder(path) + Request request = restClient.requestBuilder(pathBuilder.toString()) .addHeader("Accept", "application/json") .build(); @@ -121,7 +124,7 @@ protected ParsedSchemaMetadata retrieveSchemaMetadata(String subject, int versio /** Get schema metadata. Cached schema metadata will be used if present. */ public ParsedSchemaMetadata getSchemaMetadata(String topic, boolean ofValue, int version) - throws IOException { + throws JSONException, IOException { String subject = subject(topic, ofValue); TimedSchemaMetadata value = cache.get(subject); if (value == null || value.isExpired()) { @@ -141,7 +144,7 @@ protected Schema parseSchema(String schemaString) { * Add schema metadata to the retriever. This implementation only adds it to the cache. */ public void addSchemaMetadata(String topic, boolean ofValue, ParsedSchemaMetadata metadata) - throws IOException { + throws JSONException, IOException { String subject = subject(topic, ofValue); if (metadata.getId() == null) { RestClient restClient = getRestClient(); @@ -165,7 +168,7 @@ public void addSchemaMetadata(String topic, boolean ofValue, ParsedSchemaMetadat * @param version version to get or 0 if the latest version can be used. */ public ParsedSchemaMetadata getOrSetSchemaMetadata(String topic, boolean ofValue, Schema schema, - int version) throws IOException { + int version) throws JSONException, IOException { try { return getSchemaMetadata(topic, ofValue, version); } catch (IOException ex) { diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java b/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.java similarity index 97% rename from radar-commons/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.java index c6701951..43ed4fcd 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/TopicRequestBody.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import java.io.IOException; import okhttp3.MediaType; diff --git a/radar-commons/src/main/java/org/radarcns/producer/rest/UncheckedRequestException.java b/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.java similarity index 90% rename from radar-commons/src/main/java/org/radarcns/producer/rest/UncheckedRequestException.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.java index 582bf531..a1524427 100644 --- a/radar-commons/src/main/java/org/radarcns/producer/rest/UncheckedRequestException.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.java @@ -14,10 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; - -import static org.radarcns.producer.rest.RestClient.responseBody; -import static org.radarcns.producer.rest.TopicRequestBody.topicRequestContent; +package org.radarbase.producer.rest; import java.io.IOException; import okhttp3.Request; @@ -25,6 +22,7 @@ /** Unchecked exception for failures during request handling. */ public class UncheckedRequestException extends RuntimeException { + private static final long serialVersionUID = 1; private static final int LOG_CONTENT_LENGTH = 1024; /** @@ -69,12 +67,12 @@ public static UncheckedRequestException fail(Request request, message.append(" (HTTP status code ") .append(response.code()) .append(')'); - content = responseBody(response); + content = RestClient.responseBody(response); } else { content = null; } - String requestContent = topicRequestContent(request, LOG_CONTENT_LENGTH); + String requestContent = TopicRequestBody.topicRequestContent(request, LOG_CONTENT_LENGTH); if (requestContent != null || content != null) { message.append(':'); } diff --git a/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java b/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java similarity index 99% rename from radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java rename to radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java index 54933ff0..351098fa 100644 --- a/radar-commons/src/main/java/org/radarcns/topic/AvroTopic.java +++ b/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.topic; +package org.radarbase.topic; import java.lang.reflect.InvocationTargetException; import java.util.List; diff --git a/radar-commons/src/main/java/org/radarcns/topic/KafkaTopic.java b/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.java similarity index 98% rename from radar-commons/src/main/java/org/radarcns/topic/KafkaTopic.java rename to radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.java index d1b6fc17..61b40e26 100644 --- a/radar-commons/src/main/java/org/radarcns/topic/KafkaTopic.java +++ b/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.topic; +package org.radarbase.topic; import java.util.regex.Pattern; diff --git a/radar-commons/src/main/java/org/radarcns/topic/SensorTopic.java b/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.java similarity index 99% rename from radar-commons/src/main/java/org/radarcns/topic/SensorTopic.java rename to radar-commons/src/main/java/org/radarbase/topic/SensorTopic.java index 9feb1f62..b675dd19 100644 --- a/radar-commons/src/main/java/org/radarcns/topic/SensorTopic.java +++ b/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.topic; +package org.radarbase.topic; import org.apache.avro.Schema; import org.apache.avro.Schema.Type; diff --git a/radar-commons/src/main/java/org/radarcns/util/Base64.java b/radar-commons/src/main/java/org/radarbase/util/Base64.java similarity index 95% rename from radar-commons/src/main/java/org/radarcns/util/Base64.java rename to radar-commons/src/main/java/org/radarbase/util/Base64.java index 613de91a..60e6df86 100644 --- a/radar-commons/src/main/java/org/radarcns/util/Base64.java +++ b/radar-commons/src/main/java/org/radarbase/util/Base64.java @@ -23,7 +23,7 @@ * questions. */ -package org.radarcns.util; +package org.radarbase.util; import java.util.Arrays; @@ -49,6 +49,7 @@ * @since 1.8 */ +@SuppressWarnings("PMD.ClassNamingConventions") public class Base64 { private Base64() {} @@ -127,10 +128,10 @@ private int encode0(byte[] src, int end, byte[] dst) { while (sp < slen) { int sl0 = Math.min(sp + slen, slen); int dp0 = dp; - for (int sp0 = sp; sp0 < sl0; ) { - int bits = (src[sp0++] & 0xff) << 16 - | (src[sp0++] & 0xff) << 8 - | (src[sp0++] & 0xff); + for (int sp0 = sp; sp0 < sl0; sp0 += 3) { + int bits = (src[sp0] & 0xff) << 16 + | (src[sp0 + 1] & 0xff) << 8 + | (src[sp0 + 2] & 0xff); dst[dp0++] = BASE_64_BYTE[(bits >>> 18) & 0x3f]; dst[dp0++] = BASE_64_BYTE[(bits >>> 12) & 0x3f]; dst[dp0++] = BASE_64_BYTE[(bits >>> 6) & 0x3f]; diff --git a/radar-commons/src/main/java/org/radarcns/util/RestUtils.java b/radar-commons/src/main/java/org/radarbase/util/RestUtils.java similarity index 94% rename from radar-commons/src/main/java/org/radarcns/util/RestUtils.java rename to radar-commons/src/main/java/org/radarbase/util/RestUtils.java index 2cac3f5d..e26f82c7 100644 --- a/radar-commons/src/main/java/org/radarcns/util/RestUtils.java +++ b/radar-commons/src/main/java/org/radarbase/util/RestUtils.java @@ -14,9 +14,7 @@ * limitations under the License. */ -package org.radarcns.util; - -import static okhttp3.internal.Util.assertionError; +package org.radarbase.util; import java.security.GeneralSecurityException; import java.security.KeyManagementException; @@ -109,7 +107,7 @@ public static X509TrustManager systemDefaultTrustManager() { } return (X509TrustManager) trustManagers[0]; } catch (GeneralSecurityException e) { - throw assertionError("No System TLS", e); // The system has no TLS. Just give up. + throw new IllegalStateException("No System TLS", e); } } @@ -123,7 +121,7 @@ public static SSLSocketFactory systemDefaultSslSocketFactory(X509TrustManager tr sslContext.init(null, new TrustManager[] { trustManager }, null); return sslContext.getSocketFactory(); } catch (GeneralSecurityException e) { - throw assertionError("No System TLS", e); // The system has no TLS. Just give up. + throw new IllegalStateException("No System TLS", e); } } } diff --git a/radar-commons/src/main/java/org/radarcns/util/Serialization.java b/radar-commons/src/main/java/org/radarbase/util/Serialization.java similarity index 98% rename from radar-commons/src/main/java/org/radarcns/util/Serialization.java rename to radar-commons/src/main/java/org/radarbase/util/Serialization.java index a7307c4f..2726f21f 100644 --- a/radar-commons/src/main/java/org/radarcns/util/Serialization.java +++ b/radar-commons/src/main/java/org/radarbase/util/Serialization.java @@ -14,13 +14,14 @@ * limitations under the License. */ -package org.radarcns.util; +package org.radarbase.util; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; /** Serialization utility class. */ +@SuppressWarnings("PMD.ClassNamingConventions") public final class Serialization { private Serialization() { diff --git a/radar-commons/src/main/java/org/radarcns/util/Strings.java b/radar-commons/src/main/java/org/radarbase/util/Strings.java similarity index 97% rename from radar-commons/src/main/java/org/radarcns/util/Strings.java rename to radar-commons/src/main/java/org/radarbase/util/Strings.java index e084b162..72229cfe 100644 --- a/radar-commons/src/main/java/org/radarcns/util/Strings.java +++ b/radar-commons/src/main/java/org/radarbase/util/Strings.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.util; +package org.radarbase.util; import java.nio.charset.Charset; import java.util.Collection; @@ -24,6 +24,7 @@ /** * String utilities. */ +@SuppressWarnings("PMD.ClassNamingConventions") public final class Strings { private static final Charset UTF_8 = Charset.forName("UTF-8"); private static final char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray(); diff --git a/radar-commons/src/test/java/org/radarcns/data/AvroDatumDecoderTest.java b/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java similarity index 98% rename from radar-commons/src/test/java/org/radarcns/data/AvroDatumDecoderTest.java rename to radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java index 4601e26e..e19ec3e5 100644 --- a/radar-commons/src/test/java/org/radarcns/data/AvroDatumDecoderTest.java +++ b/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import static org.junit.Assert.assertEquals; @@ -23,7 +23,7 @@ import org.junit.Test; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; -import org.radarcns.topic.AvroTopic; +import org.radarbase.topic.AvroTopic; /** * Created by nivethika on 24-2-17. diff --git a/radar-commons/src/test/java/org/radarcns/data/AvroDatumEncoderTest.java b/radar-commons/src/test/java/org/radarbase/data/AvroDatumEncoderTest.java similarity index 98% rename from radar-commons/src/test/java/org/radarcns/data/AvroDatumEncoderTest.java rename to radar-commons/src/test/java/org/radarbase/data/AvroDatumEncoderTest.java index b3ac5fe0..410ea225 100644 --- a/radar-commons/src/test/java/org/radarcns/data/AvroDatumEncoderTest.java +++ b/radar-commons/src/test/java/org/radarbase/data/AvroDatumEncoderTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import java.io.IOException; import java.util.Arrays; @@ -22,10 +22,10 @@ import java.util.concurrent.ThreadLocalRandom; import junit.framework.TestCase; import org.apache.avro.specific.SpecificData; +import org.radarbase.topic.AvroTopic; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; import org.radarcns.passive.phone.PhoneAcceleration; -import org.radarcns.topic.AvroTopic; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/radar-commons/src/test/java/org/radarcns/data/StringEncoderTest.java b/radar-commons/src/test/java/org/radarbase/data/StringEncoderTest.java similarity index 97% rename from radar-commons/src/test/java/org/radarcns/data/StringEncoderTest.java rename to radar-commons/src/test/java/org/radarbase/data/StringEncoderTest.java index edcfd608..10cacf6b 100644 --- a/radar-commons/src/test/java/org/radarcns/data/StringEncoderTest.java +++ b/radar-commons/src/test/java/org/radarbase/data/StringEncoderTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.data; +package org.radarbase.data; import static org.junit.Assert.assertTrue; diff --git a/radar-commons/src/test/java/org/radarcns/producer/rest/AvroDataMapperFactoryTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.java similarity index 97% rename from radar-commons/src/test/java/org/radarcns/producer/rest/AvroDataMapperFactoryTest.java rename to radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.java index 402087a9..8d11e655 100644 --- a/radar-commons/src/test/java/org/radarcns/producer/rest/AvroDataMapperFactoryTest.java +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.java @@ -1,4 +1,4 @@ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import static org.junit.Assert.assertEquals; @@ -15,15 +15,13 @@ import org.junit.Before; import org.junit.Test; import org.radarcns.kafka.ObservationKey; -import org.radarcns.producer.rest.AvroDataMapper; -import org.radarcns.producer.rest.AvroDataMapperFactory; public class AvroDataMapperFactoryTest { private static final Schema MEASUREMENT_KEY_SCHEMA = new Schema.Parser().parse("{"+ " \"namespace\": \"org.radarcns.key\","+ " \"type\": \"record\","+ " \"name\": \"MeasurementKey\","+ - " \"doc\": \"Measurement key in the RADAR-CNS project\","+ + " \"doc\": \"Measurement key in the RADAR-base project\","+ " \"fields\": ["+ " {\"name\": \"userId\", \"type\": \"string\", \"doc\": \"user ID\"},"+ " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}"+ @@ -34,7 +32,7 @@ public class AvroDataMapperFactoryTest { " \"namespace\": \"org.radarcns.key\","+ " \"type\": \"record\","+ " \"name\": \"MeasurementKey\","+ - " \"doc\": \"Measurement key in the RADAR-CNS project\","+ + " \"doc\": \"Measurement key in the RADAR-base project\","+ " \"fields\": ["+ " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}"+ " ]"+ diff --git a/radar-commons/src/test/java/org/radarcns/producer/rest/BinaryRecordRequestTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.java similarity index 97% rename from radar-commons/src/test/java/org/radarcns/producer/rest/BinaryRecordRequestTest.java rename to radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.java index c50bca6c..36e888b8 100644 --- a/radar-commons/src/test/java/org/radarcns/producer/rest/BinaryRecordRequestTest.java +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.java @@ -1,4 +1,4 @@ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import static org.junit.Assert.assertArrayEquals; @@ -18,12 +18,12 @@ import org.apache.avro.io.EncoderFactory; import org.apache.avro.specific.SpecificDatumWriter; import org.junit.Test; -import org.radarcns.data.AvroRecordData; +import org.radarbase.data.AvroRecordData; +import org.radarbase.topic.AvroTopic; import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.RecordSet; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; import org.radarcns.passive.phone.PhoneAcceleration; -import org.radarcns.topic.AvroTopic; public class BinaryRecordRequestTest { diff --git a/radar-commons/src/test/java/org/radarcns/producer/rest/ConnectionStateTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.java similarity index 86% rename from radar-commons/src/test/java/org/radarcns/producer/rest/ConnectionStateTest.java rename to radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.java index e86b93d5..2dd6f271 100644 --- a/radar-commons/src/test/java/org/radarcns/producer/rest/ConnectionStateTest.java +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.java @@ -14,12 +14,12 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import static org.junit.Assert.assertEquals; -import static org.radarcns.producer.rest.ConnectionState.State.CONNECTED; -import static org.radarcns.producer.rest.ConnectionState.State.DISCONNECTED; -import static org.radarcns.producer.rest.ConnectionState.State.UNKNOWN; +import static org.radarbase.producer.rest.ConnectionState.State.CONNECTED; +import static org.radarbase.producer.rest.ConnectionState.State.DISCONNECTED; +import static org.radarbase.producer.rest.ConnectionState.State.UNKNOWN; import java.util.concurrent.TimeUnit; import org.junit.Test; diff --git a/radar-commons/src/test/java/org/radarcns/producer/rest/RestClientTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.java similarity index 98% rename from radar-commons/src/test/java/org/radarcns/producer/rest/RestClientTest.java rename to radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.java index 1c0edb15..0c983019 100644 --- a/radar-commons/src/test/java/org/radarcns/producer/rest/RestClientTest.java +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -29,7 +29,7 @@ import okhttp3.mockwebserver.RecordedRequest; import org.junit.Before; import org.junit.Test; -import org.radarcns.config.ServerConfig; +import org.radarbase.config.ServerConfig; public class RestClientTest { private MockWebServer server; diff --git a/radar-commons/src/test/java/org/radarcns/producer/rest/RestSenderTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.java similarity index 96% rename from radar-commons/src/test/java/org/radarcns/producer/rest/RestSenderTest.java rename to radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.java index e1f17e21..b244f41b 100644 --- a/radar-commons/src/test/java/org/radarcns/producer/rest/RestSenderTest.java +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -40,16 +40,17 @@ import okhttp3.mockwebserver.RecordedRequest; import org.apache.avro.Schema; import org.apache.avro.SchemaValidationException; +import org.json.JSONException; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.radarcns.config.ServerConfig; -import org.radarcns.data.AvroRecordData; +import org.radarbase.config.ServerConfig; +import org.radarbase.data.AvroRecordData; +import org.radarbase.producer.AuthenticationException; +import org.radarbase.producer.KafkaTopicSender; +import org.radarbase.topic.AvroTopic; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneLight; -import org.radarcns.producer.AuthenticationException; -import org.radarcns.producer.KafkaTopicSender; -import org.radarcns.topic.AvroTopic; public class RestSenderTest { private static final JsonFactory FACTORY = new JsonFactory(); @@ -83,8 +84,8 @@ public void sender() throws Exception { AvroTopic topic = new AvroTopic<>("test", keySchema, valueSchema, ObservationKey.class, PhoneLight.class); Headers headers = new Headers.Builder() - .add("Cookie", "ab") - .add("Cookie", "bc") + .add("Cookie: ab") + .add("Cookie: bc") .build(); sender.setHeaders(headers); KafkaTopicSender topicSender = sender.sender(topic); @@ -227,7 +228,7 @@ public void resetConnectionUnauthorized() throws Exception { @Test public void withCompression() - throws IOException, InterruptedException, SchemaValidationException { + throws IOException, InterruptedException, SchemaValidationException, JSONException { sender.setCompression(true); webServer.enqueue(new MockResponse() .setHeader("Content-Type", "application/json; charset=utf-8") diff --git a/radar-commons/src/test/java/org/radarcns/producer/rest/SchemaRetrieverTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.java similarity index 99% rename from radar-commons/src/test/java/org/radarcns/producer/rest/SchemaRetrieverTest.java rename to radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.java index 9255ece5..38d283ab 100644 --- a/radar-commons/src/test/java/org/radarcns/producer/rest/SchemaRetrieverTest.java +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.producer.rest; +package org.radarbase.producer.rest; import static org.junit.Assert.assertEquals; @@ -31,7 +31,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.radarcns.config.ServerConfig; +import org.radarbase.config.ServerConfig; public class SchemaRetrieverTest { private MockWebServer server; diff --git a/radar-commons/src/test/java/org/radarcns/topic/KafkaTopicTest.java b/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.java similarity index 97% rename from radar-commons/src/test/java/org/radarcns/topic/KafkaTopicTest.java rename to radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.java index 13d08bfc..cf9fac39 100644 --- a/radar-commons/src/test/java/org/radarcns/topic/KafkaTopicTest.java +++ b/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.java @@ -1,4 +1,4 @@ -package org.radarcns.topic; +package org.radarbase.topic; import static org.junit.Assert.assertEquals; diff --git a/radar-commons/src/test/java/org/radarcns/topic/SensorTopicTest.java b/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.java similarity index 99% rename from radar-commons/src/test/java/org/radarcns/topic/SensorTopicTest.java rename to radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.java index c1978767..681f94f0 100644 --- a/radar-commons/src/test/java/org/radarcns/topic/SensorTopicTest.java +++ b/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.topic; +package org.radarbase.topic; import static org.junit.Assert.assertEquals; diff --git a/radar-commons/src/test/java/org/radarcns/util/SerializationTest.java b/radar-commons/src/test/java/org/radarbase/util/SerializationTest.java similarity index 99% rename from radar-commons/src/test/java/org/radarcns/util/SerializationTest.java rename to radar-commons/src/test/java/org/radarbase/util/SerializationTest.java index 2645a041..d8081002 100644 --- a/radar-commons/src/test/java/org/radarcns/util/SerializationTest.java +++ b/radar-commons/src/test/java/org/radarbase/util/SerializationTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.util; +package org.radarbase.util; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; diff --git a/radar-commons/src/test/resources/org/radarcns/producer/rest/android_phone_acceleration.csv b/radar-commons/src/test/resources/org/radarbase/producer/rest/android_phone_acceleration.csv similarity index 100% rename from radar-commons/src/test/resources/org/radarcns/producer/rest/android_phone_acceleration.csv rename to radar-commons/src/test/resources/org/radarbase/producer/rest/android_phone_acceleration.csv diff --git a/settings.gradle b/settings.gradle index caa8aae6..d6543b68 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,9 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +pluginManagement { + repositories { + gradlePluginPortal() + jcenter() + maven { + name "JCenter Gradle Plugins" + url "https://dl.bintray.com/gradle/gradle-plugins" + } + } +} include ':radar-commons' include ':radar-commons-testing' include ':radar-commons-server' include ':radar-commons-unsafe' -enableFeaturePreview('STABLE_PUBLISHING')