Merge branch 'develop' of github.com:DataBiosphere/jade-data-repo int…

…o se/DR-3357-tps-consumer-tests
DataBiosphere · Jan 22, 2024 · cf6521b · cf6521b
2 parents 4f3e0d8 + 4babc09
commit cf6521b
Show file tree

Hide file tree

Showing 47 changed files with 749 additions and 203 deletions.
diff --git a/.github/workflows/int-and-connected-test-run.yml b/.github/workflows/int-and-connected-test-run.yml
@@ -201,8 +201,8 @@ jobs:
         with:
           actions_subcommand: 'helmdeploy'
           helm_create_secret_manager_secret_version: '0.0.6'
-          helm_datarepo_api_chart_version: 0.0.582
-          helm_datarepo_ui_chart_version: 0.0.302
+          helm_datarepo_api_chart_version: 0.0.589
+          helm_datarepo_ui_chart_version: 0.0.306
           helm_gcloud_sqlproxy_chart_version: 0.19.9
           helm_oidc_proxy_chart_version: 0.0.42
       - name: "Fetch gitHash for deployed integration version"

diff --git a/.github/workflows/staging-smoke-tests.yaml b/.github/workflows/staging-smoke-tests.yaml
@@ -8,8 +8,6 @@ env:
   TDR_LOG_APPENDER: Console-Standard
 on:
   workflow_dispatch: {}
-  schedule:
-    - cron: '0 4 * * *' # run at 4 AM UTC, 12PM EST.
 jobs:
   test-runner-staging:
     runs-on: ubuntu-latest

diff --git a/build.gradle b/build.gradle
@@ -26,27 +26,27 @@ buildscript {
 }
 
 plugins {
-    id "com.google.cloud.tools.jib" version "3.2.0"
+    id 'com.google.cloud.tools.jib' version '3.2.0'
     id 'org.liquibase.gradle' version '2.1.1'
-    id "org.gradle.test-retry" version "1.4.0"
+    id 'org.gradle.test-retry' version '1.4.0'
     id 'antlr'
     id 'com.github.spotbugs' version '4.7.1'
-    id "org.hidetake.swagger.generator" version "2.19.2"
-    id "org.springframework.boot" version "2.7.18"
-    id "idea"
-    id "java"
-    id "io.spring.dependency-management" version "1.0.11.RELEASE"
-    id "jacoco"
-    id "com.diffplug.spotless" version "6.7.1"
-    id "com.dorongold.task-tree" version "2.1.0"
+    id 'org.hidetake.swagger.generator' version '2.19.2'
+    id 'org.springframework.boot' version '2.7.18'
+    id 'idea'
+    id 'java'
+    id 'io.spring.dependency-management' version '1.0.11.RELEASE'
+    id 'jacoco'
+    id 'com.diffplug.spotless' version '6.7.1'
+    id 'com.dorongold.task-tree' version '2.1.0'
     // enables release info in sentry events
-    id "com.gorylenko.gradle-git-properties" version "2.4.1"
-    id "org.sonarqube" version "4.2.1.3168"
+    id 'com.gorylenko.gradle-git-properties' version '2.4.1'
+    id 'org.sonarqube' version '4.2.1.3168'
 }
 
 allprojects {
     group 'bio.terra'
-    version '1.575.0-SNAPSHOT'
+    version '1.582.0-SNAPSHOT'
 
     ext {
         resourceDir = "${rootDir}/src/main/resources/api"
@@ -59,8 +59,8 @@ allprojects {
             mavenBom org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES
         }
         dependencies {
-            dependency group: "io.swagger.core.v3", name: "swagger-annotations", version: "2.1.12"
-            dependency group: "io.swagger.codegen.v3", name: "swagger-codegen-cli", version: "3.0.47"
+            dependency 'io.swagger.core.v3:swagger-annotations:2.1.12'
+            dependency 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.47'
         }
     }
 }
@@ -174,18 +174,17 @@ configurations {
     runtimeClasspath
 }
 
-// Fix for CVE-2023-1370 which identifies a vulnerability in json-smart 2.4.8
-// 2.4.9 fixes the vulnerability and 2.4.10 fixes an additional bug
-// This override can be removed when we upgrade to spring-boot v2.6.15+, v2.7.10+, or v3.0.5+
-ext['json-smart.version'] = '2.4.10'
+// Fix for CVE-2022-25857 which identifies a vulnerability in snake-yaml 1.30
+// This override can likely be removed or bumped to 2.0 when we upgrade to spring boot 3
+ext['snakeyaml.version'] = '1.33'
 
 dependencies {
     // TODO: Unpack the "starter" and include just what we use
     implementation 'com.google.apis:google-api-services-serviceusage:v1-rev20230215-2.0.0'
     implementation 'com.google.apis:google-api-services-appengine:v1-rev20230206-2.0.0'
     implementation 'com.google.apis:google-api-services-oauth2:v2-rev20200213-2.0.0'
     implementation 'com.google.apis:google-api-services-iam:v1-rev20230209-2.0.0'
-    implementation platform('com.google.cloud:libraries-bom:26.26.0')
+    implementation platform('com.google.cloud:libraries-bom:26.30.0')
     implementation 'com.google.cloud:google-cloud-billing'
     implementation 'com.google.cloud:google-cloud-resourcemanager'
     implementation 'com.google.cloud:google-cloud-bigquery'
@@ -195,19 +194,19 @@ dependencies {
     implementation 'com.google.http-client:google-http-client'
     implementation 'org.apache.commons:commons-dbcp2:2.7.0'     // For database connection support
     implementation 'org.apache.commons:commons-lang3:3.11'
-    implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.4'
+    implementation 'org.apache.commons:commons-collections4:4.4'
     implementation 'org.apache.directory.studio:org.apache.commons.io:2.4'
-    implementation 'org.liquibase:liquibase-core:3.8.0'         // For upgrade
-    implementation 'org.postgresql:postgresql:42.2.18'          // Postgres jdbc driver
+    implementation 'org.liquibase:liquibase-core'
+    implementation 'org.postgresql:postgresql'          // Postgres jdbc driver
     implementation 'ch.qos.logback:logback-core:1.2.13'      // Unpin logback versions with DR-3404 Spring 3 Upgrade
     implementation 'ch.qos.logback:logback-classic:1.2.13'  // logback-classic natively implements slf4j api
 
     implementation 'org.codehaus.janino:janino:3.1.11'           // Provides if/else xml parsing for logback config
-    implementation "org.springframework.boot:spring-boot-starter-web"
-    implementation group: "javax.validation", name: "validation-api"
+    implementation 'org.springframework.boot:spring-boot-starter-web'
+    implementation 'javax.validation:validation-api'
 
-    implementation group: "io.swagger.core.v3", name: "swagger-annotations"
-    swaggerCodegen group: "io.swagger.codegen.v3", name: "swagger-codegen-cli"
+    implementation 'io.swagger.core.v3:swagger-annotations'
+    swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli'
 
     implementation 'org.springframework:spring-jdbc:5.1.9.RELEASE'
     implementation 'org.springframework.cloud:spring-cloud-gcp-starter-logging:1.2.8.RELEASE'
@@ -227,7 +226,7 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-thymeleaf'
     implementation 'javax.servlet:jstl:1.2'
 
-    implementation group: 'com.microsoft.sqlserver', name: 'mssql-jdbc', version: '11.2.3.jre17'
+    implementation 'com.microsoft.sqlserver:mssql-jdbc:11.2.3.jre17'
 
     // For distributed locking of Spring @Scheduled tasks across multiple instances
     implementation 'net.javacrumbs.shedlock:shedlock-provider-jdbc-template:5.2.0'
@@ -238,36 +237,35 @@ dependencies {
     // Or define the environment variable ORG_GRADLE_PROJECT_stairwayjar
     if (project.hasProperty("stairwayjar")) {
         implementation files(project.ext.get("stairwayjar"))
-        implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.4'
-        implementation group: 'org.openapitools', name: 'jackson-databind-nullable', version: '0.2.2'
+        implementation 'org.apache.commons:commons-collections4:4.4'
+        implementation 'org.openapitools:jackson-databind-nullable:0.2.2'
     } else {
         implementation 'bio.terra:stairway:0.0.78-SNAPSHOT'
     }
 
     // Similar development mode for pointing to terra common library
     if (project.hasProperty("commonlibjar")) {
         implementation files(project.ext.get("commonlibjar"))
-        implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.4'
-        implementation group: 'org.openapitools', name: 'jackson-databind-nullable', version: '0.2.2'
+        implementation 'org.apache.commons:commons-collections4:4.4'
+        implementation 'org.openapitools:jackson-databind-nullable:0.2.2'
     } else {
         implementation ('bio.terra:terra-common-lib:0.0.89-SNAPSHOT') {
             exclude group: 'org.broadinstitute.dsde.workbench', module: 'sam-client_2.12'
         }
     }
 
-    implementation group: "bio.terra", name: "terra-policy-client", version: "1.0.4-SNAPSHOT"
-    implementation group: "bio.terra", name: "terra-resource-buffer-client", version: "0.4.3-SNAPSHOT"
-    implementation group: "bio.terra", name: "externalcreds-client-resttemplate", version: "0.72.0-SNAPSHOT"
-    implementation group: "org.glassfish.jersey.inject", name: "jersey-hk2", version: "2.30.1"
+    implementation 'bio.terra:terra-policy-client:1.0.4-SNAPSHOT'
+    implementation 'bio.terra:terra-resource-buffer-client:0.4.3-SNAPSHOT'
+    implementation 'bio.terra:externalcreds-client-resttemplate:0.72.0-SNAPSHOT'
+    implementation 'org.glassfish.jersey.inject:jersey-hk2:2.30.1'
 
     // Pin the version of the OkHttp client to avoid a conflict with the version used by the Sam client
-    implementation group: "com.squareup.okhttp3", name: "okhttp", version: "4.10.0"
+    implementation 'com.squareup.okhttp3:okhttp:4.10.0'
     implementation 'org.springframework.boot:spring-boot-starter-actuator'
     implementation 'io.micrometer:micrometer-registry-prometheus'
 
-    implementation 'com.fasterxml.jackson.core:jackson-core:2.16.1'
-    implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1'
-    implementation 'com.fasterxml.jackson.core:jackson-annotations:2.16.1'
+    implementation 'com.fasterxml.jackson.core:jackson-core:2.13.2'
+    implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.2'
 
     // Azure related dependencies
     implementation 'com.azure:azure-identity:1.9.1'
@@ -282,11 +280,11 @@ dependencies {
     implementation('io.sentry:sentry-spring-boot-starter')
     implementation('io.sentry:sentry-logback')
 
-    testImplementation "org.apache.parquet:parquet-common:1.12.0"
-    testImplementation "org.apache.parquet:parquet-hadoop:1.12.0"
-    testImplementation "org.apache.parquet:parquet-hadoop-bundle:1.12.0"
-    testImplementation "org.apache.parquet:parquet-encoding:1.12.0"
-    testImplementation "org.apache.parquet:parquet-column:1.12.0"
+    testImplementation 'org.apache.parquet:parquet-common:1.12.0'
+    testImplementation 'org.apache.parquet:parquet-hadoop:1.12.0'
+    testImplementation 'org.apache.parquet:parquet-hadoop-bundle:1.12.0'
+    testImplementation 'org.apache.parquet:parquet-encoding:1.12.0'
+    testImplementation 'org.apache.parquet:parquet-column:1.12.0'
     testImplementation ('org.apache.hadoop:hadoop-common:3.3.1')  {
         exclude group: 'com.sun.jersey', module: 'jersey-core'
         exclude group: 'com.sun.jersey', module: 'jersey-servlet'
@@ -310,22 +308,21 @@ dependencies {
     testImplementation 'au.com.dius.pact.provider:junit5spring:4.6.1'
     testImplementation 'au.com.dius.pact.consumer:junit5:4.6.1'
 
-    antlr "org.antlr:antlr4:4.8"
+    antlr 'org.antlr:antlr4:4.8'
     spotbugs 'com.github.spotbugs:spotbugs:4.2.3'
 
     // Need groovy on the class path for the logback config. Could use XML and skip this dependency,
     // but the groovy config is... well... groovy.
-    runtimeOnly group: 'org.apache.groovy', name: 'groovy', version: '4.0.11'
+    runtimeOnly 'org.apache.groovy:groovy:4.0.11'
 
     // Findbugs annotations, so we can selectively suppress findbugs findings
     compileOnly 'com.google.code.findbugs:annotations:3.0.1'
 
-    liquibaseRuntime 'org.liquibase:liquibase-core:3.8.0'
-    liquibaseRuntime 'org.postgresql:postgresql:42.2.7'
+    liquibaseRuntime 'org.liquibase:liquibase-core:4.9.1'
+    liquibaseRuntime 'org.postgresql:postgresql:42.3.8'
 
-    testImplementation group: 'org.junit.vintage', name: 'junit-vintage-engine', version: '5.3.0-M1'
-    testImplementation "org.springframework.boot:spring-boot-starter-test"
-    testImplementation 'org.postgresql:postgresql:42.2.8'
+    testImplementation 'org.junit.vintage:junit-vintage-engine:5.3.0-M1'
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
     testImplementation 'com.google.code.findbugs:annotations:3.0.1'
     testImplementation 'io.zonky.test:embedded-database-spring-test:2.1.1'
     testImplementation 'io.zonky.test:embedded-postgres:1.3.1'
@@ -337,11 +334,11 @@ dependencies {
     // boot-starter-validation required for javax.validation.Valid references, @Valid tags
     generatedCompile 'org.springframework.boot:spring-boot-starter-validation'
     implementation 'org.springframework.boot:spring-boot-starter-validation'
-    implementation group: "org.webjars", name: "webjars-locator-core", version: "0.46"
-    runtimeOnly group: "org.webjars.npm", name: "swagger-ui-dist", version: "4.3.0"
-    generatedCompile group: "io.swagger.core.v3", name: "swagger-annotations"
+    implementation 'org.webjars:webjars-locator-core:0.46'
+    runtimeOnly 'org.webjars.npm:swagger-ui-dist:4.3.0'
+    generatedCompile 'io.swagger.core.v3:swagger-annotations'
 
-    annotationProcessor "org.springframework.boot:spring-boot-configuration-processor"
+    annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
 }
 
 def getGitHash = { ->

diff --git a/src/main/java/bio/terra/app/configuration/ApplicationConfiguration.java b/src/main/java/bio/terra/app/configuration/ApplicationConfiguration.java
@@ -30,6 +30,7 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Primary;
 import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.lang.Nullable;
 import org.springframework.web.client.RestTemplate;
 
 @Configuration
@@ -429,7 +430,7 @@ public RestTemplate restTemplate(RestTemplateBuilder builder) {
   }
 
   @Bean("tdrServiceAccountEmail")
-  public String tdrServiceAccountEmail() throws IOException {
+  public @Nullable String tdrServiceAccountEmail() throws IOException {
     GoogleCredentials defaultCredentials = GoogleCredentials.getApplicationDefault();
     if (defaultCredentials instanceof ServiceAccountCredentials) {
       return ((ServiceAccountCredentials) defaultCredentials).getClientEmail();

diff --git a/src/main/java/bio/terra/app/controller/DatasetsApiController.java b/src/main/java/bio/terra/app/controller/DatasetsApiController.java
@@ -199,7 +199,7 @@ public ResponseEntity<SnapshotBuilderGetConceptsResponse> getConcepts(
         IamResourceType.DATASET,
         id.toString(),
         IamAction.VIEW_SNAPSHOT_BUILDER_SETTINGS);
-    return ResponseEntity.ok(snapshotBuilderService.getConceptChildren(id, conceptId));
+    return ResponseEntity.ok(snapshotBuilderService.getConceptChildren(id, conceptId, userRequest));
   }
 
   @Override

diff --git a/src/main/java/bio/terra/app/usermetrics/UserMetricsInterceptor.java b/src/main/java/bio/terra/app/usermetrics/UserMetricsInterceptor.java
@@ -10,8 +10,8 @@
 import java.util.concurrent.ExecutorService;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import liquibase.util.StringUtils;
 import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Component;

diff --git a/src/main/java/bio/terra/common/Column.java b/src/main/java/bio/terra/common/Column.java
@@ -36,7 +36,11 @@ public static SynapseColumn toSynapseColumn(Column datasetColumn) {
     return (SynapseColumn)
         new SynapseColumn()
             .synapseDataType(
-                SynapseColumn.translateDataType(datasetColumn.getType(), datasetColumn.isArrayOf()))
+                SynapseColumn.translateDataType(
+                    datasetColumn.getType(), datasetColumn.isArrayOf(), false))
+            .synapseDataTypeForCsv(
+                SynapseColumn.translateDataType(
+                    datasetColumn.getType(), datasetColumn.isArrayOf(), true))
             .requiresCollate(
                 SynapseColumn.checkForCollateArgRequirement(
                     datasetColumn.getType(), datasetColumn.isArrayOf()))

diff --git a/src/main/java/bio/terra/common/SynapseColumn.java b/src/main/java/bio/terra/common/SynapseColumn.java
@@ -9,6 +9,7 @@ public class SynapseColumn extends Column {
   private static final Set<TableDataType> FILE_TYPES =
       Set.of(TableDataType.FILEREF, TableDataType.DIRREF);
   private String synapseDataType;
+  private String synapseDataTypeForCsv;
   private boolean requiresCollate;
   private boolean requiresJSONCast;
 
@@ -32,6 +33,15 @@ public SynapseColumn synapseDataType(String synapseDataType) {
     return this;
   }
 
+  public String getSynapseDataTypeForCsv() {
+    return synapseDataTypeForCsv;
+  }
+
+  public SynapseColumn synapseDataTypeForCsv(String synapseDataTypeForCsv) {
+    this.synapseDataTypeForCsv = synapseDataTypeForCsv;
+    return this;
+  }
+
   public boolean getRequiresTypeCast() {
     return requiresTypeCast;
   }
@@ -64,8 +74,13 @@ public boolean getIsFileType() {
   }
 
   public static String translateDataType(TableDataType datatype, boolean isArrayOf) {
+    return translateDataType(datatype, isArrayOf, false);
+  }
+
+  public static String translateDataType(
+      TableDataType datatype, boolean isArrayOf, boolean isForCsv) {
     if (isArrayOf) {
-      return "varchar(8000)";
+      return "varchar(%s)".formatted(isForCsv ? "8000" : "max");
     }
     return switch (datatype) {
       case BOOLEAN -> "bit";
@@ -78,7 +93,8 @@ public static String translateDataType(TableDataType datatype, boolean isArrayOf
       case NUMERIC -> "real";
         // DIRREF and FILEREF store a UUID on ingest
         // But, are translated to DRS URI on Snapshot Creation
-      case DIRREF, FILEREF, TEXT, STRING -> "varchar(8000)";
+        // Note that the Synapse CSV parser does not support varchars larger than 8000 bytes
+      case DIRREF, FILEREF, TEXT, STRING -> "varchar(%s)".formatted(isForCsv ? "8000" : "max");
       case TIME -> "time";
         // Data of type RECORD contains table-like that can be nested or repeated
         // It's provided in JSON format, making it hard to parse from inside a CSV/JSON ingest
@@ -103,29 +119,20 @@ static boolean checkForCollateArgRequirement(TableDataType dataType, boolean isA
     if (isArrayOf) {
       return true;
     }
-    switch (dataType) {
-      case DIRREF:
-      case FILEREF:
-      case TEXT:
-      case STRING:
-        return true;
-      default:
-        return false;
-    }
+    return switch (dataType) {
+      case DIRREF, FILEREF, TEXT, STRING -> true;
+      default -> false;
+    };
   }
 
   static boolean checkForJSONCastRequirement(TableDataType dataType, boolean isArrayOf) {
     if (isArrayOf) {
       return false;
-    }
-    switch (dataType) {
-      case DIRREF:
-      case FILEREF:
-      case TEXT:
-      case STRING:
-        return false;
-      default:
-        return true;
+    } else {
+      return switch (dataType) {
+        case TEXT, STRING -> true;
+        default -> false;
+      };
     }
   }