[Hive3]Update dependencies for TDP 1.1

jdbc/src/java/org/apache/hive/jdbc/Utils.java

@@ -454,6 +454,17 @@ public static JdbcConnectionParams extractURLComponents(String uri, Properties i
       }
     }
 
+    if (!connParams.getSessionVars().containsKey(JdbcConnectionParams.AUTH_PASSWD)) {
+      if (info.containsKey(JdbcConnectionParams.AUTH_USER)) {
+        connParams.getSessionVars().put(JdbcConnectionParams.AUTH_USER,
+                info.getProperty(JdbcConnectionParams.AUTH_USER));
+      }
+      if (info.containsKey(JdbcConnectionParams.AUTH_PASSWD)) {
+        connParams.getSessionVars().put(JdbcConnectionParams.AUTH_PASSWD,
+                info.getProperty(JdbcConnectionParams.AUTH_PASSWD));
+      }
+    }
+
     if (info.containsKey(JdbcConnectionParams.AUTH_TYPE)) {
       connParams.getSessionVars().put(JdbcConnectionParams.AUTH_TYPE,
           info.getProperty(JdbcConnectionParams.AUTH_TYPE));

pom.xml

@@ -147,7 +147,7 @@
     <guava.version>19.0</guava.version>
     <groovy.version>2.4.11</groovy.version>
     <h2database.version>1.3.166</h2database.version>
-    <hadoop.version>3.1.1-0.0</hadoop.version>
+    <hadoop.version>3.1.4-1.0-SNAPSHOT</hadoop.version>
     <hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path>
     <hamcrest.version>1.3</hamcrest.version>
     <hbase.version>2.0.0-alpha4</hbase.version>
@@ -196,7 +196,7 @@
     <slf4j.version>1.7.10</slf4j.version>
     <ST4.version>4.0.4</ST4.version>
     <storage-api.version>2.7.0</storage-api.version>
-    <tez.version>0.9.1-1.0</tez.version>
+    <tez.version>0.9.1-2.0-SNAPSHOT</tez.version>
     <super-csv.version>2.2.0</super-csv.version>
     <spark.version>2.3.4-1.0</spark.version>
     <scala.binary.version>2.11</scala.binary.version>
@@ -206,7 +206,7 @@
     <wadl-resourcedoc-doclet.version>1.4</wadl-resourcedoc-doclet.version>
     <velocity.version>2.3</velocity.version>
     <xerces.version>2.9.1</xerces.version>
-    <zookeeper.version>3.4.6</zookeeper.version>
+    <zookeeper.version>3.4.14</zookeeper.version>
     <jpam.version>1.1</jpam.version>
     <felix.version>2.4.0</felix.version>
     <curator.version>2.12.0</curator.version>

service/pom.xml

@@ -303,6 +303,12 @@
       <artifactId>apacheds-server-integ</artifactId>
       <version>${apache-directory-server.version}</version>
       <scope>test</scope>
+        <exclusions>
+            <exclusion>
+                <groupId>org.apache.directory.client.ldap</groupId>
+                <artifactId>ldap-client-api</artifactId>
+            </exclusion>
+        </exclusions>
     </dependency>
 
     <dependency>

service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java

@@ -68,7 +68,11 @@ private void cleanRawStore() {
   public void cacheThreadLocalRawStore() {
     Long threadId = this.getId();
     RawStore threadLocalRawStore = HiveMetaStore.HMSHandler.getRawStore();
-    if (threadLocalRawStore != null && !threadRawStoreMap.containsKey(threadId)) {
+    if (threadLocalRawStore == null) {
+      LOG.debug("Thread Local RawStore is null, for the thread: " +
+              this.getName() + " and so removing entry from threadRawStoreMap.");
+      threadRawStoreMap.remove(threadId);
+    } else {
       LOG.debug("Adding RawStore: " + threadLocalRawStore + ", for the thread: " +
           this.getName() + " to threadRawStoreMap for future cleanup.");
       threadRawStoreMap.put(threadId, threadLocalRawStore);

standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java

@@ -696,9 +696,15 @@ public static RawStore getMSForConf(Configuration conf) throws MetaException {
       RawStore ms = threadLocalMS.get();
       if (ms == null) {
         ms = newRawStoreForConf(conf);
-        ms.verifySchema();
+        try {
+          ms.verifySchema();
+        } catch (MetaException e) {
+          ms.shutdown();
+          throw e;
+        }
         threadLocalMS.set(ms);
         ms = threadLocalMS.get();
+        LOG.info("Created RawStore: " + ms + " from thread id: " + Thread.currentThread().getId());
       }
       return ms;
     }

standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java

@@ -421,6 +421,7 @@ private void initialize(Properties dsProps) {
         initializeHelper(dsProps);
         return; // If we reach here, we succeed.
       } catch (Exception e){
+        shutdown();
         numTries--;
         boolean retriable = isRetriableException(e);
         if ((numTries > 0) && retriable){
@@ -482,6 +483,8 @@ private void initializeHelper(Properties dsProps) {
     LOG.info("ObjectStore, initialize called");
     prop = dsProps;
     pm = getPersistenceManager();
+    LOG.info("RawStore: {}, with PersistenceManager: {}" +
+            " created in the thread with id: {}", this, pm, Thread.currentThread().getId());
     try {
       String productName = MetaStoreDirectSql.getProductName(pm);
       sqlGenerator = new SQLGenerator(DatabaseProduct.determineDatabaseProduct(productName), conf);
@@ -499,8 +502,6 @@ private void initializeHelper(Properties dsProps) {
         directSql = new MetaStoreDirectSql(pm, conf, schema);
       }
     }
-    LOG.debug("RawStore: {}, with PersistenceManager: {}" +
-        " created in the thread with id: {}", this, pm, Thread.currentThread().getId());
   }
 
   private DatabaseProduct determineDatabaseProduct() {
@@ -696,7 +697,7 @@ public PersistenceManager getPersistenceManager() {
 
   @Override
   public void shutdown() {
-    LOG.debug("RawStore: {}, with PersistenceManager: {} will be shutdown", this, pm);
+    LOG.info("RawStore: {}, with PersistenceManager: {} will be shutdown", this, pm);
     if (pm != null) {
       pm.close();
       pm = null;

standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/ColumnsStatsUtils.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats;
+
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
+import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector;
+import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector;
+import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector;
+import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector;
+
+/**
+ * Utils class for columnstats package.
+ */
+public final class ColumnsStatsUtils {
+
+  private ColumnsStatsUtils(){}
+
+  /**
+   * Convertes to DateColumnStatsDataInspector if it's a DateColumnStatsData.
+   * @param cso ColumnStatisticsObj
+   * @return DateColumnStatsDataInspector
+   */
+  public static DateColumnStatsDataInspector dateInspectorFromStats(ColumnStatisticsObj cso) {
+    DateColumnStatsDataInspector dateColumnStats;
+    if (cso.getStatsData().getDateStats() instanceof DateColumnStatsDataInspector) {
+      dateColumnStats =
+          (DateColumnStatsDataInspector)(cso.getStatsData().getDateStats());
+    } else {
+      dateColumnStats = new DateColumnStatsDataInspector(cso.getStatsData().getDateStats());
+    }
+    return dateColumnStats;
+  }
+
+  /**
+   * Convertes to StringColumnStatsDataInspector
+   * if it's a StringColumnStatsData.
+   * @param cso ColumnStatisticsObj
+   * @return StringColumnStatsDataInspector
+   */
+  public static StringColumnStatsDataInspector stringInspectorFromStats(ColumnStatisticsObj cso) {
+    StringColumnStatsDataInspector columnStats;
+    if (cso.getStatsData().getStringStats() instanceof StringColumnStatsDataInspector) {
+      columnStats =
+          (StringColumnStatsDataInspector)(cso.getStatsData().getStringStats());
+    } else {
+      columnStats = new StringColumnStatsDataInspector(cso.getStatsData().getStringStats());
+    }
+    return columnStats;
+  }
+
+  /**
+   * Convertes to LongColumnStatsDataInspector if it's a LongColumnStatsData.
+   * @param cso ColumnStatisticsObj
+   * @return LongColumnStatsDataInspector
+   */
+  public static LongColumnStatsDataInspector longInspectorFromStats(ColumnStatisticsObj cso) {
+    LongColumnStatsDataInspector columnStats;
+    if (cso.getStatsData().getLongStats() instanceof LongColumnStatsDataInspector) {
+      columnStats =
+          (LongColumnStatsDataInspector)(cso.getStatsData().getLongStats());
+    } else {
+      columnStats = new LongColumnStatsDataInspector(cso.getStatsData().getLongStats());
+    }
+    return columnStats;
+  }
+
+  /**
+   * Convertes to DoubleColumnStatsDataInspector
+   * if it's a DoubleColumnStatsData.
+   * @param cso ColumnStatisticsObj
+   * @return DoubleColumnStatsDataInspector
+   */
+  public static DoubleColumnStatsDataInspector doubleInspectorFromStats(ColumnStatisticsObj cso) {
+    DoubleColumnStatsDataInspector columnStats;
+    if (cso.getStatsData().getDoubleStats() instanceof DoubleColumnStatsDataInspector) {
+      columnStats =
+          (DoubleColumnStatsDataInspector)(cso.getStatsData().getDoubleStats());
+    } else {
+      columnStats = new DoubleColumnStatsDataInspector(cso.getStatsData().getDoubleStats());
+    }
+    return columnStats;
+  }
+
+  /**
+   * Convertes to DecimalColumnStatsDataInspector
+   * if it's a DecimalColumnStatsData.
+   * @param cso ColumnStatisticsObj
+   * @return DecimalColumnStatsDataInspector
+   */
+  public static DecimalColumnStatsDataInspector decimalInspectorFromStats(ColumnStatisticsObj cso) {
+    DecimalColumnStatsDataInspector columnStats;
+    if (cso.getStatsData().getDecimalStats() instanceof DecimalColumnStatsDataInspector) {
+      columnStats =
+          (DecimalColumnStatsDataInspector)(cso.getStatsData().getDecimalStats());
+    } else {
+      columnStats = new DecimalColumnStatsDataInspector(cso.getStatsData().getDecimalStats());
+    }
+    return columnStats;
+  }
+}

standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java

@@ -38,6 +38,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.dateInspectorFromStats;
+
 public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
     IExtrapolatePartStatus {
 
@@ -62,8 +64,8 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
             cso.getStatsData().getSetField());
         LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, doAllPartitionContainStats);
       }
-      DateColumnStatsDataInspector dateColumnStats =
-          (DateColumnStatsDataInspector) cso.getStatsData().getDateStats();
+      DateColumnStatsDataInspector dateColumnStats = dateInspectorFromStats(cso);
+
       if (dateColumnStats.getNdvEstimator() == null) {
         ndvEstimator = null;
         break;
@@ -95,9 +97,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
       double densityAvgSum = 0.0;
       for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
         ColumnStatisticsObj cso = csp.getColStatsObj();
-        DateColumnStatsDataInspector newData =
-            (DateColumnStatsDataInspector) cso.getStatsData().getDateStats();
-        lowerBound = Math.max(lowerBound, newData.getNumDVs());
+        DateColumnStatsDataInspector newData = dateInspectorFromStats(cso);
         higherBound += newData.getNumDVs();
         densityAvgSum += (diff(newData.getHighValue(), newData.getLowValue()))
             / newData.getNumDVs();
@@ -174,8 +174,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
         for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
           ColumnStatisticsObj cso = csp.getColStatsObj();
           String partName = csp.getPartName();
-          DateColumnStatsDataInspector newData =
-              (DateColumnStatsDataInspector) cso.getStatsData().getDateStats();
+          DateColumnStatsDataInspector newData = dateInspectorFromStats(cso);
           // newData.isSetBitVectors() should be true for sure because we
           // already checked it before.
           if (indexMap.get(partName) != curIndex) {

standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java

@@ -40,6 +40,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.decimalInspectorFromStats;
+
 public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implements
     IExtrapolatePartStatus {
 
@@ -65,8 +67,8 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
         LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName,
             doAllPartitionContainStats);
       }
-      DecimalColumnStatsDataInspector decimalColumnStatsData =
-          (DecimalColumnStatsDataInspector) cso.getStatsData().getDecimalStats();
+      DecimalColumnStatsDataInspector decimalColumnStatsData = decimalInspectorFromStats(cso);
+
       if (decimalColumnStatsData.getNdvEstimator() == null) {
         ndvEstimator = null;
         break;
@@ -98,8 +100,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
       double densityAvgSum = 0.0;
       for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
         ColumnStatisticsObj cso = csp.getColStatsObj();
-        DecimalColumnStatsDataInspector newData =
-            (DecimalColumnStatsDataInspector) cso.getStatsData().getDecimalStats();
+        DecimalColumnStatsDataInspector newData = decimalInspectorFromStats(cso);
         lowerBound = Math.max(lowerBound, newData.getNumDVs());
         higherBound += newData.getNumDVs();
         densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils
@@ -187,8 +188,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
         for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
           ColumnStatisticsObj cso = csp.getColStatsObj();
           String partName = csp.getPartName();
-          DecimalColumnStatsDataInspector newData =
-              (DecimalColumnStatsDataInspector) cso.getStatsData().getDecimalStats();
+          DecimalColumnStatsDataInspector newData = decimalInspectorFromStats(cso);
           // newData.isSetBitVectors() should be true for sure because we
           // already checked it before.
           if (indexMap.get(partName) != curIndex) {

standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java

@@ -37,6 +37,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.doubleInspectorFromStats;
+
 public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implements
     IExtrapolatePartStatus {
 
@@ -63,7 +65,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
             doAllPartitionContainStats);
       }
       DoubleColumnStatsDataInspector doubleColumnStatsData =
-          (DoubleColumnStatsDataInspector) cso.getStatsData().getDoubleStats();
+          doubleInspectorFromStats(cso);
       if (doubleColumnStatsData.getNdvEstimator() == null) {
         ndvEstimator = null;
         break;
@@ -95,8 +97,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
       double densityAvgSum = 0.0;
       for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
         ColumnStatisticsObj cso = csp.getColStatsObj();
-        DoubleColumnStatsDataInspector newData =
-            (DoubleColumnStatsDataInspector) cso.getStatsData().getDoubleStats();
+        DoubleColumnStatsDataInspector newData = doubleInspectorFromStats(cso);
         lowerBound = Math.max(lowerBound, newData.getNumDVs());
         higherBound += newData.getNumDVs();
         densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
@@ -173,7 +174,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
           ColumnStatisticsObj cso = csp.getColStatsObj();
           String partName = csp.getPartName();
           DoubleColumnStatsDataInspector newData =
-              (DoubleColumnStatsDataInspector) cso.getStatsData().getDoubleStats();
+              doubleInspectorFromStats(cso);
           // newData.isSetBitVectors() should be true for sure because we
           // already checked it before.
           if (indexMap.get(partName) != curIndex) {

standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java

@@ -38,6 +38,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.longInspectorFromStats;
+
 public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
     IExtrapolatePartStatus {
 
@@ -63,8 +65,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
         LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName,
             doAllPartitionContainStats);
       }
-      LongColumnStatsDataInspector longColumnStatsData =
-          (LongColumnStatsDataInspector) cso.getStatsData().getLongStats();
+      LongColumnStatsDataInspector longColumnStatsData = longInspectorFromStats(cso);
       if (longColumnStatsData.getNdvEstimator() == null) {
         ndvEstimator = null;
         break;
@@ -96,8 +97,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
       double densityAvgSum = 0.0;
       for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
         ColumnStatisticsObj cso = csp.getColStatsObj();
-        LongColumnStatsDataInspector newData =
-            (LongColumnStatsDataInspector) cso.getStatsData().getLongStats();
+        LongColumnStatsDataInspector newData = longInspectorFromStats(cso);
         lowerBound = Math.max(lowerBound, newData.getNumDVs());
         higherBound += newData.getNumDVs();
         densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
@@ -174,8 +174,7 @@ public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWit
         for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
           ColumnStatisticsObj cso = csp.getColStatsObj();
           String partName = csp.getPartName();
-          LongColumnStatsDataInspector newData =
-              (LongColumnStatsDataInspector) cso.getStatsData().getLongStats();
+          LongColumnStatsDataInspector newData = longInspectorFromStats(cso);
           // newData.isSetBitVectors() should be true for sure because we
           // already checked it before.
           if (indexMap.get(partName) != curIndex) {