Skip to content

Commit

Permalink
fix so RDDsServlet class is Spark 3.5 compatible (see h2oai/sparkling…
Browse files Browse the repository at this point in the history
…-water#5620); bump version to evg2
  • Loading branch information
Ian Springer committed Apr 11, 2024
1 parent c5f7fea commit 5839f61
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.catalyst.ScalaReflection.universe._
import org.apache.spark.sql.types.{StructField, StructType}
import water.exceptions.H2ONotFoundArgumentException

Expand Down Expand Up @@ -89,14 +90,17 @@ private[api] class RDDsServlet extends ServletBase {
case _: java.sql.Timestamp =>
hc.asH2OFrame(rdd.asInstanceOf[RDD[java.sql.Timestamp]], name)
case _: Product =>
val first = rdd.asInstanceOf[RDD[Product]].first()
val fields = ScalaReflection.getConstructorParameters(first.getClass).map { v =>
val cls = rdd.asInstanceOf[RDD[Product]].first().getClass
val rm = runtimeMirror(cls.getClassLoader)
val classSymbol = rm.staticClass(cls.getName)
val tpe = classSymbol.selfType
val fields = ScalaReflection.getConstructorParameters(tpe).map { v =>
val schema = ScalaReflection.schemaFor(v._2)
StructField(v._1, schema.dataType, schema.nullable)
}
val df = SparkSessionUtils.active
.createDataFrame(rdd.asInstanceOf[RDD[Product]].map(Row.fromTuple), StructType(fields))
hc.asH2OFrame(df, name)
hc.asH2OFrame(df, name)
case t => throw new IllegalArgumentException(s"Do not understand type $t")
}
}
Expand Down
2 changes: 1 addition & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ pythonEnvironments=2.7 3.6 3.7 3.8
# Select for which Spark version is Sparkling Water built by default
spark=3.5
# Sparkling Water Version
version=3.36.1.6-1-SNAPSHOT
version=3.36.1.5-1-evg2
# Spark version from which is Kubernetes Supported
kubernetesSupportSinceSpark=2.4
databricksTestSinceSpark=2.4
Expand Down

0 comments on commit 5839f61

Please sign in to comment.