Support for Spark DL notebooks on databricks/dataproc

.gitignore

@@ -7,6 +7,11 @@
 *.pyo
 *.swp
 *~
+*.log
+*.db
+*.pt
+*.rst
+*.tar.gz
 .DS_Store
 .cache
 .classpath
@@ -22,3 +27,21 @@
 .settings
 hs_err*.log
 target
+/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton
+/examples/ML+DL-Examples/Spark-DL/dl_inference/archive
+# /examples/ML+DL-Examples/Spark-Rapids-ML/pca
+/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/new_ideas
+/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/torch_notebooks
+/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/tf_notebooks
+/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/datasets
+/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/spark-dl-datasets
+/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models
+**/spark-dl-datasets
+**/models
+**/predictions
+**/dl_inference/databricks/*.ipynb
+**/dl_inference/dataproc/*.ipynb
+**/dl_inference/huggingface/.ipynb_checkpoints
+/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/spark-dl-notebooks
+/examples/ML+DL-Examples/Spark-DL/dl_inference/test.ipynb
+/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/copy_notebooks_gcp.sh

.scala-build/.bloop/spark-rapids-examples_f4dd477a3a-test.json

@@ -0,0 +1 @@
+{"version":"1.4.0","project":{"name":"spark-rapids-examples_f4dd477a3a-test","directory":"/home/rishic/spark-rapids-examples/.scala-build","workspaceDir":"/home/rishic/spark-rapids-examples","sources":["/home/rishic/spark-rapids-examples/examples/MIG-Support/device-plugins/gpu-mig/src/test/java/com/nvidia/spark/TestNvidiaGPUMigPluginForRuntimeV2.java"],"dependencies":["spark-rapids-examples_f4dd477a3a"],"classpath":["/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.4.2/scala3-library_3-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar","/home/rishic/spark-rapids-examples/.scala-build/spark-rapids-examples_f4dd477a3a/classes/main","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/com/sourcegraph/semanticdb-javac/0.10.0/semanticdb-javac-0.10.0.jar"],"out":"/home/rishic/spark-rapids-examples/.scala-build/.bloop/spark-rapids-examples_f4dd477a3a-test","classesDir":"/home/rishic/spark-rapids-examples/.scala-build/spark-rapids-examples_f4dd477a3a/classes/test","scala":{"organization":"org.scala-lang","name":"scala-compiler","version":"3.4.2","options":["-Xsemanticdb","-sourceroot","/home/rishic/spark-rapids-examples"],"jars":["/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.4.2/scala3-compiler_3-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-interfaces/3.4.2/scala3-interfaces-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.4.2/scala3-library_3-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/tasty-core_3/3.4.2/tasty-core_3-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-asm/9.6.0-scala-1/scala-asm-9.6.0-scala-1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/compiler-interface/1.9.6/compiler-interface-1.9.6.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-reader/3.25.1/jline-reader-3.25.1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal/3.25.1/jline-terminal-3.25.1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal-jna/3.25.1/jline-terminal-jna-3.25.1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/util-interface/1.9.8/util-interface-1.9.8.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-native/3.25.1/jline-native-3.25.1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/net/java/dev/jna/jna/5.14.0/jna-5.14.0.jar"],"bridgeJars":["/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-sbt-bridge/3.4.2/scala3-sbt-bridge-3.4.2.jar"]},"java":{"options":["-Xplugin:semanticdb -sourceroot:/home/rishic/spark-rapids-examples -targetroot:javac-classes-directory","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"]},"test":{"frameworks":[{"names":["com.novocode.junit.JUnitFramework"]},{"names":["org.scalatest.tools.Framework","org.scalatest.tools.ScalaTestFramework"]},{"names":["org.scalacheck.ScalaCheckFramework"]},{"names":["org.specs.runner.SpecsFramework","org.specs2.runner.Specs2Framework","org.specs2.runner.SpecsFramework"]},{"names":["utest.runner.Framework"]},{"names":["munit.Framework"]}],"options":{"excludes":[],"arguments":[]}},"platform":{"name":"jvm","config":{"home":"/usr/lib/jvm/java-8-openjdk-amd64","options":[]},"mainClass":[]},"resolution":{"modules":[{"organization":"org.scala-lang","name":"scala3-library_3","version":"3.4.2","artifacts":[{"name":"scala3-library_3","path":"/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.4.2/scala3-library_3-3.4.2.jar"},{"name":"scala3-library_3","classifier":"sources","path":"/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.4.2/scala3-library_3-3.4.2-sources.jar"}]},{"organization":"org.scala-lang","name":"scala-library","version":"2.13.12","artifacts":[{"name":"scala-library","path":"/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar"},{"name":"scala-library","classifier":"sources","path":"/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12-sources.jar"}]}]},"tags":["test"]}}

.scala-build/.bloop/spark-rapids-examples_f4dd477a3a.json

@@ -0,0 +1 @@
+{"version":"1.4.0","project":{"name":"spark-rapids-examples_f4dd477a3a","directory":"/home/rishic/spark-rapids-examples/.scala-build","workspaceDir":"/home/rishic/spark-rapids-examples","sources":["/home/rishic/spark-rapids-examples/examples/MIG-Support/device-plugins/gpu-mig/src/main/java/com/nvidia/spark/NvidiaGPUMigPluginForRuntimeV2.java","/home/rishic/spark-rapids-examples/examples/ML+DL-Examples/Spark-cuML/pca/main.scala","/home/rishic/spark-rapids-examples/examples/ML+DL-Examples/Spark-cuML/pca/scala/src/com/nvidia/spark/examples/pca/Main.scala","/home/rishic/spark-rapids-examples/examples/ML+DL-Examples/Spark-cuML/pca/target/classes/com/nvidia/spark/examples/pca/Main.scala","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/hive/DecimalFraction.java","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/hive/StringWordCount.java","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/hive/URLDecode.java","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/hive/URLEncode.java","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/java/CosineSimilarity.java","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/java/DecimalFraction.java","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/java/NativeUDFExamplesLoader.java","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/java/URLDecode.java","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/java/URLEncode.java","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/scala/com/nvidia/spark/rapids/udf/scala/URLDecode.scala","/home/rishic/spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/scala/com/nvidia/spark/rapids/udf/scala/URLEncode.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/agaricus/scala/src/com/nvidia/spark/examples/agaricus/Main.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/agaricus/target/classes/com/nvidia/spark/examples/agaricus/Main.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/scala/src/com/nvidia/spark/examples/mortgage/CrossValidationMain.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/scala/src/com/nvidia/spark/examples/mortgage/ETLMain.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/scala/src/com/nvidia/spark/examples/mortgage/Main.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/scala/src/com/nvidia/spark/examples/mortgage/Mortgage.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/scala/src/com/nvidia/spark/examples/mortgage/XGBoostETL.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/target/classes/com/nvidia/spark/examples/mortgage/CrossValidationMain.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/target/classes/com/nvidia/spark/examples/mortgage/ETLMain.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/target/classes/com/nvidia/spark/examples/mortgage/Main.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/target/classes/com/nvidia/spark/examples/mortgage/Mortgage.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/mortgage/target/classes/com/nvidia/spark/examples/mortgage/XGBoostETL.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/taxi/scala/src/com/nvidia/spark/examples/taxi/CrossValidationMain.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/taxi/scala/src/com/nvidia/spark/examples/taxi/ETLMain.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/taxi/scala/src/com/nvidia/spark/examples/taxi/Main.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/taxi/scala/src/com/nvidia/spark/examples/taxi/Taxi.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/taxi/target/classes/com/nvidia/spark/examples/taxi/CrossValidationMain.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/taxi/target/classes/com/nvidia/spark/examples/taxi/ETLMain.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/taxi/target/classes/com/nvidia/spark/examples/taxi/Main.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/taxi/target/classes/com/nvidia/spark/examples/taxi/Taxi.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/utility/scala/src/com/nvidia/spark/examples/utility/Benchmark.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/utility/scala/src/com/nvidia/spark/examples/utility/SparkSetup.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/utility/scala/src/com/nvidia/spark/examples/utility/Vectorize.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/utility/scala/src/com/nvidia/spark/examples/utility/XGBoostArgs.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/utility/target/classes/com/nvidia/spark/examples/utility/Benchmark.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/utility/target/classes/com/nvidia/spark/examples/utility/SparkSetup.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/utility/target/classes/com/nvidia/spark/examples/utility/Vectorize.scala","/home/rishic/spark-rapids-examples/examples/XGBoost-Examples/utility/target/classes/com/nvidia/spark/examples/utility/XGBoostArgs.scala"],"dependencies":[],"classpath":["/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.4.2/scala3-library_3-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/com/sourcegraph/semanticdb-javac/0.10.0/semanticdb-javac-0.10.0.jar"],"out":"/home/rishic/spark-rapids-examples/.scala-build/.bloop/spark-rapids-examples_f4dd477a3a","classesDir":"/home/rishic/spark-rapids-examples/.scala-build/spark-rapids-examples_f4dd477a3a/classes/main","scala":{"organization":"org.scala-lang","name":"scala-compiler","version":"3.4.2","options":["-Xsemanticdb","-sourceroot","/home/rishic/spark-rapids-examples"],"jars":["/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.4.2/scala3-compiler_3-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-interfaces/3.4.2/scala3-interfaces-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.4.2/scala3-library_3-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/tasty-core_3/3.4.2/tasty-core_3-3.4.2.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-asm/9.6.0-scala-1/scala-asm-9.6.0-scala-1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/compiler-interface/1.9.6/compiler-interface-1.9.6.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-reader/3.25.1/jline-reader-3.25.1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal/3.25.1/jline-terminal-3.25.1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal-jna/3.25.1/jline-terminal-jna-3.25.1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/util-interface/1.9.8/util-interface-1.9.8.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-native/3.25.1/jline-native-3.25.1.jar","/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/net/java/dev/jna/jna/5.14.0/jna-5.14.0.jar"],"bridgeJars":["/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-sbt-bridge/3.4.2/scala3-sbt-bridge-3.4.2.jar"]},"java":{"options":["-Xplugin:semanticdb -sourceroot:/home/rishic/spark-rapids-examples -targetroot:javac-classes-directory","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"]},"platform":{"name":"jvm","config":{"home":"/usr/lib/jvm/java-8-openjdk-amd64","options":[]},"mainClass":[]},"resolution":{"modules":[{"organization":"org.scala-lang","name":"scala3-library_3","version":"3.4.2","artifacts":[{"name":"scala3-library_3","path":"/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.4.2/scala3-library_3-3.4.2.jar"},{"name":"scala3-library_3","classifier":"sources","path":"/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.4.2/scala3-library_3-3.4.2-sources.jar"}]},{"organization":"org.scala-lang","name":"scala-library","version":"2.13.12","artifacts":[{"name":"scala-library","path":"/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar"},{"name":"scala-library","classifier":"sources","path":"/home/rishic/.cache/coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12-sources.jar"}]}]},"tags":["library"]}}

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "files.watcherExclude": {
+        "**/target": true
+    },
+    "java.compile.nullAnalysis.mode": "disabled"
+}