Skip to content

Commit

Permalink
refine shade pacakges
Browse files Browse the repository at this point in the history 
Signed-off-by: chenxu <chenxu@dmetasoul.com>
  • Loading branch information
@dmetasoul01
dmetasoul01 committed Mar 21, 2024
1 parent 6ef0076 commit c242dab
Show file tree
Hide file tree
Showing 5 changed files with 194 additions and 11 deletions.
31 changes: 29 additions & 2 deletions lakesoul-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,7 @@ SPDX-License-Identifier: Apache-2.0
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
<useDependencyReducedPomInJar>true</useDependencyReducedPomInJar>
<createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet>
<includes>
<include>**</include>
Expand Down Expand Up @@ -170,6 +169,34 @@ SPDX-License-Identifier: Apache-2.0
<pattern>org.objectweb.asm</pattern>
<shadedPattern>com.lakesoul.shaded.org.objectweb.asm</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.gson</pattern>
<shadedPattern>com.lakesoul.shaded.com.google.gson</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
<shadedPattern>com.lakesoul.shaded.com.fasterxml.jackson</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons</pattern>
<shadedPattern>com.lakesoul.shaded.org.apache.commons</shadedPattern>
</relocation>
<relocation>
<pattern>dev.failsafe</pattern>
<shadedPattern>com.lakesoul.shaded.dev.failsafe</shadedPattern>
</relocation>
<relocation>
<pattern>org.aspectj</pattern>
<shadedPattern>com.lakesoul.shaded.org.aspectj</shadedPattern>
</relocation>
<relocation>
<pattern>org.checkerframework</pattern>
<shadedPattern>com.lakesoul.shaded.org.checkerframework</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>com.lakesoul.shaded.com.google.protobuf</shadedPattern>
</relocation>
</relocations>
</configuration>
</plugin>
Expand Down
2 changes: 0 additions & 2 deletions lakesoul-flink/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,8 +562,6 @@ SPDX-License-Identifier: Apache-2.0
<include>org.apache.flink:flink-python</include>
<include>org.apache.spark:spark-core_${scala.binary.version}</include>
<include>org.apache.spark:spark-catalyst_${scala.binary.version}</include>
<include>org.apache.spark:spark-tags_${scala.binary.version}</include>
<include>org.apache.spark:spark-unsafe_${scala.binary.version}</include>
<include>org.json4s:json4s-jackson_2.12</include>
<include>org.json4s:json4s-core_2.12</include>
<include>org.json4s:json4s-ast_2.12</include>
Expand Down
167 changes: 161 additions & 6 deletions native-io/lakesoul-io-java/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,166 @@ SPDX-License-Identifier: Apache-2.0
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sketch_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-unsafe_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-network-common_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-network-shuffle_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-launcher_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-kvstore_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.janino</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.github.luben</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.roaringbitmap</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.lz4</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.ning</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>net.razorvine</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>net.sf.py4j</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.twitter</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.ivy</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>oro</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.antlr</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.xbean</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>commons-io</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>commons-collections</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.univocity</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>com.clearspring.analytics</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>javax.activation</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
Expand Down Expand Up @@ -389,8 +549,7 @@ SPDX-License-Identifier: Apache-2.0
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
<useDependencyReducedPomInJar>true</useDependencyReducedPomInJar>
<createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet>
<includes>
<include>**</include>
Expand All @@ -414,10 +573,6 @@ SPDX-License-Identifier: Apache-2.0
</filter>
</filters>
<relocations>
<relocation>
<pattern>org.apache.spark.sql</pattern>
<shadedPattern>com.lakesoul.shaded.org.apache.spark.sql</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.spark.sql</pattern>
<shadedPattern>com.lakesoul.shaded.org.apache.spark.sql</shadedPattern>
Expand Down
2 changes: 1 addition & 1 deletion native-io/lakesoul-io-java/src/main/scala/org/apache/spark/sql/arrow/ArrowUtils.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import org.apache.spark.sql.types._

import java.util

private[sql] object ArrowUtils {
object ArrowUtils {

val rootAllocator = new RootAllocator(Long.MaxValue)

Expand Down
3 changes: 3 additions & 0 deletions ...kesoul-io-java/src/test/scala/com/dmetasoul/lakesoul/TestLakeSoulNativeReaderWriter.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package com.dmetasoul.lakesoul
import com.dmetasoul.lakesoul.lakesoul.io.{NativeIOReader, NativeIOWriter}
import org.apache.arrow.vector.types.FloatingPointPrecision
import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema}
import org.apache.spark.sql.arrow.ArrowUtils

import scala.collection.JavaConverters.asJavaIterableConverter

Expand Down Expand Up @@ -38,6 +39,8 @@ case class TestLakeSoulNativeReaderWriter() extends org.scalatest.funsuite.AnyFu
).asJava)
reader.setSchema(schema)
reader.initializeReader()
val sparkSchema = ArrowUtils.fromArrowSchema(schema)
println(sparkSchema)

// val schema = reader.getSchema

Expand Down

0 comments on commit c242dab

Please sign in to comment.