Skip to content

Commit

Permalink
Move both Iceberg and S3A CICD to snapshot builds (#186)
Browse files Browse the repository at this point in the history
## Description of change

This commit makes sure we can use `-PsnapshotBuild=true` not only for
publishing artefacts but also to build them. This is for convenience:
for CICD, we always want to test the latest built artefact. Such an
artefact must:
1. Have a name that does not exist in any repositories (either local
Maven cache or public Maven central).
2. Have a name that does not change across versions to make it easy to
point automation to the latest artefact.

After this commit, Hadoop build should be healthy.

Commit was tested locally, by invoking manual builds and inspecting
artefacts. There is no mention of old artefact names like
`input-stream.jar`, `object-client.jar`, `common.jar` anymore in the
code base.
  • Loading branch information
CsengerG authored Nov 28, 2024
1 parent 0aa6ad3 commit 39869de
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 54 deletions.
60 changes: 9 additions & 51 deletions .github/workflows/build-upload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,27 +36,12 @@ jobs:

- name: Build with Gradle
run: |
./gradlew build
./gradlew -PsnapshotBuild=true build
- uses: actions/upload-artifact@v4
with:
path: "common/build/libs/common.jar"
name: "common.jar"

- uses: actions/upload-artifact@v4
with:
path: "input-stream/build/libs/input-stream-all.jar"
name: "input-stream-all.jar"

- uses: actions/upload-artifact@v4
with:
path: "input-stream/build/libs/input-stream.jar"
name: "input-stream.jar"

- uses: actions/upload-artifact@v4
with:
path: "object-client/build/libs/object-client.jar"
name: "object-client.jar"
path: "input-stream/build/libs/analyticsaccelerator-s3-SNAPSHOT.jar"
name: "analyticsaccelerator-s3-SNAPSHOT.jar"

BuildHadoopAndUploadArtifact:
name: Build Hadoop artifacts
Expand All @@ -73,10 +58,7 @@ jobs:

- name: Build and publish to local Maven with Gradle
run: |
./gradlew publishToMavenLocal
- name: Publish uber jar to local Maven
run: mvn install:install-file -Dfile=input-stream/build/libs/input-stream-all.jar -DgroupId=com.amazon.connector.s3 -DartifactId=input-stream -Dversion=1.0.0 -Dpackaging=jar -DgeneratePom=true
./gradlew -PsnapshotBuild=true publishToMavenLocal
- name: Setup Hadoop SSH deploy key
uses: webfactory/ssh-agent@v0.9.0
Expand Down Expand Up @@ -161,19 +143,7 @@ jobs:
steps:
- uses: actions/download-artifact@v4
with:
name: "common.jar"

- uses: actions/download-artifact@v4
with:
name: "input-stream-all.jar"

- uses: actions/download-artifact@v4
with:
name: "input-stream.jar"

- uses: actions/download-artifact@v4
with:
name: "object-client.jar"
name: "analyticsaccelerator-s3-SNAPSHOT.jar"

- uses: actions/download-artifact@v4
with:
Expand All @@ -190,23 +160,11 @@ jobs:
role-session-name: GitHub_to_AWS_via_FederatedOIDC
aws-region: ${{ env.AWS_REGION }}

- name: Upload common JAR to S3a treatment bucket
run: aws s3 cp common.jar s3://${{ env.S3_BUCKET }}/s3a/common-1.0.0.jar

- name: Upload common JAR to S3FileIO treatment bucket
run: aws s3 cp common.jar s3://${{ env.S3_BUCKET }}/s3fileio/common.jar

- name: Upload input-stream JAR to S3a treatment bucket
run: aws s3 cp input-stream-all.jar s3://${{ env.S3_BUCKET }}/s3a/input-stream-1.0.0.jar

- name: Upload input-stream JAR to S3FileIO treatment bucket
run: aws s3 cp input-stream.jar s3://${{ env.S3_BUCKET }}/s3fileio/input-stream.jar

- name: Upload object-client JAR to S3a treatment bucket
run: aws s3 cp object-client.jar s3://${{ env.S3_BUCKET }}/s3a/object-client-1.0.0.jar
- name: Upload analyticsaccelerator JAR to S3a treatment bucket
run: aws s3 cp analyticsaccelerator-s3-SNAPSHOT.jar s3://${{ env.S3_BUCKET }}/s3a/analyticsaccelerator-s3-SNAPSHOT.jar

- name: Upload object-client JAR to S3FileIO treatment bucket
run: aws s3 cp object-client.jar s3://${{ env.S3_BUCKET }}/s3fileio/object-client.jar
- name: Upload analyticsaccelerator JAR to S3FileIO treatment bucket
run: aws s3 cp analyticsaccelerator-s3-SNAPSHOT.jar s3://${{ env.S3_BUCKET }}/s3fileio/analyticsaccelerator-s3-SNAPSHOT.jar

- name: Upload Iceberg JAR to S3FileIO treatment bucket
run: aws s3 cp iceberg-spark-runtime-3.5_2.12-1.6.0-SNAPSHOT.jar s3://${{ env.S3_BUCKET }}/s3fileio/iceberg-spark-runtime-3.5_2.12-1.6.0-SNAPSHOT.jar
Expand Down
8 changes: 5 additions & 3 deletions input-stream/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ import com.github.jk1.license.render.TextReportRenderer

val group = "software.amazon.s3.analyticsaccelerator"
val artefact = "analyticsaccelerator-s3"
val currentVersion = "0.0.1"
val currentVersionNumber = "0.0.1"

val isSnapshot = findProperty("snapshotBuild") == "true"
val currentVersion = if (isSnapshot) "SNAPSHOT" else currentVersionNumber;

plugins {
id("buildlogic.java-library-conventions")
Expand Down Expand Up @@ -282,8 +285,7 @@ publishing {
groupId = group
artifactId = artefact

val isSnapshot = findProperty("snapshotBuild") == "true"
version = if (isSnapshot) "SNAPSHOT" else currentVersion;
version = currentVersion

pom {
name = "S3 Analytics Accelerator Library for Amazon S3"
Expand Down

0 comments on commit 39869de

Please sign in to comment.