update cache to S3

huggingface · Jul 10, 2024 · 0826987 · 0826987
1 parent 3620a34
commit 0826987
Show file tree

Hide file tree

Showing 7 changed files with 20 additions and 51 deletions.
diff --git a/.github/workflows/build_75.yaml b/.github/workflows/build_75.yaml
@@ -84,8 +84,8 @@
              DEFAULT_USE_FLASH_ATTENTION=False
            tags: ${{ steps.meta-75.outputs.tags }}
            labels: ${{ steps.meta-75.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-75,mode=max
-           cache-to: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-75,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-75,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
+           cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-75,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
 
        - name: Extract metadata (tags, labels) for Docker
          id: meta-75-grpc
@@ -121,4 +121,4 @@
              DEFAULT_USE_FLASH_ATTENTION=False
            tags: ${{ steps.meta-75-grpc.outputs.tags }}
            labels: ${{ steps.meta-75-grpc.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-75,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-75,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
diff --git a/.github/workflows/build_80.yaml b/.github/workflows/build_80.yaml
@@ -95,8 +95,8 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-80.outputs.tags }}
            labels: ${{ steps.meta-80.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-80,mode=max
-           cache-to: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-80,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-80,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
+           cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-80,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
 
        - name: Extract metadata (tags, labels) for Docker
          id: meta-80-grpc
@@ -131,4 +131,4 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-80-grpc.outputs.tags }}
            labels: ${{ steps.meta-80-grpc.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-80,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-80,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
diff --git a/.github/workflows/build_86.yaml b/.github/workflows/build_86.yaml
@@ -83,8 +83,8 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-86.outputs.tags }}
            labels: ${{ steps.meta-86.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-86,mode=max
-           cache-to: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-86,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-86,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
+           cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-86,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
 
        - name: Extract metadata (tags, labels) for Docker
          id: meta-86-grpc
@@ -119,4 +119,4 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-86-grpc.outputs.tags }}
            labels: ${{ steps.meta-86-grpc.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-86,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-86,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
diff --git a/.github/workflows/build_89.yaml b/.github/workflows/build_89.yaml
@@ -83,8 +83,8 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-89.outputs.tags }}
            labels: ${{ steps.meta-89.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-89,mode=max
-           cache-to: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-89,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-89,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
+           cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-89,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
 
        - name: Extract metadata (tags, labels) for Docker
          id: meta-89-grpc
@@ -119,4 +119,4 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-89-grpc.outputs.tags }}
            labels: ${{ steps.meta-89-grpc.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-89,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-89,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
diff --git a/.github/workflows/build_90.yaml b/.github/workflows/build_90.yaml
@@ -83,8 +83,8 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-90.outputs.tags }}
            labels: ${{ steps.meta-90.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-90,mode=max
-           cache-to: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-90,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-90,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
+           cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-90,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
 
        - name: Extract metadata (tags, labels) for Docker
          id: meta-90-grpc
@@ -119,4 +119,4 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-90-grpc.outputs.tags }}
            labels: ${{ steps.meta-90-grpc.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-90,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-90,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
diff --git a/.github/workflows/build_all.yaml b/.github/workflows/build_all.yaml
@@ -72,36 +72,5 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta.outputs.tags }}
            labels: ${{ steps.meta.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-all,mode=max
-           cache-to: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-all,mode=max
-
-       - name: Extract metadata (tags, labels) for Docker
-         id: meta-sagemaker
-         uses: docker/metadata-action@v4.3.0
-         with:
-           images: |
-             registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference/sagemaker
-           flavor: |
-             latest=false
-           tags: |
-             type=semver,pattern=cuda-{{version}}
-             type=semver,pattern=cuda-{{major}}.{{minor}}
-             type=raw,value=cuda-latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
-             type=raw,value=cuda-sha-${{ env.GITHUB_SHA_SHORT }}
-
-       - name: Build and push Docker image
-         id: build-and-push-sagemaker
-         uses: docker/build-push-action@v4
-         with:
-           context: .
-           file: Dockerfile-cuda-all
-           push: ${{ github.event_name != 'pull_request' }}
-           platforms: 'linux/amd64'
-           target: sagemaker
-           build-args: |
-             GIT_SHA=${{ env.GITHUB_SHA }}
-             DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
-           tags: ${{ steps.meta-sagemaker.outputs.tags }}
-           labels: ${{ steps.meta-sagemaker.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-all,mode=max
-           cache-to: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-all,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-all,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
+           cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-all,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
diff --git a/.github/workflows/build_cpu.yaml b/.github/workflows/build_cpu.yaml
@@ -94,8 +94,8 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-cpu.outputs.tags }}
            labels: ${{ steps.meta-cpu.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-cpu,mode=max
-           cache-to: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-cpu,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-cpu,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
+           cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-cpu,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max
 
        - name: Extract metadata (tags, labels) for Docker
          id: meta-cpu-grpc
@@ -129,4 +129,4 @@
              DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}
            tags: ${{ steps.meta-cpu-grpc.outputs.tags }}
            labels: ${{ steps.meta-cpu-grpc.outputs.labels }}
-           cache-from: type=registry,ref=registry-push.github-runners.huggingface.tech/api-inference/text-embeddings-inference:cache-cpu,mode=max
+           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-embeddings-inference-cache-cpu,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=max