python-check.yaml

name: Python Check

on:
  push:
    branches:
      - main
  pull_request:
    types: [opened, reopened, synchronize, labeled]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

defaults:
  run:
    working-directory: ./icechunk-python

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install Just
        run: sudo snap install --edge --classic just
      - name: Stand up MinIO
        run: |
          docker compose up -d minio

      - name: Wait for MinIO to be ready
        run: |
          for _ in {1..10}; do
              if curl --silent --fail http://minio:9000/minio/health/live; then
              break
              fi
              sleep 3
          done
          docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          working-directory: icechunk-python
          # target: ${{ matrix.platform.target }}
          args: --release --out dist --find-interpreter
          sccache: 'true'
          # manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245

      - name: Cargo lint + format icechunk-python
        shell: bash
        working-directory: icechunk
        run: |
          just pre-commit-python

      - name: mypy
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          pip install icechunk['test'] --find-links dist --force-reinstall
          mypy python

      - name: pre-commit lint
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          source .venv/bin/activate
          pip install pre-commit
          SKIP=just-rust pre-commit run --all-files

      - name: Restore cached hypothesis directory
        id: restore-hypothesis-cache
        uses: actions/cache/restore@v4
        with:
          path: icechunk-python/.hypothesis/
          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
          restore-keys: |
            cache-hypothesis-

      - name: describe environment
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          pip list

      - name: pytest
        shell: bash
        working-directory: icechunk-python
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          pytest -n 4

      # explicitly save the cache so it gets updated, also do this even if it fails.
      - name: Save cached hypothesis directory
        id: save-hypothesis-cache
        if: always()
        uses: actions/cache/save@v4
        with:
          path: icechunk-python/.hypothesis/
          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}

  xarray-backends:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          path: "icechunk"

      - name: Stand up MinIO
        working-directory: icechunk
        run: |
          docker compose up -d minio

      - name: Wait for MinIO to be ready
        working-directory: icechunk
        run: |
          for _ in {1..10}; do
              if curl --silent --fail http://minio:9000/minio/health/live; then
              break
              fi
              sleep 3
          done
          docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123

      - uses: actions/checkout@v4
        with:
          repository: "pydata/xarray"
          path: "xarray"
          fetch-depth: 0 # Fetch all history for all branches and tags.

      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          working-directory: icechunk/icechunk-python
          args: --release --out dist --find-interpreter
          sccache: 'true'

      - name: pytest
        shell: bash
        working-directory: icechunk/icechunk-python
        env:
          ICECHUNK_XARRAY_BACKENDS_TESTS: 1
        run: |
          set -e
          python3 -m venv .venv
          source .venv/bin/activate
          pip install icechunk['test'] --find-links dist --force-reinstall
          pip install pytest-mypy-plugins
          # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture
          pytest -c=../../xarray/pyproject.toml -W ignore tests/run_xarray_backends_tests.py