Merge remote-tracking branch 'upstream/main' into groupby-save-codes-new

* upstream/main: (39 commits)
  Support the new compression argument in netCDF4 > 1.6.0 (pydata#6981)
  Remove setuptools-scm-git-archive, require setuptools-scm>=7 (pydata#7253)
  Fix mypy failures (pydata#7343)
  Docs: add example of writing and reading groups to netcdf (pydata#7338)
  Reset file pointer to 0 when reading file stream (pydata#7304)
  Enable mypy warn unused ignores (pydata#7335)
  Optimize some copying (pydata#7209)
  Add parse_dims func (pydata#7051)
  Fix coordinate attr handling in `xr.where(..., keep_attrs=True)` (pydata#7229)
  Remove code used to support h5py<2.10.0 (pydata#7334)
  [pre-commit.ci] pre-commit autoupdate (pydata#7330)
  Fix PR number in what’s new (pydata#7331)
  Enable `origin` and `offset` arguments in `resample` (pydata#7284)
  fix doctests: supress urllib3 warning (pydata#7326)
  fix flake8 config (pydata#7321)
  implement Zarr v3 spec support (pydata#6475)
  Fix polyval overloads (pydata#7315)
  deprecate pynio backend (pydata#7301)
  mypy - Remove some ignored packages and modules (pydata#7319)
  Switch to T_DataArray in .coords (pydata#7285)
  ...

.git_archival.txt

@@ -1 +1,4 @@
+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$
 ref-names: $Format:%D$

.github/workflows/ci-additional.yaml

@@ -119,7 +119,7 @@ jobs:
           python xarray/util/print_versions.py
       - name: Install mypy
         run: |
-          python -m pip install mypy
+          python -m pip install 'mypy<0.990'
 
       - name: Run mypy
         run: |
@@ -134,6 +134,64 @@ jobs:
           name: codecov-umbrella
           fail_ci_if_error: false
 
+  mypy38:
+    name: Mypy 3.8
+    runs-on: "ubuntu-latest"
+    needs: detect-ci-trigger
+    # temporarily skipping due to https://github.com/pydata/xarray/issues/6551
+    if: needs.detect-ci-trigger.outputs.triggered == 'false'
+    defaults:
+      run:
+        shell: bash -l {0}
+    env:
+      CONDA_ENV_FILE: ci/requirements/environment.yml
+      PYTHON_VERSION: "3.8"
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Fetch all history for all branches and tags.
+
+      - name: set environment variables
+        run: |
+          echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+      - name: Setup micromamba
+        uses: mamba-org/provision-with-micromamba@v14
+        with:
+          environment-file: ${{env.CONDA_ENV_FILE}}
+          environment-name: xarray-tests
+          extra-specs: |
+            python=${{env.PYTHON_VERSION}}
+            conda
+          cache-env: true
+          cache-env-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+      - name: Install xarray
+        run: |
+          python -m pip install --no-deps -e .
+      - name: Version info
+        run: |
+          conda info -a
+          conda list
+          python xarray/util/print_versions.py
+      - name: Install mypy
+        run: |
+          python -m pip install 'mypy<0.990'
+
+      - name: Run mypy
+        run: |
+          python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
+
+      - name: Upload mypy coverage to Codecov
+        uses: codecov/codecov-action@v3.1.1
+        with:
+          file: mypy_report/cobertura.xml
+          flags: mypy38
+          env_vars: PYTHON_VERSION
+          name: codecov-umbrella
+          fail_ci_if_error: false
+
+
+
   min-version-policy:
     name: Minimum Version Policy
     runs-on: "ubuntu-latest"

.github/workflows/upstream-dev-ci.yaml

@@ -80,6 +80,7 @@ jobs:
         if: success()
         id: status
         run: |
+          export ZARR_V3_EXPERIMENTAL_API=1
           python -m pytest --timeout=60 -rf \
             --report-log output-${{ matrix.python-version }}-log.jsonl
       - name: Generate and publish the report
@@ -88,6 +89,6 @@ jobs:
           && steps.status.outcome == 'failure'
           && github.event_name == 'schedule'
           && github.repository_owner == 'pydata'
-        uses: xarray-contrib/issue-from-pytest-log@v1.1
+        uses: xarray-contrib/issue-from-pytest-log@v1
         with:
           log-path: output-${{ matrix.python-version }}-log.jsonl

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 # https://pre-commit.com/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -10,7 +10,7 @@ repos:
       - id: mixed-line-ending
     # This wants to go before isort & flake8
   - repo: https://github.com/PyCQA/autoflake
-    rev: "v1.7.7"
+    rev: "v2.0.0"
     hooks:
       - id: autoflake # isort should run before black as black sometimes tweaks the isort output
         args: ["--in-place", "--ignore-init-module-imports"]
@@ -19,7 +19,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.2.0
+    rev: v3.2.2
     hooks:
       - id: pyupgrade
         args:
@@ -31,14 +31,14 @@ repos:
       - id: black
       - id: black-jupyter
   - repo: https://github.com/keewis/blackdoc
-    rev: v0.3.7
+    rev: v0.3.8
     hooks:
       - id: blackdoc
         exclude: "generate_aggregations.py"
         additional_dependencies: ["black==22.10.0"]
       - id: blackdoc-autoupdate-black
   - repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
       - id: flake8
   # - repo: https://github.com/Carreau/velin
@@ -47,7 +47,7 @@ repos:
   #     - id: velin
   #       args: ["--write", "--compact"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.982
+    rev: v0.991
     hooks:
       - id: mypy
         # Copied from setup.cfg

asv_bench/asv.conf.json

@@ -58,8 +58,7 @@
     //     "pip+emcee": [""],   // emcee is only available for install with pip.
     // },
     "matrix": {
-        "setuptools_scm[toml]": [""],  // GH6609
-        "setuptools_scm_git_archive": [""],  // GH6609
+        "setuptools_scm": [""],  // GH6609
         "numpy": [""],
         "pandas": [""],
         "netcdf4": [""],

asv_bench/benchmarks/renaming.py

@@ -0,0 +1,27 @@
+import numpy as np
+
+import xarray as xr
+
+
+class SwapDims:
+    param_names = ["size"]
+    params = [[int(1e3), int(1e5), int(1e7)]]
+
+    def setup(self, size: int) -> None:
+        self.ds = xr.Dataset(
+            {"a": (("x", "t"), np.ones((size, 2)))},
+            coords={
+                "x": np.arange(size),
+                "y": np.arange(size),
+                "z": np.arange(size),
+                "x2": ("x", np.arange(size)),
+                "y2": ("y", np.arange(size)),
+                "z2": ("z", np.arange(size)),
+            },
+        )
+
+    def time_swap_dims(self, size: int) -> None:
+        self.ds.swap_dims({"x": "xn", "y": "yn", "z": "zn"})
+
+    def time_swap_dims_newindex(self, size: int) -> None:
+        self.ds.swap_dims({"x": "x2", "y": "y2", "z": "z2"})

ci/min_deps_check.py

@@ -30,12 +30,12 @@
 POLICY_MONTHS = {"python": 24, "numpy": 18}
 POLICY_MONTHS_DEFAULT = 12
 POLICY_OVERRIDE: Dict[str, Tuple[int, int]] = {}
-has_errors = False
+errors = []
 
 
 def error(msg: str) -> None:
-    global has_errors
-    has_errors = True
+    global errors
+    errors.append(msg)
     print("ERROR:", msg)
 
 
@@ -48,7 +48,7 @@ def parse_requirements(fname) -> Iterator[Tuple[str, int, int, Optional[int]]]:
 
     Yield (package name, major version, minor version, [patch version])
     """
-    global has_errors
+    global errors
 
     with open(fname) as fh:
         contents = yaml.safe_load(fh)
@@ -157,9 +157,9 @@ def process_pkg(
         status = "> (!)"
         delta = relativedelta(datetime.now(), policy_published_actual).normalized()
         n_months = delta.years * 12 + delta.months
-        error(
-            f"Package is too new: {pkg}={req_major}.{req_minor} was "
-            f"published on {versions[req_major, req_minor]:%Y-%m-%d} "
+        warning(
+            f"Package is too new: {pkg}={policy_major}.{policy_minor} was "
+            f"published on {versions[policy_major, policy_minor]:%Y-%m-%d} "
             f"which was {n_months} months ago (policy is {policy_months} months)"
         )
     else:
@@ -193,13 +193,18 @@ def main() -> None:
         for pkg, major, minor, patch in parse_requirements(fname)
     ]
 
-    print("Package           Required             Policy               Status")
+    print("\nPackage           Required             Policy               Status")
     print("----------------- -------------------- -------------------- ------")
     fmt = "{:17} {:7} ({:10}) {:7} ({:10}) {}"
     for row in rows:
         print(fmt.format(*row))
 
-    assert not has_errors
+    if errors:
+        print("\nErrors:")
+        print("-------")
+        for i, e in enumerate(errors):
+            print(f"{i+1}. {e}")
+        sys.exit(1)
 
 
 if __name__ == "__main__":

ci/requirements/all-but-dask.yml

@@ -30,7 +30,6 @@ dependencies:
   - pip
   - pseudonetcdf
   - pydap
-  # - pynio  # not compatible with netCDF4>1.5.3, see #4491
   - pytest
   - pytest-cov
   - pytest-env

ci/requirements/bare-minimum.yml

@@ -11,5 +11,5 @@ dependencies:
   - pytest-env
   - pytest-xdist
   - numpy=1.20
-  - packaging=21.0
+  - packaging=21.3
   - pandas=1.3

ci/requirements/environment-windows.yml

@@ -30,7 +30,6 @@ dependencies:
   - pre-commit
   - pseudonetcdf
   - pydap
-  # - pynio  # Not available on Windows
   - pytest
   - pytest-cov
   - pytest-env

ci/requirements/environment.yml

@@ -34,7 +34,6 @@ dependencies:
   - pre-commit
   - pseudonetcdf
   - pydap
-  # - pynio  # not compatible with netCDF4>1.5.3, see #4491
   - pytest
   - pytest-cov
   - pytest-env

ci/requirements/min-all-deps.yml

@@ -8,40 +8,38 @@ dependencies:
   # When upgrading python, numpy, or pandas, must also change
   # doc/user-guide/installing.rst, doc/user-guide/plotting.rst and setup.py.
   - python=3.8
-  - boto3=1.18
+  - boto3=1.20
   - bottleneck=1.3
-  - cartopy=0.19
+  - cartopy=0.20
   - cdms2=3.1
   - cfgrib=0.9
   - cftime=1.5
   - coveralls
-  - dask-core=2021.09
-  - distributed=2021.09
+  - dask-core=2021.11
+  - distributed=2021.11
   - flox=0.5
   - h5netcdf=0.11
-  # h5py and hdf5 tend to cause conflicrs
+  # h5py and hdf5 tend to cause conflicts
   # for e.g. hdf5 1.12 conflicts with h5py=3.1
   # prioritize bumping other packages instead
-  - h5py=3.1
-  - hdf5=1.10
+  - h5py=3.6
+  - hdf5=1.12
   - hypothesis
   - iris=3.1
   - lxml=4.6  # Optional dep of pydap
-  - matplotlib-base=3.4
-  - nc-time-axis=1.3
+  - matplotlib-base=3.5
+  - nc-time-axis=1.4
   # netcdf follows a 1.major.minor[.patch] convention
   # (see https://github.com/Unidata/netcdf4-python/issues/1090)
-  # bumping the netCDF4 version is currently blocked by #4491
-  - netcdf4=1.5.3
+  - netcdf4=1.5.7
   - numba=0.54
   - numpy=1.20
-  - packaging=21.0
+  - packaging=21.3
   - pandas=1.3
-  - pint=0.17
+  - pint=0.18
   - pip
-  - pseudonetcdf=3.1
+  - pseudonetcdf=3.2
   - pydap=3.2
-  # - pynio=1.5.5
   - pytest
   - pytest-cov
   - pytest-env
@@ -51,7 +49,7 @@ dependencies:
   - seaborn=0.11
   - sparse=0.13
   - toolz=0.11
-  - typing_extensions=3.10
+  - typing_extensions=4.0
   - zarr=2.10
   - pip:
     - numbagg==0.1

doc/getting-started-guide/installing.rst

@@ -8,7 +8,7 @@ Required dependencies
 
 - Python (3.8 or later)
 - `numpy <https://www.numpy.org/>`__ (1.20 or later)
-- `packaging <https://packaging.pypa.io/en/latest/#>`__ (21.0 or later)
+- `packaging <https://packaging.pypa.io/en/latest/#>`__ (21.3 or later)
 - `pandas <https://pandas.pydata.org/>`__ (1.3 or later)
 
 .. _optional-dependencies: