Merge pull request #1476 from dopplershift/io-update

IO Updates

src/metpy/interpolate/points.py

@@ -154,7 +154,7 @@ def natural_neighbor_point(xp, yp, variable, grid_loc, tri, neighbors, circumcen
         except (ZeroDivisionError, qhull.QhullError) as e:
             message = ('Error during processing of a grid. '
                        'Interpolation will continue but be mindful '
-                       'of errors in output. ') + str(e)
+                       f'of errors in output. {e}')
 
             log.warning(message)
             return np.nan
@@ -270,7 +270,7 @@ def inverse_distance_to_points(points, values, xi, r, gamma=None, kappa=None, mi
                 img[idx] = barnes_point(dists, values_subset, kappa, gamma)
 
             else:
-                raise ValueError(str(kind) + ' interpolation not supported.')
+                raise ValueError(f'{kind} interpolation not supported.')
 
     return img
 

src/metpy/io/_tools.py

@@ -6,10 +6,13 @@
 import bz2
 from collections import namedtuple
 import gzip
+from io import BytesIO
 import logging
 from struct import Struct
 import zlib
 
+import numpy as np
+
 log = logging.getLogger(__name__)
 
 
@@ -19,9 +22,33 @@ def open_as_needed(filename, mode='rb'):
     Handles opening with the right class based on the file extension.
 
     """
+    # Handle file-like objects
     if hasattr(filename, 'read'):
+        # See if the file object is really gzipped or bzipped.
+        lead = filename.read(4)
+
+        # If we can seek, seek back to start, otherwise read all the data into an
+        # in-memory file-like object.
+        if hasattr(filename, 'seek'):
+            filename.seek(0)
+        else:
+            filename = BytesIO(lead + filename.read())
+
+        # If the leading bytes match one of the signatures, pass into the appropriate class.
+        try:
+            lead = lead.encode('ascii')
+        except AttributeError:
+            pass
+        if lead.startswith(b'\x1f\x8b'):
+            filename = gzip.GzipFile(fileobj=filename)
+        elif lead.startswith(b'BZh'):
+            filename = bz2.BZ2File(filename)
+
         return filename
 
+    # This will convert pathlib.Path instances to strings
+    filename = str(filename)
+
     if filename.endswith('.bz2'):
         return bz2.BZ2File(filename, mode)
     elif filename.endswith('.gz'):
@@ -74,6 +101,11 @@ def unpack_file(self, fobj):
         """Unpack the next bytes from a file object."""
         return self.unpack(fobj.read(self.size))
 
+    def pack(self, **kwargs):
+        """Pack the arguments into bytes using the structure."""
+        t = self.make_tuple(**kwargs)
+        return super().pack(*t)
+
 
 # This works around times when we have more than 255 items and can't use
 # NamedStruct. This is a CPython limit for arguments.
@@ -93,7 +125,7 @@ def _create(self, items):
         return dict(zip(self._names, items))
 
     def unpack(self, s):
-        """Parse bytes and return a namedtuple."""
+        """Parse bytes and return a dict."""
         return self._create(super().unpack(s))
 
     def unpack_from(self, buff, offset=0):
@@ -171,14 +203,18 @@ class IOBuffer:
     def __init__(self, source):
         """Initialize the IOBuffer with the source data."""
         self._data = bytearray(source)
-        self._offset = 0
-        self.clear_marks()
+        self.reset()
 
     @classmethod
     def fromfile(cls, fobj):
         """Initialize the IOBuffer with the contents of the file object."""
         return cls(fobj.read())
 
+    def reset(self):
+        """Reset buffer back to initial state."""
+        self._offset = 0
+        self.clear_marks()
+
     def set_mark(self):
         """Mark the current location and return its id so that the buffer can return later."""
         self._bookmarks.append(self._offset)
@@ -231,9 +267,15 @@ def read_binary(self, num, item_type='B'):
 
         return list(self.read_struct(Struct(order + '{:d}'.format(int(num)) + item_type)))
 
-    def read_int(self, code):
+    def read_int(self, size, endian, signed):
         """Parse the current buffer offset as the specified integer code."""
-        return self.read_struct(Struct(code))[0]
+        return int.from_bytes(self.read(size), endian, signed=signed)
+
+    def read_array(self, count, dtype):
+        """Read an array of values from the buffer."""
+        ret = np.frombuffer(self._data, offset=self._offset, dtype=dtype, count=count)
+        self.skip(ret.nbytes)
+        return ret
 
     def read(self, num_bytes=None):
         """Read and return the specified bytes from the buffer."""
@@ -304,7 +346,9 @@ def zlib_decompress_all_frames(data):
         try:
             frames.extend(decomp.decompress(data))
             data = decomp.unused_data
+            log.debug('Decompressed zlib frame. %d bytes remain.', len(data))
         except zlib.error:
+            log.debug('Remaining %d bytes are not zlib compressed.', len(data))
             frames.extend(data)
             break
     return frames

src/metpy/io/gini.py

@@ -49,7 +49,7 @@ def _name_lookup(names):
     mapper = dict(zip(range(len(names)), names))
 
     def lookup(val):
-        return mapper.get(val, 'Unknown')
+        return mapper.get(val, 'UnknownValue')
     return lookup
 
 
@@ -101,7 +101,8 @@ class GiniFile(AbstractDataStore):
                 'Sounder (9.71 micron)', 'Sounder (7.43 micron)', 'Sounder (7.02 micron)',
                 'Sounder (6.51 micron)', 'Sounder (4.57 micron)', 'Sounder (4.52 micron)',
                 'Sounder (4.45 micron)', 'Sounder (4.13 micron)', 'Sounder (3.98 micron)',
-                'Sounder (3.74 micron)', 'Sounder (Visible)']
+                # Percent Normal TPW found empirically from Service Change Notice 20-03
+                'Sounder (3.74 micron)', 'Sounder (Visible)', 'Percent Normal TPW']
 
     prod_desc_fmt = NamedStruct([('source', 'b'),
                                  ('creating_entity', 'b', _name_lookup(crafts)),
@@ -341,32 +342,6 @@ def _make_coord_vars(self):
 
         return [('x', x_var), ('y', y_var), ('lon', lon_var), ('lat', lat_var)]
 
-    # FIXME: Work around xarray <=0.10.3 docstring for load angering sphinx
-    # That's the only reason this exists.
-    def load(self):
-        """
-        Load the variables and attributes simultaneously.
-
-        A centralized loading function makes it easier to create
-        data stores that do automatic encoding/decoding.
-
-        For example::
-
-            class SuffixAppendingDataStore(AbstractDataStore):
-
-                def load(self):
-                    variables, attributes = AbstractDataStore.load(self)
-                    variables = {'%s_suffix' % k: v
-                                 for k, v in iteritems(variables)}
-                    attributes = {'%s_suffix' % k: v
-                                  for k, v in iteritems(attributes)}
-                    return variables, attributes
-
-        This function will be called anytime variables or attributes
-        are requested, so care should be taken to make sure its fast.
-        """
-        return super().load()
-
     def get_variables(self):
         """Get all variables in the file.