dev

Author: davidhassell

cf/data/dask_regrid.py

@@ -1,7 +1,7 @@
 """Regridding functions used within a dask graph."""
 
 import numpy as np
-from cfdm.data.dask_utils import cfdm_asanyarray
+from cfdm.data.dask_utils import cfdm_to_memory
 
 
 def regrid(
@@ -175,12 +175,12 @@ def regrid(
     """
     weights, dst_mask = weights_dst_mask
 
-    a = cfdm_asanyarray(a)
+    a = cfdm_to_memory(a)
     if dst_mask is not None:
-        dst_mask = cfdm_asanyarray(dst_mask)
+        dst_mask = cfdm_to_memory(dst_mask)
 
     if ref_src_mask is not None:
-        ref_src_mask = cfdm_asanyarray(ref_src_mask)
+        ref_src_mask = cfdm_to_memory(ref_src_mask)
 
     # ----------------------------------------------------------------
     # Reshape the array into a form suitable for the regridding dot

cf/data/data.py

@@ -1,4 +1,4 @@
-Importre logging
+import logging
 import math
 from functools import partial, reduce
 from itertools import product
@@ -456,12 +456,12 @@ def __contains__(self, value):
                 return False
 
             # 'cf_contains' has its own calls to 'cfdm_to_memory', so
-            # we can set '_force_in_memory=False'.
-            value = value.to_dask_array(_force_in_memory=False)
+            # we can set '_force_to_memory=False'.
+            value = value.to_dask_array(_force_to_memory=False)
 
         # 'cf_contains' has its own calls to 'cfdm_to_memory', so we
-        # can set '_force_in_memory=False'.
-        dx = self.to_dask_array(_force_in_memory=False)
+        # can set '_force_to_memory=False'.
+        dx = self.to_dask_array(_force_to_memory=False)
 
         out_ind = tuple(range(dx.ndim))
         dx_ind = out_ind
@@ -1567,8 +1567,8 @@ def percentile(
             axes = tuple(sorted(d._parse_axes(axes)))
 
         # 'cf_percentile' has its own call to 'cfdm_to_memory', so we
-        # can set '_force_in_memory=False'.
-        dx = d.to_dask_array(_force_in_memory=False)
+        # can set '_force_to_memory=False'.
+        dx = d.to_dask_array(_force_to_memory=False)
         dtype = dx.dtype
         shape = dx.shape
 
@@ -2129,8 +2129,8 @@ def _asdatetime(self, inplace=False):
 
         if not d._isdatetime():
             # 'cf_rt2dt' has its own call to 'cfdm_to_memory', so we
-            # can set '_force_in_memory=False'.
-            dx = d.to_dask_array(_force_in_memory=False)
+            # can set '_force_to_memory=False'.
+            dx = d.to_dask_array(_force_to_memory=False)
             dx = dx.map_blocks(cf_rt2dt, units=units, dtype=object)
             d._set_dask(dx)
 
@@ -2186,8 +2186,8 @@ def _asreftime(self, inplace=False):
 
         if d._isdatetime():
             # 'cf_dt2rt' has its own call to 'cfdm_to_memory', so we
-            # can set '_force_in_memory=False'.
-            dx = d.to_dask_array(_force_in_memory=False)
+            # can set '_force_to_memory=False'.
+            dx = d.to_dask_array(_force_to_memory=False)
             dx = dx.map_blocks(cf_dt2rt, units=units, dtype=float)
             d._set_dask(dx)
 
@@ -2784,8 +2784,8 @@ def _regrid(
             )
 
         # 'regrid' has its own calls to 'cfdm_to_memory', so we can set
-        # '_force_in_memory=False'.
-        dx = self.to_dask_array(_force_in_memory=False)
+        # '_force_to_memory=False'.
+        dx = self.to_dask_array(_force_to_memory=False)
 
         # Rechunk so that each chunk contains data in the form
         # expected by the regrid operator, i.e. the regrid axes all
@@ -3028,10 +3028,10 @@ def concatenate(
             copied = not copy  # to avoid making two copies in a given case
 
         # Get data as dask arrays and apply concatenation
-        # operation. We can set '_force_in_memory=False' because at compute
+        # operation. We can set '_force_to_memory=False' because at compute
         # time the concatenation operation does not need to access the
         # actual data.
-        dxs = [d.to_dask_array(_force_in_memory=False) for d in processed_data]
+        dxs = [d.to_dask_array(_force_to_memory=False) for d in processed_data]
         dx = da.concatenate(dxs, axis=axis)
 
         # Set the CFA write status
@@ -3066,7 +3066,7 @@ def concatenate(
                 # If and only if any two input Data objects have
                 # different __in_memory__ values, then set
                 # in_memory=False on the concatenation.
-                in_memory= False
+                in_memory = False
                 break
 
         # Set the new dask array
@@ -3464,8 +3464,8 @@ def Units(self, value):
             cf_func = partial(cf_units, from_units=old_units, to_units=value)
 
             # 'cf_units' has its own call to 'cfdm_to_memory', so we
-            # can set '_force_in_memory=False'.
-            dx = self.to_dask_array(_force_in_memory=False)
+            # can set '_force_to_memory=False'.
+            dx = self.to_dask_array(_force_to_memory=False)
             dx = dx.map_blocks(cf_func, dtype=dtype)
 
             # Setting equivalent units doesn't affect the CFA write
@@ -3508,8 +3508,8 @@ def is_masked(self):
 
         """
         # 'cf_is_masked' has its own call to 'cfdm_to_memory', so we
-        # can set '_force_in_memory=False'.
-        dx = self.to_dask_array(_force_in_memory=False)
+        # can set '_force_to_memory=False'.
+        dx = self.to_dask_array(_force_to_memory=False)
 
         out_ind = tuple(range(dx.ndim))
         dx_ind = out_ind
@@ -4215,8 +4215,8 @@ def convert_reference_time(
                 d.Units = units0
 
         # 'cf_rt2dt' its own call to 'cfdm_to_memory', so we can set
-        # '_force_in_memory=False'.
-        dx = d.to_dask_array(_force_in_memory=False)
+        # '_force_to_memory=False'.
+        dx = d.to_dask_array(_force_to_memory=False)
 
         # Convert to the correct date-time objects
         dx = dx.map_blocks(cf_rt2dt, units=units0, dtype=object)
@@ -4283,10 +4283,10 @@ def get_deterministic_name(self):
         units = self._Units
 
         # The dask graph is never going to be computed, so we can set
-        # '_force_in_memory=False'.
+        # '_force_to_memory=False'.
         return tokenize(
             self.to_dask_array(
-                _apply_mask_hardness=False, _force_in_memory=False
+                _force_mask_hardness=False, _force_to_memory=False
             ).name,
             units.formatted(definition=True, names=True),
             units._canonical_calendar,
@@ -4324,8 +4324,8 @@ def add_file_location(self, location):
         updated = False
 
         # The dask graph is never going to be computed, so we can set
-        # '_force_in_memory=False'.
-        dsk = self.todict(_force_in_memory=False)
+        # '_force_to_memory=False'.
+        dsk = self.todict(_force_to_memory=False)
         for key, a in dsk.items():
             try:
                 dsk[key] = a.add_file_location(location)
@@ -4338,7 +4338,7 @@ def add_file_location(self, location):
             updated = True
 
         if updated:
-            dx = self.to_dask_array(_force_in_memory=False)
+            dx = self.to_dask_array(_force_to_memory=False)
             dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta)
             self._set_dask(dx, clear=self._NONE, in_memory=None)
 
@@ -5725,8 +5725,8 @@ def unique(self, split_every=None):
         d.soften_mask()
 
         # The applicable chunk function will have its own call to
-        # 'cfdm_to_memory', so we can set '_force_in_memory=False'.
-        dx = d.to_dask_array(_force_in_memory=False)
+        # 'cfdm_to_memory', so we can set '_force_to_memory=False'.
+        dx = d.to_dask_array(_force_to_memory=False)
         dx = Collapse().unique(dx, split_every=split_every)
 
         d._set_dask(dx)
@@ -6240,8 +6240,8 @@ def file_locations(self):
         out = set()
 
         # The dask graph is never going to be computed, so we can set
-        # '_force_in_memory=False'.
-        for key, a in self.todict(_force_in_memory=False).items():
+        # '_force_to_memory=False'.
+        for key, a in self.todict(_force_to_memory=False).items():
             try:
                 out.update(a.file_locations())
             except AttributeError:
@@ -6801,8 +6801,8 @@ def del_file_location(self, location):
         updated = False
 
         # The dask graph is never going to be computed, so we can set
-        # '_force_in_memory=False'.
-        dsk = self.todict(_force_in_memory=False)
+        # '_force_to_memory=False'.
+        dsk = self.todict(_force_to_memory=False)
         for key, a in dsk.items():
             try:
                 dsk[key] = a.del_file_location(location)
@@ -6815,7 +6815,7 @@ def del_file_location(self, location):
             updated = True
 
         if updated:
-            dx = self.to_dask_array(_force_in_memory=False)
+            dx = self.to_dask_array(_force_to_memory=False)
             dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta)
             self._set_dask(dx, clear=self._NONE, in_memory=None)
 
@@ -7852,8 +7852,8 @@ def where(
         # hardness has been applied.
         #
         # 'cf_where' has its own calls to 'cfdm_to_memory', so we can
-        # set '_force_in_memory=False'.
-        dx = d.to_dask_array(_force_in_memory=False)
+        # set '_force_to_memory=False'.
+        dx = d.to_dask_array(_force_to_memory=False)
 
         units = d.Units
 
@@ -7869,8 +7869,8 @@ def where(
         condition = type(self).asdata(condition)
         condition = where_broadcastable(d, condition, "condition")
         # 'cf_where' has its own calls to 'cfdm_to_memory', so we can
-        # set '_force_in_memory=False'.
-        condition = condition.to_dask_array(_force_in_memory=False)
+        # set '_force_to_memory=False'.
+        condition = condition.to_dask_array(_force_to_memory=False)
 
         # If x or y is self then change it to None. This prevents an
         # unnecessary copy; and, at compute time, an unncessary numpy

cf/data/utils.py

@@ -417,10 +417,10 @@ def collapse(
         kwargs["ddof"] = ddof
 
     # The applicable chunk function will have its own call to
-    # 'cfdm_to_memory', so we can set '_force_in_memory=False'. Also,
-    # setting _force_in_memory=False will ensure that any active
+    # 'cfdm_to_memory', so we can set '_force_to_memory=False'. Also,
+    # setting _force_to_memory=False will ensure that any active
     # storage operations are not compromised.
-    dx = d.to_dask_array(_force_in_memory=False)
+    dx = d.to_dask_array(_force_to_memory=False)
     dx = func(dx, **kwargs)
     d._set_dask(dx)
 

cf/read_write/netcdf/netcdfwrite.py

@@ -748,8 +748,8 @@ def _cfa_write_non_standard_terms(
             # missing data.
             #
             # '_cfa_unique' has its own call to 'cfdm_to_memory', so
-            # we can set '_asanyarray=False'.
-            dx = data.to_dask_array(_asanyarray=False)
+            # we can set '_force_to_memory=False'.
+            dx = data.to_dask_array(_force_to_memory=False)
             dx_ind = tuple(range(dx.ndim))
             out_ind = dx_ind
             dx = da.blockwise(
@@ -963,7 +963,7 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         # ------------------------------------------------------------
         dtype = np.dtype(np.int32)
         if (
-            max(data.to_dask_array(_asanyarray=False).chunksize)
+            max(data.to_dask_array(_force_to_memory=False).chunksize)
             > np.iinfo(dtype).max
         ):
             dtype = np.dtype(np.int64)

cf/test/individual_tests.sh

@@ -5,9 +5,6 @@ do
   echo "Running $file"
   python $file
   rc=$?
-#  if [[ $rc != 0 ]]; then
-#    exit $rc
-#  fi
 done
 
 file=setup_create_field.py
@@ -18,14 +15,30 @@ if [[ $rc != 0 ]]; then
   exit $rc
 fi
 
+style="lots"
+
 for file in test_*.py
 do
   echo "Running $file"
   python $file
   rc=$?
   if [[ $rc != 0 ]]; then
-    exit $rc
-    # echo -e "\n\n$file FAILED \n\n"
+      if [[ "$file" == "test_style.py" ]] ; then
+	  style="none"
+      else
+	  exit $rc
+          # echo -e "\n\n$file FAILED \n\n"
+      fi
   fi
 done
 
+echo
+if [[ "$style" == "none" ]] ; then
+    echo "------------------------------------------"
+    echo "All tests passed, APART FROM test_style.py"
+    echo "------------------------------------------"
+else
+    echo "================"
+    echo "All tests passed"    
+    echo "================"
+fi

cf/test/test_Data.py

@@ -4540,7 +4540,7 @@ def test_Data_cull_graph(self):
             len(
                 dict(
                     d.to_dask_array(
-                        _apply_mask_hardness=False, _asanyarray=False
+                        _force_mask_hardness=False, _force_to_memory=False
                     ).dask
                 )
             ),
@@ -4553,7 +4553,7 @@ def test_Data_cull_graph(self):
             len(
                 dict(
                     d.to_dask_array(
-                        _apply_mask_hardness=False, _asanyarray=False
+                        _force_mask_hardness=False, _force_to_memory=False
                     ).dask
                 )
             ),
@@ -4725,7 +4725,7 @@ def test_Data_file_location(self):
     def test_Data_todict(self):
         """Test Data.todict."""
         d = cf.Data([1, 2, 3, 4], chunks=2)
-        key = d.to_dask_array(_apply_mask_hardness=False).name
+        key = d.to_dask_array(_force_mask_hardness=False).name
 
         x = d.todict()
         self.assertIsInstance(x, dict)

cf/test/test_DimensionCoordinate.py

@@ -614,7 +614,7 @@ def test_DimensiconCoordinate_persist(self):
         self.assertEqual(
             len(
                 e.data.to_dask_array(
-                    _apply_mask_hardness=False, _asanyarray=False
+                    _force_mask_hardness=False, _force_to_memory=False
                 ).dask.layers
             ),
             1,

cf/test/test_Field.py

@@ -2767,7 +2767,7 @@ def test_Field_persist(self):
         self.assertGreater(
             len(
                 f.data.to_dask_array(
-                    _apply_mask_hardness=False, _asanyarray=False
+                    _force_mask_hardness=False, _force_to_memory=False
                 ).dask.layers
             ),
             2,
@@ -2778,7 +2778,7 @@ def test_Field_persist(self):
         self.assertEqual(
             len(
                 g.data.to_dask_array(
-                    _apply_mask_hardness=False, _asanyarray=False
+                    _force_mask_hardness=False, _force_to_memory=False
                 ).dask.layers
             ),
             1,