Merge pull request #186 from rigoudyg/process_multiple_ensembles

Process multiple ensembles

Author: rigoudyg

climaf/cache.py

@@ -193,7 +193,9 @@ def do_move(crs, filename, outfilename):
         time.sleep(0.1)
         waited += 1
     # time.sleep(0.5)
-    if os.path.exists(filename):
+    if not os.path.exists(filename):
+        raise Climaf_Cache_Error("File %s wasn't created upstream (or not quick enough)" % (filename))
+    else :
         if stamping is False:
             clogger.debug('No stamping')
             return do_move(crs, filename, outfilename)
@@ -242,8 +244,6 @@ def do_move(crs, filename, outfilename):
                     elif stamping is None:
                         clogger.critical("Cannot stamp by %s" % command)
                         return True
-    else:
-        clogger.error("file %s does not exist (for crs %s)" % (filename, crs))
 
 
 def getCRS(filename):
@@ -427,11 +427,18 @@ def cdrop(obj, rm=True, force=False):
     else:
         clogger.error("%s is not a CliMAF object" % repr(obj))
         return
+    fil=None
     if crs in crs2filename:
         clogger.info("Discarding cached value for %s (except if protected)" % crs)
         fil = crs2filename[crs]
         if not os.path.exists(fil):
             fil = alternate_filename(fil)
+    else:
+        # In case the cache index is not up-to-date
+        fil=hasExactObject(obj)
+        if fil :
+            crs2filename[crs]=fil
+    if fil:
         if rm:
             try:
                 if force:

climaf/driver.py

@@ -115,8 +115,9 @@ def capply_script(script_name, *operands, **parameters):
         opscopy = opscopy[1:]
     else:
         opscopy = list()
-    if True in [isinstance(op, classes.cens) for op in opscopy]:
-        raise Climaf_Driver_Error("Cannot yet have an ensemble as operand except as first one")
+    # Next watch dog disabled for tests !!!!!
+    #if True in [isinstance(op, classes.cens) for op in opscopy]:
+    #    raise Climaf_Driver_Error("Cannot yet have an ensemble as operand except as first one")
     #
     # If first operand is an ensemble, and the script is not ensemble-capable,
     # result is the ensemble of applying the script ot each member of first operand
@@ -311,7 +312,7 @@ def ceval_for_ctree(cobject, userflags=None, format="MaskedArray", deep=None, de
     clogger.debug("Evaluating compound object : " + repr(cobject))
     #################################################################
     if deep is not None:
-        cache.cdrop(cobject.crs)
+        cache.cdrop(cobject)
     #
     clogger.debug("Searching cache for exact object : " + repr(cobject))
     #################################################################
@@ -422,7 +423,7 @@ def ceval_for_scriptChild(cobject, userflags=None, format="MaskedArray", deep=No
     clogger.debug("Evaluating compound object : " + repr(cobject))
     #################################################################
     if deep is not None:
-        cache.cdrop(cobject.crs)
+        cache.cdrop(cobject)
     #
     clogger.debug("Searching cache for exact object : " + repr(cobject))
     #################################################################
@@ -507,7 +508,7 @@ def ceval_for_cpage(cobject, userflags=None, format="MaskedArray", deep=None, de
     clogger.debug("Evaluating compound object : " + repr(cobject))
     #################################################################
     if deep is not None:
-        cache.cdrop(cobject.crs)
+        cache.cdrop(cobject)
     #
     clogger.debug("Searching cache for exact object : " + repr(cobject))
     #################################################################
@@ -550,7 +551,7 @@ def ceval_for_cpage_pdf(cobject, userflags=None, format="MaskedArray", deep=None
     clogger.debug("Evaluating compound object : " + repr(cobject))
     #################################################################
     if deep is not None:
-        cache.cdrop(cobject.crs)
+        cache.cdrop(cobject)
     #
     clogger.debug("Searching cache for exact object : " + repr(cobject))
     #################################################################
@@ -594,7 +595,7 @@ def ceval_for_cens(cobject, userflags=None, format="MaskedArray", deep=None, der
     clogger.debug("Evaluating compound object : " + repr(cobject))
     #################################################################
     if deep is not None:
-        cache.cdrop(cobject.crs)
+        cache.cdrop(cobject)
     #
     clogger.debug("Searching cache for exact object : " + repr(cobject))
     #################################################################
@@ -1210,7 +1211,9 @@ def cfile(object, target=None, ln=None, hard=None, deep=None):
         target_dir = os.path.dirname(target)
         if isinstance(object, climaf.classes.cens):
             raise Climaf_Driver_Error("Cannot yet copy or link result files for an ensemble")
-        if result is not None:
+        if result is None:
+            raise Climaf_Driver_Error("Issue when evaluating %s"%object)        
+        else:
             if ln or hard:
                 if ln and hard:
                     Climaf_Driver_Error("flags ln and hard are mutually exclusive")
@@ -1337,7 +1340,10 @@ def get_fig_sizes(figfile):
     # On some sites, getoutput first lines have warning messages
     # Furthermore, in case of missing file, last line could be an error -> only consider lines beginning with figfile
     output_figsize = getoutput(" ".join(args_figsize)).split("\n")
-    output_figsize = [line for line in output_figsize if line.startswith(figfile)][-1]
+    with_figfile = [l for l in output_figsize if l.startswith(figfile)]
+    if len(with_figfile) == 0 :
+        raise ValueError("No relevant line for fig size in command (%s) output %s"%(args_figsize,output_figsize))
+    output_figsize = with_figfile[-1]
     # comm_figsize = subprocess.Popen(args_figsize, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     # output_figsize = comm_figsize.stdout.read()
     figsize = str(output_figsize).split(" ").pop(2)
@@ -1446,17 +1452,24 @@ def cfilePage(cobj, deep, recurse_list=None):
                      "-pointsize", "%d" % cobj.pt, "-annotate", annotate, '"%s"' % cobj.title])
 
     args.append(out_fig)
-    clogger.debug("Compositing figures : %s" % repr(args))
 
+    command=" ".join(args)
+    clogger.debug("Compositing figures : %s" % command)
+    
     try:
         with open("tmp.err", "w") as fic:
-            out = subprocess.check_output(" ".join(args), shell=True, stderr=fic)
+            out = subprocess.check_output(command, shell=True, stderr=fic)
     except subprocess.CalledProcessError:
         with open("tmp.err") as fic:
             err = fic.read()
-        raise Climaf_Driver_Error("Compositing failed : %s" % err)
-    finally:
-        os.remove("tmp.err")
+        raise Climaf_Driver_Error("Compositing failed : %s for %s" % (err,command))
+
+    # There are cases where subprocess doesn't raise an Error, while compositing failed
+    if not os.path.exists(out_fig) :
+        with open("tmp.err") as fic:
+            err = fic.readlines()
+        raise Climaf_Driver_Error("Compositing failed %s for %s" % (err,command))
+    os.remove("tmp.err")
 
     if cache.register(out_fig, cobj.crs):
         clogger.debug("Registering file %s for cpage %s" % (out_fig, cobj.crs))

climaf/operators.py

@@ -118,9 +118,8 @@ def __init__(self, name, command, format="nc", select=True, canOpendap=False,
              a single string (surrounded with double quotes) will carry
              multiple URLs
 
-          - **mmin** stands for the case where the script accepts an
-            ensemble of datasets (only for first input stream
-            yet). CliMAF will replace the keyword by a string
+          - **mmin** stands for the case where the script accepts as argument 
+            an ensemble of datasets. CliMAF will replace the keyword by a string
             composed of the corresponding input filenames (not surrounded
             by quotes - please add them yourself in declaration); see also
             ``labels`` below
@@ -257,9 +256,6 @@ def __init__(self, name, command, format="nc", select=True, canOpendap=False,
                     serie = (oc.group("serie") is not None)
                     multiple = (oc.group("mult") is not None)
                     if multiple:
-                        if rank != 0:
-                            raise Climaf_Operator_Error(
-                                "Only first operand may accept members")
                         if serie:
                             raise Climaf_Operator_Error(
                                 "Operand %s cannot both accept"

climaf/period.py

@@ -417,7 +417,7 @@ def firstyears(period, nyears):
     Returns a period beginning at PERIOD's begin and which duration is at most NYEARS
     """
     if isinstance(period, six.string_types):
-        period = cperiod(period)
+        period = init_period(period)
     rep = cperiod(period.start, period.end)
     yend = rep.end.year
     ystart = rep.start.year

doc/news.rst

@@ -6,6 +6,15 @@ What's new
 
 Changes, newest first:
 
+- V2.0.1:
+
+  - **Scripts can now process multiple ensembles, and ensembles which are not the first argument**:
+
+    - Each ensemble argument will be replaced by the list of files corresponding to the ensemble,
+      each list being ordered by the ensemble order (see :py:func:`~climaf.classes.cens`);
+      unless intentionnaly, the order is the same among all ensembles (if they are indexed by
+      the same list of member labels) 
+
 - V2.0.0:
 
   - **Python 3 compatibility**:

scripts/gplot.ncl

@@ -25,11 +25,11 @@ begin
     else
 
       if ismissing(str_match(str,".")) then ; integer or string
-        val=stringtoint(str)
-        if ismissing(val) then 
+        a=str_match_regex(str,"^[0-9]*$")
+        if ismissing(a) then 
           return(str)
         else
-          return(val)
+          return(stringtoint(str))
         end if
       else                                  ; float (or string)
         valf=stringtofloat(str)
@@ -804,7 +804,7 @@ begin
           status_exit(1)
         end if
         print("---------------------")
-        print("   STIPPLING FIELD   ")
+        print("   Shade2 FIELD      ")
         print("---------------------")
         useLatDim2=False
         useXY_dummy=False
@@ -817,6 +817,12 @@ begin
         array_shade2=field_reduce(shade2_file, shade2_var, "lin", useXY_dummy, horizontal_dummy, \
         useLatDim2, vprofile_dummy, shade2_field_select, lat_exist2, lon_exist2, proj_data_dummy)
         fld_shade2=array_shade2[0]
+        ; If field min = field max, set some field values in order to force some contours
+        ; (otherwise, no shading occurs)
+        if (min(fld_shade2) .eq. max(fld_shade2)) then
+          fld_shade2(0,0) = str_convert(shade2_below)
+          fld_shade2(0,1) = str_convert(shade2_above)
+        end if
       end if
     end if
   end if
@@ -1294,6 +1300,7 @@ begin
 
       res2@cnFillOn=False 
       res2@cnLineLabelsOn = False
+      res2@cnNoDataLabelOn = False
       res2@cnLineLabelBackgroundColor = -1
       res2@cnInfoLabelString = ""
       res2@cnInfoLabelOn = False
@@ -1330,8 +1337,8 @@ begin
         res2@lbLabelBarOn = False
         res2@cnFillOn=True
         ;res2@cnFillColor    = "black"   ; foreground by default
-        opt = True
-        opt@gsnShadeFillType = "pattern" ; "color" or "pattern"
+        opt_shade = True
+        opt_shade@gsnShadeFillType = "pattern" ; "color" or "pattern"
       end if
 
     end if ; NOT AUX_FIELD
@@ -1699,9 +1706,26 @@ begin
           plot1=gsn_csm_contour(wks,fld,res2)
         end if
       else
-        if isvar("aux_options") then res_list(aux_options, res2) end if
-        plot1=gsn_csm_contour(wks,fld2,res2)
-        if ( isvar("shade_below") .or. isvar("shade_above") ) then plot1 = shading(plot1,fld2,opt) end if
+        if isvar("aux_options") then
+          res_list(aux_options, res2)
+        end if
+        
+        fmin=min(fld2)
+        fmax=max(fld2)
+        contour=True
+        if (ismissing(fmin) .and. ismissing(fmax)) then
+          contour=False
+        else
+          if (fmin .eq. fmax) then
+            contour=False
+          end if
+        end if
+        if (contour) then
+          plot1=gsn_csm_contour(wks,fld2,res2)
+          if ( isvar("shade_below") .or. isvar("shade_above") ) then
+            plot1 = shading(plot1,fld2,opt_shade)
+          end if
+        end if
       end if
 
       if isvar("fld_shade2") then
@@ -1716,6 +1740,7 @@ begin
 
         res_shade2@cnFillOn=True
         res_shade2@cnLineLabelsOn = False
+        res_shade2@cnNoDataLabelOn = False
         res_shade2@cnLineLabelBackgroundColor = -1
         res_shade2@cnInfoLabelString = ""
         res_shade2@cnInfoLabelOn = False
@@ -1819,14 +1844,16 @@ begin
         else
           if (y .eq. "log") then plot1=gsn_csm_pres_hgt(wks,fld2,res2) end if
         end if
-        if ( isvar("shade_below") .or. isvar("shade_above") ) then plot1 = shading(plot1,fld2,opt) end if
+        if ( isvar("shade_below") .or. isvar("shade_above") ) then
+          plot1 = shading(plot1,fld2,opt_shade)
+        end if
       end if
 
     end if ; cross-section or (t,z) profile
 
   end if
 
-  if LEVELS_CONTOURS .or. AUX_FIELD then
+  if (LEVELS_CONTOURS .or. AUX_FIELD) .and. isvar("plot1") then
     overlay(plot0,plot1)   
   end if
 

tests/test_operators.py

@@ -79,8 +79,6 @@ def test_duplicate_entry(self):
         self.assertTrue("mycdo3" in sys.modules["__main__"].__dict__)
 
     def test_errors(self):
-        with self.assertRaises(Climaf_Operator_Error):
-            cscript('mycdo2', '(cdo ${operator} ${in_1} ${mmin_2} ${out})')
         with self.assertRaises(Climaf_Operator_Error):
             cscript('mycdo2', '(cdo ${operator} ${in_1} ${mmins_2} ${out})')
         with self.assertRaises(Climaf_Operator_Error):