Update of FAST.Farm for windows (#63)

* Update of FAST.Farm for windows

* Fixing unittests

Author: ebranlard

data/.gitignore

@@ -0,0 +1,2 @@
+*.lin
+*.sum.yaml

openfast_toolbox/__init__.py

@@ -8,7 +8,10 @@
 from .io.fast_input_deck import FASTInputDeck
 
 # Add version to package
-with open(os.path.join(os.path.dirname(__file__), "..", "VERSION")) as fid:
-    __version__ = fid.read().strip()
+try:
+    with open(os.path.join(os.path.dirname(__file__), "..", "VERSION")) as fid:
+        __version__ = fid.read().strip()
+except:
+    __version__='v0.0.0-Unknown'
 
 

openfast_toolbox/case_generation/runner.py

@@ -1,9 +1,11 @@
 # --- For cmd.py
 import os
+import sys
 import subprocess
 import multiprocessing
 
 import collections
+from contextlib import contextmanager
 import glob
 import pandas as pd
 import numpy as np
@@ -14,9 +16,19 @@
 # --- Fast libraries
 from openfast_toolbox.io.fast_input_file import FASTInputFile
 from openfast_toolbox.io.fast_output_file import FASTOutputFile
+from openfast_toolbox.tools.strings import FAIL, OK
 
 FAST_EXE='openfast'
 
+@contextmanager
+def safe_cd(newdir):
+    prevdir = os.getcwd()
+    try:
+        os.chdir(newdir)
+        yield
+    finally:
+        os.chdir(prevdir)
+
 # --------------------------------------------------------------------------------}
 # --- Tools for executing FAST
 # --------------------------------------------------------------------------------{
@@ -72,10 +84,10 @@ def _report(p):
     # --- Giving a summary
     if len(Failed)==0:
         if verbose:
-            print('[ OK ] All simulations run successfully.')
+            OK('All simulations run successfully.')
         return True, Failed
     else:
-        print('[FAIL] {}/{} simulations failed:'.format(len(Failed),len(inputfiles)))
+        FAIL('{}/{} simulations failed:'.format(len(Failed),len(inputfiles)))
         for p in Failed:
             print('      ',p.input_file)
         return False, Failed
@@ -113,34 +125,178 @@ class Dummy():
     p.exe            = exe
     return p
 
-def runBatch(batchfiles, showOutputs=True, showCommand=True, verbose=True):
+def in_jupyter():
+    try:
+        from IPython import get_ipython
+        return 'ipykernel' in str(type(get_ipython()))
+    except:
+        return False
+
+def stream_output(std, buffer_lines=5, prefix='|', line_count=True):
+    if in_jupyter():
+        from IPython.display import display, update_display
+        # --- Jupyter mode ---
+        #handles = [display("DUMMY LINE FOR BUFFER", display_id=True) for _ in range(buffer_lines)]
+        #buffer = []
+        #for line in std:
+        #    line = line.rstrip()
+        #    buffer.append(line)
+        #    if len(buffer) > buffer_lines:
+        #        buffer.pop(0)
+        #    # update all display slots
+        #    for i, handle in enumerate(handles):
+        #        text = buffer[i] if i < len(buffer) else ""
+        #        update_display(text, display_id=handle.display_id)
+
+        # --- alternative using HTML
+        from IPython.display import display, update_display, HTML
+        import html as _html
+
+        # --- Jupyter mode with HTML ---
+        handles = [display(HTML("<pre style='margin:0'>{}DUMMY LINE FOR BUFFER</pre>".format(prefix)), display_id=True)
+                   for _ in range(buffer_lines)]
+        buffer = []
+        iLine=0 
+        for line in std:
+            iLine+=1
+            line = line.rstrip("\r\n")
+            if line_count:
+                line = f"{iLine:>5}: {line}"
+            line = f"{prefix}{line}"
+            buffer.append(line)
+            if len(buffer) > buffer_lines:
+                buffer.pop(0)
+            # update all display slots
+            for i, handle in enumerate(handles):
+                text = buffer[i] if i < len(buffer) else ""
+
+                html_text = "<pre style='margin:0'>{}</pre>".format(_html.escape(text) if text else "&nbsp;")
+                update_display(HTML(html_text), display_id=handle.display_id)
+
+    else:
+        import shutil
+
+        term_width = shutil.get_terminal_size((80, 20)).columns
+        for _ in range(buffer_lines):
+           print('DummyLine')
+        # --- Terminal mode ---
+        buffer = []
+        iLine = 0
+        for line in std:
+            iLine += 1
+            line = line.rstrip()
+            line = line.rstrip()
+            if line_count:
+                line = f"{iLine:>5}: {line}"
+            line = f"{prefix}{line}"
+            line = line[:term_width]  # truncate to fit in one line
+            buffer.append(line)
+            if len(buffer) > buffer_lines:
+                buffer.pop(0)
+            sys.stdout.write("\033[F\033[K" * len(buffer))
+            for l in buffer:
+                print(l)
+            sys.stdout.flush()
+
+def stdHandler(std, method='show'):
+    from collections import deque
+    import sys
+
+    if method =='show':
+        for line in std:
+            print(line, end='')
+        return None
+
+    elif method =='store':
+        return std.read()  # read everything
+
+    elif method.startswith('buffer'):
+        buffer_lines = int(method.split('_')[1])
+        buffer = deque(maxlen=buffer_lines)
+        print('------ Beginning of buffer outputs ----------------------------------')
+        stream_output(std, buffer_lines=buffer_lines)
+        print('------ End of buffer outputs ----------------------------------------')
+        return None
+
+
+
+
+
+def runBatch(batchfiles, showOutputs=True, showCommand=True, verbose=True, newWindow=False, closeWindow=True, shell_cmd='bash', nBuffer=0):
     """ 
     Run one or several batch files
     TODO: error handling, status, parallel
+
+    showOutputs=True => stdout & stderr printed live
+    showOutputs=False => stdout captured internally, stderr printed live
+
+    For output to show in a Jupyter notebook, we cannot use stdout=None, or stderr=None, we need to use Pipe
+
     """
+    import sys
+    windows = (os.name == "nt")
 
     if showOutputs:
         STDOut= None
+        std_method = 'show'
+        if nBuffer>0:
+            std_method=f'buffer_{nBuffer}'
     else:
-        STDOut= open(os.devnull, 'w') 
-
-    curDir = os.getcwd()
-    print('Current directory', curDir)
+        std_method = 'store'
+        #STDOut= open(os.devnull, 'w') 
+        #STDOut= subprocess.DEVNULL
+        STDOut= subprocess.PIPE
 
     def runOneBatch(batchfile):
         batchfile = batchfile.strip()
         batchfile = batchfile.replace('\\','/')
         batchDir  = os.path.dirname(batchfile)
+        batchfileRel = os.path.relpath(batchfile, batchDir)
+        if windows:
+            command = [batchfileRel]
+        else:
+            command = [shell_cmd, batchfileRel]
+
         if showCommand:
-            print('>>>> Running batch file:', batchfile)
-            print('           in directory:', batchDir)
-        try:
-            os.chdir(batchDir)
-            returncode=subprocess.call([batchfile], stdout=STDOut, stderr=subprocess.STDOUT, shell=shell)
-        except:
-            os.chdir(curDir)
-            returncode=-10
-        return returncode
+            print('[INFO] Running batch file:', batchfileRel)
+            print('            using command:', command)
+            print('             in directory:', batchDir)
+
+        if newWindow:
+            # --- Launch a new window (windows only for now)
+            if windows:
+                cmdflag= '/c' if closeWindow else '/k'
+                subprocess.Popen(f'start cmd {cmdflag} {batchfileRel}', shell=True, cwd=batchDir)
+                return 0
+            else:
+                raise NotImplementedError('Running batch in `newWindow` only implemented on Windows.')
+        else:
+            # --- We wait for outputs
+            stdout_data = None
+            with safe_cd(batchDir): # Automatically go back to current directory
+                try:
+                    # --- Option 2
+                    # Use Popen so we can print outputs live
+                    #proc = subprocess.Popen([batchfileRel], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, text=True )
+                    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, text=True )
+                    # Print or store stdout
+                    stdout_data = stdHandler(proc.stdout, method=std_method)
+                    # Always print errors output line by line
+                    for line in proc.stderr:
+                        print(line, end='')
+                    proc.wait()
+                    returncode = proc.returncode
+                    # Dump stdout if there was an error
+                    if returncode != 0 and stdout_data:
+                        print("\n--- Captured stdout ---")
+                        print(stdout_data)
+                except FileNotFoundError as e:
+                    print('[FAIL] Running Batch failed, a file or command was not found see below:\n'+str(e))
+                    returncode=-10
+                except Exception as e:
+                    print('[FAIL] Running Batch failed, see below:\n'+str(e))
+                    returncode=-10
+            return returncode
 
     shell=False
     if isinstance(batchfiles,list):
@@ -152,20 +308,20 @@ def runOneBatch(batchfile):
                 Failed.append(batchfile)
         if len(Failed)>0:
             returncode=1
-            print('[FAIL] {}/{} Batch files failed.'.format(len(Failed),len(batchfiles)))
+            FAIL('{}/{} Batch files failed.'.format(len(Failed),len(batchfiles)))
             print(Failed)
         else:
             returncode=0
             if verbose:
-                print('[ OK ] {} batch filse ran successfully.'.format(len(batchfiles)))
+                OK('{} batch files ran successfully.'.format(len(batchfiles)))
         # TODO
     else:
         returncode = runOneBatch(batchfiles)
         if returncode==0:
             if verbose:
-                print('[ OK ] Batch file ran successfully.')
+                OK('Batch file ran successfully.')
         else:
-            print('[FAIL] Batch file failed:',batchfiles)
+            FAIL('Batch file failed: '+str(batchfiles))
 
     return returncode
 
@@ -200,6 +356,7 @@ def writeBatch(batchfile, fastfiles, fastExe=None, nBatches=1, pause=False, flag
         discard_if_ext_present=None,
         dispatch=False,
         stdOutToFile=False,
+        preCommands=None,
         echo=True):
     """ Write one or several batch file, all paths are written relative to the batch file directory.
     The batch file will consist of lines of the form:
@@ -255,6 +412,8 @@ def writeb(batchfile, fastfiles):
             if not echo:
                 if os.name == 'nt':
                     f.write('@echo off\n')
+            if preCommands is not None:
+                f.write(preCommands+'\n')
             for ff in fastfiles:
                 ff_abs = os.path.abspath(ff)
                 ff_rel = os.path.relpath(ff_abs, batchdir)

openfast_toolbox/converters/openfastToHawc2.py

@@ -25,6 +25,8 @@ def FAST2Hawc2(fstIn, htcTemplate, htcOut, OPfile=None, TwrFAFreq=0.1, TwrSSFreq
     ED    = fst.fst_vt['ElastoDyn']
     AD    = fst.fst_vt['AeroDyn15']
     Bld   = fst.fst_vt['AeroDynBlade']
+    if isinstance(Bld, list):
+        Bld = Bld[0]
     AF    = fst.fst_vt['af_data']
     twrOF = fst.fst_vt['ElastoDynTower']
     BD    = fst.fst_vt['BeamDyn']

openfast_toolbox/fastfarm/AMRWindSimulation.py

@@ -2,6 +2,7 @@
 import os
 
 from openfast_toolbox.fastfarm.FASTFarmCaseCreation import getMultipleOf
+from openfast_toolbox.tools.strings import INFO, FAIL, OK, WARN, print_bold
 
 class AMRWindSimulation:
     '''
@@ -180,20 +181,20 @@ def _checkInputs(self):
         # For convenience, the turbines should not be zero-indexed
         if 'name' in self.wts[0]:
             if self.wts[0]['name'] != 'T1':
-                if self.verbose>0: print(f"WARNING: Recommended turbine numbering should start at 1. Currently it is zero-indexed.")
+                if self.verbose>0: WARN(f"Recommended turbine numbering should start at 1. Currently it is zero-indexed.")
 
 
         # Flags of given/calculated spatial resolution for warning/error printing purposes
         self.given_ds_hr = False
         self.given_ds_lr = False
         warn_msg = ""
         if self.ds_hr is not None:
-            warn_msg += f"WARNING: HIGH-RES SPATIAL RESOLUTION GIVEN. CONVERTING FATAL ERRORS ON HIGH-RES BOXES CHECKS TO WARNINGS."
+            warn_msg += f"HIGH-RES SPATIAL RESOLUTION GIVEN. CONVERTING FATAL ERRORS ON HIGH-RES BOXES CHECKS TO WARNINGS."
             self.given_ds_hr = True
         if self.ds_lr is not None:
-            warn_msg += f"WARNING: LOW-RES SPATIAL RESOLUTION GIVEN. CONVERTING FATAL ERRORS ON LOW-RES BOX CHECKS TO WARNINGS."
+            warn_msg += f"LOW-RES SPATIAL RESOLUTION GIVEN. CONVERTING FATAL ERRORS ON LOW-RES BOX CHECKS TO WARNINGS."
             self.given_ds_lr = True
-        if self.verbose>0: print(f'{warn_msg}\n')
+        if self.verbose>0 and len(warn_msg)>0: WARN(f'{warn_msg}')
         a=1
 
 
@@ -275,6 +276,8 @@ def _calc_sampling_time(self):
             # Calculate dt of high-res per guidelines
             dt_hr_max = 1 / (2 * self.fmax_max)
             self.dt_high_les = getMultipleOf(dt_hr_max, multipleof=self.dt) # Ensure dt_hr is a multiple of the AMR-Wind timestep
+            if self.dt_high_les ==0:
+                raise ValueError(f"AMR-Wind timestep dt={self.dt} is too coarse for high resolution domain! The time step based on `fmax` is {dt_hr_max}, which is too small to be rounded as a multiple of dt.")
         else:
             # The dt of high-res is given
             self.dt_high_les = self.dt_hr
@@ -339,7 +342,7 @@ def _calc_grid_resolution(self):
             error_msg = f"AMR-Wind grid spacing of {self.ds_max_at_hr_level} m at the high-res box level of {self.level_hr} is too coarse for "\
                         f"the high resolution domain. AMR-Wind grid spacing at level {self.level_hr} must be at least {self.ds_high_les} m."
             if self.given_ds_hr:
-                if self.verbose>0: print(f'WARNING: {error_msg}')
+                if self.verbose>0 and len(error_msg)>0: WARN(f'{error_msg}')
             else:
                 raise ValueError(error_msg)
 
@@ -351,7 +354,7 @@ def _calc_grid_resolution(self):
                         f"to the call to `AMRWindSimulation`. Note that sampled values will no longer be at the cell centers, as you will be requesting "\
                         f"sampling at {self.ds_low_les} m while the underlying grid will be at {self.ds_max_at_lr_level} m.\n --- SUPRESSING FURTHER ERRORS ---"
             if self.given_ds_lr:
-                if self.verbose>0: print(f'WARNING: {error_msg}')
+                if self.verbose>0 and len(error_msg)>0: WARN(f'{error_msg}')
             else:
                 raise ValueError(error_msg)
 
@@ -382,8 +385,8 @@ def _calc_grid_resolution_lr(self):
         # For curled wake model: ds_lr_max = self.cmeander_max * self.dt_low_les * self.vhub**2 / 5
 
         ds_low_les = getMultipleOf(ds_lr_max, multipleof=self.ds_hr) 
-        if self.verbose>0: print(f"Low-res spatial resolution should be at least {ds_lr_max:.2f} m, but since it needs to be a multiple of high-res "\
-              f"resolution of {self.ds_hr}, we pick ds_low to be {ds_low_les} m")
+        if self.verbose>0: INFO(f"Low-res spatial resolution (ds_low) should be >={ds_lr_max:.2f} m.\nTo be a multiple of ds_high"\
+              f"={self.ds_hr} m, we pick ds_low={ds_low_les} m")
 
         #self.ds_lr = self.ds_low_les
         return ds_low_les
@@ -636,7 +639,7 @@ def _check_grid_placement_single(self, sampling_xyzgrid_lhr, amr_xyzgrid_at_lhr_
         		f"AMR-Wind grid (subset): {amr_xyzgrid_at_lhr_level_cc[amr_index  ]}, {amr_xyzgrid_at_lhr_level_cc[amr_index+1]}, "\
         		f"{amr_xyzgrid_at_lhr_level_cc[amr_index+2]}, {amr_xyzgrid_at_lhr_level_cc[amr_index+3]}, ..."
             if self.given_ds_lr:
-                if self.verbose>0: print(f'WARNING: {error_msg}')
+                if self.verbose>0 and len(error_msg)>0: WARN(f'{error_msg}')
             else:
                 raise ValueError(error_msg)