MarsVars Tests

.github/workflows/marscalendar_test.yml

@@ -1,4 +1,8 @@
 name: MarsCalendar Test Workflow
+# Cancel any in-progress job or previous runs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 on:
   # Trigger the workflow on push to devel branch
   push:

.github/workflows/marsfiles_test.yml

@@ -72,21 +72,9 @@ jobs:
           # Fix path handling for Windows
           $content = $content -replace "os\.path\.join\(self\.test_dir, file\)", "os.path.normpath(os.path.join(self.test_dir, file))"
           Set-Content tests/test_marsfiles.py -Value $content
-
-      # Run the help message test - fast and doesn't involve file operations
-      - name: Run help message test
-        run: |
-          cd tests
-          python -m unittest test_marsfiles.TestMarsFiles.test_help_message
-
-      # Run the most reliable tests first to get quick feedback
-      - name: Run reliable tests
-        run: |
-          cd tests
-          python -m unittest test_marsfiles.TestMarsFiles.test_help_message
-
-      # Run all tests with increased timeout
-      - name: Run all tests
+
+      # Run the tests
+      - name: Run MarsFiles tests
         timeout-minutes: 25
         run: |
           cd tests

.github/workflows/marspull_test.yml

@@ -1,5 +1,8 @@
 name: MarsPull Test Workflow
-
+# Cancel any in-progress job or previous runs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 on:
   # Trigger the workflow on push to devel branch
   push:
@@ -25,31 +28,29 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ['3.9', '3.10', '3.11']
-
     runs-on: ${{ matrix.os }}
-
     steps:
-    # Checkout the repository
-    - uses: actions/checkout@v3
-
-    # Set up the specified Python version
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    # Install dependencies
-    - name: Install dependencies
-      shell: bash
-      run: |
-        python -m pip install --upgrade pip
-        pip install requests numpy
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
-    # Install the package in editable mode
-    - name: Install package
-      run: pip install -e .
-
-    # Run the tests
-    - name: Run MarsPull tests
-      run: python -m unittest -v tests/test_marspull.py
+      # Checkout the repository
+      - uses: actions/checkout@v3
+      
+      # Set up the specified Python version
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      
+      # Install dependencies
+      - name: Install dependencies
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests numpy
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      
+      # Install the package in editable mode
+      - name: Install package
+        run: pip install -e .
+      
+      # Run the tests
+      - name: Run MarsPull tests
+        run: python -m unittest -v tests/test_marspull.py

.github/workflows/marsvars_test.yml

@@ -0,0 +1,102 @@
+name: MarsVars Test Workflow
+# Cancel any in-progress job or previous runs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+on:
+  # Trigger the workflow on push to devel branch
+  push:
+    branches: [ devel ]
+    paths:
+      - 'bin/MarsVars.py'
+      - 'tests/test_marsvars.py'
+      - '.github/workflows/marsvars_test.yml'
+      - 'amescap/FV3_utils.py'
+      - 'amescap/Script_utils.py'
+      - 'amescap/Ncdf_wrapper.py'
+  # Allow manual triggering of the workflow
+  workflow_dispatch:
+  # Trigger on pull requests that modify relevant files
+  pull_request:
+    branches: [ devel ]
+    paths:
+      - 'bin/MarsVars.py'
+      - 'tests/test_marsvars.py'
+      - '.github/workflows/marsvars_test.yml'
+      - 'amescap/FV3_utils.py'
+      - 'amescap/Script_utils.py'
+      - 'amescap/Ncdf_wrapper.py'
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: true
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache pip dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      # Install dependencies
+      - name: Install dependencies
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install numpy netCDF4 xarray scipy matplotlib
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      # Install the package in editable mode
+      - name: Install package
+        run: pip install -e .
+
+      # Set HOME for Windows since it might be used by the script
+      - name: Set HOME environment variable for Windows
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          echo "HOME=$env:USERPROFILE" >> $env:GITHUB_ENV
+
+      # Set up AmesCAP configuration - handle platform differences
+      - name: Set up AmesCAP configuration
+        shell: bash
+        run: |
+          mkdir -p $HOME/.amescap
+          cp mars_templates/amescap_profile $HOME/.amescap_profile
+
+      # Create a patch for the test file to fix Windows path issues
+      - name: Create test_marsvars.py path fix for Windows
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          $content = Get-Content tests/test_marsvars.py -Raw
+          # Fix path handling for Windows
+          $content = $content -replace "os\.path\.join\(self\.test_dir, file\)", "os.path.normpath(os.path.join(self.test_dir, file))"
+          Set-Content tests/test_marsvars.py -Value $content
+
+      # Run all tests with increased timeout
+      - name: Run all tests
+        timeout-minutes: 25
+        run: |
+          cd tests
+          python -m unittest test_marsvars
+
+      # Report file paths if things fail on Windows
+      - name: Debug Windows paths
+        if: runner.os == 'Windows' && failure()
+        shell: pwsh
+        run: |
+          Write-Host "Current directory: $(Get-Location)"
+          Write-Host "Test directory contents: $(Get-ChildItem -Path tests)"

amescap/Ncdf_wrapper.py

@@ -505,14 +505,15 @@ def write_to_average(self, day_average=5):
 
         # Define dimensions
         for ivar in ["lat", "lon", "pfull", "phalf", "zgrid"]:
-            if ivar =="lon":
-                cart_ax="X"
-            if ivar =="lat":
-                cart_ax="Y"
+            if ivar == "lon":
+                cart_ax = "X"
+            if ivar == "lat":
+                cart_ax = "Y"
             if ivar in ["pfull", "phalf", "zgrid"]:
-                cart_ax="Z"
+                cart_ax = "Z"
             fort_var = self.variables[ivar]
-            Log.add_dim_with_content(dimension_name = ivar, DATAin = fort_var,
+            Log.add_dim_with_content(dimension_name = ivar, 
+                                     DATAin = fort_var,
                                      longname_txt = fort_var.long_name,
                                      units_txt = fort_var.units,
                                      cart_txt = cart_ax)
@@ -527,15 +528,20 @@ def write_to_average(self, day_average=5):
         time_in = self.variables["time"]
         time_out = daily_to_average(varIN = fort_var,
                                     dt_in = (time_in[1]-time_in[0]),
-                                    nday = day_average, trim = True)
-        Log.log_axis1D(variable_name = "time", DATAin = time_out,
-                       dim_name = "time", longname_txt = time_in.long_name,
-                       units_txt = time_in.units, cart_txt = "T")
+                                    nday = day_average, 
+                                    trim = True)
+        Log.log_axis1D(variable_name = "time", 
+                       DATAin = time_out,
+                       dim_name = "time", 
+                       longname_txt = time_in.long_name,
+                       units_txt = time_in.units, 
+                       cart_txt = "T")
 
         # Log static variables
         for ivar in ["pk", "bk"]:
             fort_var = self.variables[ivar]
-            Log.log_variable(variable_name = ivar, DATAin = fort_var,
+            Log.log_variable(variable_name = ivar, 
+                             DATAin = fort_var,
                              dim_array = fort_var.dimensions,
                              longname_txt = fort_var.long_name,
                              units_txt = fort_var.units)
@@ -546,8 +552,10 @@ def write_to_average(self, day_average=5):
                 fort_var = self.variables[ivar]
                 var_out = daily_to_average(varIN = fort_var,
                                            dt_in = (time_in[1]-time_in[0]),
-                                           nday = day_average, trim = True)
-                Log.log_variable(variable_name = ivar, DATAin = var_out,
+                                           nday = day_average, 
+                                           trim = True)
+                Log.log_variable(variable_name = ivar, 
+                                 DATAin = var_out,
                                  dim_array = fort_var.dimensions,
                                  longname_txt = fort_var.long_name,
                                  units_txt = fort_var.units)
@@ -570,7 +578,8 @@ def write_to_diurn(self, day_average=5):
             if ivar in ["pfull" , "phalf", "zgrid"]:
                 cart_ax="Z"
             fort_var=self.variables[ivar]
-            Log.add_dim_with_content(dimension_name = ivar, DATAin = fort_var,
+            Log.add_dim_with_content(dimension_name = ivar, 
+                                     DATAin = fort_var,
                                      longname_txt = fort_var.long_name,
                                      units_txt = fort_var.units,
                                      cart_txt = cart_ax)

amescap/Script_utils.py

@@ -242,21 +242,13 @@ def give_permission(filename):
     except subprocess.CalledProcessError:
         pass
 
-def check_file_tape(fileNcdf, abort=False):
+def check_file_tape(fileNcdf):
     """
-    For use in the NAS environnment only.
-    Checks whether a file is exists on the disk by running the command
-    ``dmls -l`` on NAS. This prevents the program from stalling if the
-    files need to be migrated from the disk to the tape.
-
+    Checks whether a file exists on the disk. If on a NAS system,
+    also checks if the file needs to be migrated from tape.
+
     :param fileNcdf: full path to a netcdf file or a file object with a name attribute
     :type fileNcdf: str or file object
-
-    :param abort: If True, exit the program. Defaults to False
-    :type abort: bool, optional
-
-    :return: None
-
     """
     # Get the filename, whether passed as string or as file object
     filename = fileNcdf if isinstance(fileNcdf, str) else fileNcdf.name
@@ -265,42 +257,58 @@ def check_file_tape(fileNcdf, abort=False):
     if not re.search(".nc", filename):
         print(f"{Red}{filename} is not a netCDF file{Nclr}")
         exit()
-
-    try:
-        # Check if the file exists on the disk, exit otherwise. If it
-        # exists, copy it over from the disk.
-        # Check if dmls command is available
-        subprocess.check_call(["dmls"], shell = True,
-                              stdout = open(os.devnull, "w"),
-                              stderr = open(os.devnull, "w"))
-        # Get the last columns of the ls command (filename and status)
-        cmd_txt = f"dmls -l {filename}| awk '{{print $8,$9}}'"
-        # Get 3-letter identifier from dmls -l command, convert byte to
-        # string for Python3
-        dmls_out = subprocess.check_output(cmd_txt,
-                                           shell = True).decode("utf-8")
-        if dmls_out[1:4] not in ["DUL", "REG", "MIG"]:
-            # If file is OFFLINE, UNMIGRATING etc...
-            if abort :
-                print(f"{Red}*** Error ***\n{dmls_out}\n{dmls_out[6:-1]} "
-                      f"is not available on disk, status is: {dmls_out[0:5]}"
-                      f"CHECK file status with ``dmls -l *.nc`` and run "
-                      f"``dmget *.nc`` to migrate the files.\nExiting now..."
+
+    # First check if the file exists at all
+    if not os.path.isfile(filename):
+        print(f"{Red}File {filename} does not exist{Nclr}")
+        exit()
+
+    # Check if we're on a NAS system by looking for specific environment variables
+    # or filesystem paths that are unique to NAS
+    is_nas = False
+
+    # Method 1: Check for NAS-specific environment variables
+    nas_env_vars = ['PBS_JOBID', 'SGE_ROOT', 'NOBACKUP', 'NASA_ROOT']
+    for var in nas_env_vars:
+        if var in os.environ:
+            is_nas = True
+            break
+
+    # Method 2: Check for NAS-specific directories
+    nas_paths = ['/nobackup', '/nobackupp', '/u/scicon']
+    for path in nas_paths:
+        if os.path.exists(path):
+            is_nas = True
+            break
+
+    # Only perform NAS-specific operations if we're on a NAS system
+    if is_nas:
+        try:
+            # Check if dmls command is available
+            subprocess.check_call(["dmls"], shell=True,
+                                  stdout=open(os.devnull, "w"),
+                                  stderr=open(os.devnull, "w"))
+
+            # Get the last columns of the ls command (filename and status)
+            cmd_txt = f"dmls -l {filename}| awk '{{print $8,$9}}'"
+
+            # Get identifier from dmls -l command
+            dmls_out = subprocess.check_output(cmd_txt, shell=True).decode("utf-8")
+
+            if dmls_out[1:4] not in ["DUL", "REG", "MIG"]:
+                # If file is OFFLINE, UNMIGRATING etc...
+                print(f"{Yellow}*** Warning ***\n{dmls_out[6:-1]} "
+                      f"is not loaded on disk (Status: {dmls_out[0:5]}). "
+                      f"Please migrate it to disk with:\n"
+                      f"``dmget {dmls_out[6:-1]}``\n then try again."
                       f"\n{Nclr}")
                 exit()
-            else:
-                print(f"{Yellow}*** Warning ***\n{dmls_out[6:-1]} is not  "
-                      f"available on the disk. Status: {dmls_out[0:5]}.\n"
-                      f"Consider checking file status with ``dmls -l *.nc`` "
-                      f"and run ``dmget *.nc`` to migrate the files.\n"
-                      f"Waiting for file to be migrated to the disk, this "
-                      f"may take awhile...")
-    except subprocess.CalledProcessError:
-        # Return an eror message
-        if abort:
-             exit()
-        else:
+        except (subprocess.CalledProcessError, FileNotFoundError, IndexError):
+            # If there's any issue with the dmls command, log it but continue
+            if "--debug" in sys.argv:
+                print(f"{Yellow}Warning: Could not check tape status for {filename}{Nclr}")
             pass
+    # Otherwise, we're not on a NAS system, so just continue
 
 
 def get_Ncdf_path(fNcdf):