Merge pull request #86 from falconstryker/devel

Devel

Author: falconstryker

.github/workflows/marsformat_test.yml

@@ -0,0 +1,77 @@
+name: MarsFormat Test Workflow
+
+on:
+  # Trigger the workflow on push to devel branch
+  push:
+    branches: [ devel ]
+    paths:
+      - 'bin/MarsFormat.py'
+      - 'tests/test_marsformat.py'
+      - '.github/workflows/marsformat_test.yml'
+  # Allow manual triggering of the workflow
+  workflow_dispatch:
+  # Trigger on pull requests that modify MarsFormat or tests
+  pull_request:
+    branches: [ devel ]
+    paths:
+      - 'bin/MarsFormat.py'
+      - 'tests/test_marsformat.py'
+      - '.github/workflows/marsformat_test.yml'
+
+jobs:
+  test:
+    # Run on multiple OS and Python versions for comprehensive testing
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.9', '3.10', '3.11']
+    
+    runs-on: ${{ matrix.os }}
+    
+    steps:
+      # Checkout the repository
+      - uses: actions/checkout@v3
+      
+      # Set up the specified Python version
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      
+      # Install dependencies
+      - name: Install dependencies
+        shell: bash  
+        run: |
+          python -m pip install --upgrade pip
+          pip install numpy xarray netCDF4 matplotlib scipy
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      
+      # Install the package in editable mode
+      - name: Install package
+        run: pip install -e .
+      
+      # Set HOME for Windows since it might be used by the script
+      - name: Set HOME environment variable for Windows
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          echo "HOME=$env:USERPROFILE" >> $env:GITHUB_ENV
+      
+      # Set up AmesCAP configuration - handle platform differences
+      - name: Set up AmesCAP configuration
+        shell: bash
+        run: |
+          mkdir -p $HOME/.amescap
+          cp mars_templates/amescap_profile $HOME/.amescap_profile
+    
+      # Print out environment info
+      - name: Show environment info
+        run: |
+          python -c "import os, sys, numpy, netCDF4, xarray; print(f'Python: {sys.version}, NumPy: {numpy.__version__}, NetCDF4: {netCDF4.__version__}, xarray: {xarray.__version__}')"
+          echo "Working directory: $(pwd)"
+          echo "Home directory: $HOME"
+          echo "Environment variables: $(env)"
+            
+      # Run the tests
+      - name: Run MarsFormat tests
+        run: python -m unittest -v tests/test_marsformat.py

.github/workflows/marspull_test.yml

@@ -8,11 +8,9 @@ on:
       - 'bin/MarsPull.py'
       - 'tests/test_marspull.py'
       - '.github/workflows/marspull_test.yml'
-
   # Allow manual triggering of the workflow
   workflow_dispatch:
-
-  # Trigger on pull requests that modify MarsPull or tests
+  # Trigger on pull requests that modify MarsFormat or tests
   pull_request:
     branches: [ devel ]
     paths:
@@ -25,7 +23,7 @@ jobs:
     # Run on multiple OS and Python versions for comprehensive testing
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ['3.9', '3.10', '3.11']
 
     runs-on: ${{ matrix.os }}
@@ -42,6 +40,7 @@ jobs:
 
     # Install dependencies
     - name: Install dependencies
+      shell: bash
       run: |
         python -m pip install --upgrade pip
         pip install requests numpy
@@ -53,5 +52,4 @@ jobs:
 
     # Run the tests
     - name: Run MarsPull tests
-      run: |
-        python -m unittest tests/test_marspull.py
+      run: python -m unittest -v tests/test_marspull.py

bin/MarsFiles.py

@@ -177,6 +177,7 @@ def wrapper(*args, **kwargs):
         f"  - ``daily``  : instantaneous data\n"
         f"  - ``diurn``  : 5-sol averages binned by time of day\n"
         f"  - ``average``: 5-sol averages\n"
+        f"Works on 'fort.11' files only.\n"
         f"{Green}Example:\n"
         f"> MarsFiles fort.11_* -bin fixed daily diurn average\n"
         f"> MarsFiles fort.11_* -bin fixed diurn"
@@ -187,7 +188,7 @@ def wrapper(*args, **kwargs):
 parser.add_argument('-c', '--concatenate', action='store_true',
     help=(
         f"Combine sequential files of the same type into one file.\n"
-        f"and ``diurn``).\n"
+        f"Works on 'daily', 'diurn', and 'average' files.\n"
         f"{Green}Example:\n"
         f"> ls\n"
         f"00334.atmos_average.nc 00668.atmos_average.nc\n"
@@ -214,6 +215,7 @@ def wrapper(*args, **kwargs):
         f"Defaults to ``areo`` (Ls) unless otherwise specified using "
         f"-dim.\nIf a file contains multiple Mars Years of data, the "
         f"function splits the file in the first Mars Year.\n"
+        f"Works on 'daily', 'diurn', and 'average' files.\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_average.nc --split 0 90\n"
         f"> MarsFiles 01336.atmos_average.nc --split 270\n"
@@ -232,6 +234,7 @@ def wrapper(*args, **kwargs):
         f"time.\nUseful for comparing data at a specific time of day "
         f"across all longitudes.\nWorks on vertically interpolated "
         f"files.\n"
+        f"Works on 'diurn' files only.\n"
         f"{Yellow}Generates a new file ending in ``_T.nc``{Nclr}\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_diurn.nc -t"
@@ -253,7 +256,7 @@ def wrapper(*args, **kwargs):
         f"{Yellow}Generates a new file ending in ``_to_average.nc``\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_daily.nc -ba {Blue}(5-sol bin){Green}\n"
-        f"> MarsFiles 01336.atmos_daily_pstd.nc -ba 10 {Blue}(10-sol bin)"
+        f"> MarsFiles 01336.atmos_daily.nc -ba 10 {Blue}(10-sol bin)"
         f"{Nclr}\n\n"
     )
 )
@@ -270,9 +273,9 @@ def wrapper(*args, **kwargs):
         f"{Yellow}Generates a new file ending in ``_to_diurn.nc``\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_daily.nc -bd {Blue}(5-sol bin){Green}\n"
-        f"> MarsFiles 01336.atmos_daily_pstd.nc -bd -ba 10 "
+        f"> MarsFiles 01336.atmos_daily.nc -bd -ba 10 "
         f"{Blue}(10-sol bin){Green}\n"
-        f"> MarsFiles 01336.atmos_daily_pstd.nc -bd -ba 1 "
+        f"> MarsFiles 01336.atmos_daily.nc -bd -ba 1 "
         f"{Blue}(no binning, mimics raw Legacy output)"
         f"{Nclr}\n\n"
     )
@@ -284,9 +287,9 @@ def wrapper(*args, **kwargs):
     nargs='+', type=float,
     help=(
         f"Temporal high-pass filtering: removes low-frequencies \n"
-        f"Only works with 'daily' or 'average' files. Requires a cutoff frequency "
-        f"in Sols.\n"
-        f"Data is detrended before filtering. Use ``--add_trend`` \n"
+        f"Only works with 'daily' or 'average' files. Requires a cutoff "
+        f"frequency in Sols.\n"
+        f"Data is detrended before filtering. Use ``-add_trend`` \n"
         f"to add the original linear trend to the amplitudes \n"
         f"\n{Yellow}Generates a new file ending in ``_hpt.nc``\n"
         f"{Green}Example:\n"
@@ -301,9 +304,9 @@ def wrapper(*args, **kwargs):
     nargs='+', type=float,
     help=(
         f"Temporal low-pass filtering: removes high-frequencies \n"
-        f"Only works with 'daily' or 'average' files. Requires a cutoff frequency "
-        f"in Sols.\n"
-        f"Data is detrended before filtering. Use ``--add_trend`` \n"
+        f"Only works with 'daily' or 'average' files. Requires a cutoff "
+        f"frequency in Sols.\n"
+        f"Data is detrended before filtering. Use ``-add_trend`` \n"
         f"to add the original linear trend to the amplitudes \n"
         f"\n{Yellow}Generates a new file ending in ``_lpt.nc``\n"
         f"{Green}Example:\n"
@@ -320,7 +323,7 @@ def wrapper(*args, **kwargs):
         f"Temporal band-pass filtering"
         f"specified by user.\nOnly works with 'daily' or 'average' "
         f"files. Requires two cutoff frequencies in Sols.\n"
-        f"Data is detrended before filtering. Use ``--add_trend`` \n"
+        f"Data is detrended before filtering. Use ``-add_trend`` \n"
         f"to add the original linear trend to the amplitudes \n"
         f"\n{Yellow}Generates a new file ending in ``bpt.nc``\n"
         f"{Green}Example:\n"
@@ -340,7 +343,7 @@ def wrapper(*args, **kwargs):
         f"in Sols.\n"
         f"{Yellow}Generates a new file ending in ``_hps.nc``\n"
         f"{Green}Example:\n"
-        f"> MarsFiles 01336.atmos_daily.nc -hps 10 --add_trend\n"
+        f"> MarsFiles 01336.atmos_daily.nc -hps 10 -add_trend\n"
         f"{Nclr}\n\n"
     )
 )
@@ -355,7 +358,7 @@ def wrapper(*args, **kwargs):
         f"cutoff frequency in Sols.\n"
         f"{Yellow}Generates a new file ending in ``_lps.nc``\n"
         f"{Green}Example:\n"
-        f"> MarsFiles 01336.atmos_daily.nc -lps 20 --add_trend\n"
+        f"> MarsFiles 01336.atmos_daily.nc -lps 20 -add_trend\n"
         f"{Nclr}\n\n"
     )
 )
@@ -370,7 +373,7 @@ def wrapper(*args, **kwargs):
         f"cutoff frequency in Sols.\nData detrended before filtering.\n"
         f"{Yellow}Generates a new file ending in ``_bps.nc``\n"
         f"{Green}Example:\n"
-        f"> MarsFiles 01336.atmos_daily.nc -bps 10 20 --add_trend\n"
+        f"> MarsFiles 01336.atmos_daily.nc -bps 10 20 -add_trend\n"
         f"{Nclr}\n\n"
     )
 )
@@ -384,6 +387,7 @@ def wrapper(*args, **kwargs):
         f"harmonics.\nOnly works with 'diurn' files.\nReturns the phase "
         f"and amplitude of the variable.\n"
         f"N = 1 diurnal tide, N = 2 semi-diurnal, etc.\n"
+        f"Works on 'diurn' files only.\n"
         f"{Yellow}Generates a new file ending in ``_tide_decomp.nc``\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_diurn.nc -tide 2 -incl ps temp\n"
@@ -402,6 +406,7 @@ def wrapper(*args, **kwargs):
         f"user-provided source file.\nBoth files must have the same "
         f"vertical dimensions (i.e., should be vertically\ninterpolated "
         f"to the same standard grid [zstd, zagl, pstd, etc.].\n"
+        f"Works on 'daily', 'diurn', and 'average' files.\n"
         f"{Yellow}Generates a new file ending in ``_regrid.nc``\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_average_pstd.nc -regrid "
@@ -421,6 +426,7 @@ def wrapper(*args, **kwargs):
         f"Zonally average the entire file over the longitudinal "
         f"dimension.\nDoes not work if the longitude dimension has "
         f"length <= 1.\n"
+        f"Works on 'daily', 'diurn', and 'average' files.\n"
         f"{Yellow}Generates a new file ending in ``_zavg.nc``\n"
         f"{Green}Example:\n"
         "> MarsFiles 01336.atmos_diurn.nc -zavg"
@@ -435,6 +441,7 @@ def wrapper(*args, **kwargs):
         f"Must be used with [-split --split]. Flag indicates the "
         f"dimension on which to trim the file.\nAcceptable values are "
         f"'areo', 'lev', 'lat', 'lon'. Default = 'areo'.\n"
+        f"Works on 'daily', 'diurn', and 'average' files.\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_average.nc --split 0 90 -dim areo\n"
         f"> MarsFiles 01336.atmos_average.nc --split -70 -dim lat"
@@ -450,6 +457,7 @@ def wrapper(*args, **kwargs):
         f"For use with ``-tide``: Returns the result in percent "
         f"amplitude.\n"
         f"N = 1 diurnal tide, N = 2 semi-diurnal, etc.\n"
+        f"Works on 'diurn' files only.\n"
         f"{Yellow}Generates a new file ending in ``_norm.nc``\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_diurn.nc -tide 6 -incl ps "
@@ -482,6 +490,7 @@ def wrapper(*args, **kwargs):
     nargs=0,
     help=(
         f"Reconstructs the first N harmonics.\n"
+        f"Works on 'diurn' files only.\n"
         f"{Yellow}Generates a new file ending in ``_reconstruct.nc``\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_diurn.nc -tide 6 -incl ps temp "
@@ -495,8 +504,7 @@ def wrapper(*args, **kwargs):
         f"Flag to use only the variables specified in a calculation.\n"
         f"All dimensional and 1D variables are ported to the new file "
         f"automatically.\n"
-        f"{Yellow}Overwrites existing target file. To override, use "
-        f"-ext.{Nclr}\n"
+        f"Works on 'daily', 'diurn', and 'average' files.\n"
         f"{Green}Example:\n"
         f"> MarsFiles 01336.atmos_daily.nc -ba -incl ps temp ts"
         f"{Nclr}\n\n"
@@ -510,8 +518,8 @@ def wrapper(*args, **kwargs):
         f"CAP to create a new file with the extension name specified "
         f"here.\n"
         f"{Green}Example:\n"
-        f"> MarsFiles 00334.atmos_average.nc -c -ext _comb\n"
-        f"{Blue}(Produces 00334.atmos_average_comb.nc and "
+        f"> MarsFiles 01336.atmos_average.nc -c -ext _my_concat\n"
+        f"{Blue}(Produces 01336.atmos_average_my_concat.nc and "
         f"preserves all other files)"
         f"{Nclr}\n\n"
     )

bin/MarsFormat.py

@@ -125,6 +125,7 @@ def wrapper(*args, **kwargs):
     help=(
         f"Calculate 5-day averages binned by hour from instantaneous "
         f"data. Generates MGCM-like 'diurn' files.\n"
+        f"Works on non-MGCM files only.\n"
         f"{Green}Example:\n"
         f"> MarsFormat openmars_file.nc -gcm openmars -bd\n"
         f"{Blue}(Creates openmars_file_daily.nc;  5-sol bin){Green}\n"
@@ -178,17 +179,22 @@ def main():
         print(f"{Yellow}***Notice***  No operation requested. Use '-gcm' and specify openmars, marswrf, pcm, emars")
         exit()  # Exit cleanly
 
+    print(f"Running MarsFormat with args: {args}")
+    print(f"Current working directory: {os.getcwd()}")
+    print(f"Files in input_file: {[f.name for f in args.input_file]}")
+    print(f"File exists check: {all(os.path.exists(f.name) for f in args.input_file)}")
+
     path2data = os.getcwd()
 
     # Load all of the netcdf files
     file_list = [f.name for f in args.input_file]
     model_type = args.gcm_name  # e.g. 'legacy'
     for filei in file_list:
-        # Add path unless full path is provided
-        if '/' not in filei:
-            fullnameIN = path2data + '/' + filei
-        else:
+        # Use os.path.join for platform-independent path handling
+        if os.path.isabs(filei):
             fullnameIN = filei
+        else:
+            fullnameIN = os.path.join(path2data, filei)
 
         print('Processing...')
         # Load model variables, dimensions
@@ -536,7 +542,8 @@ def main():
             DS_average[model.dim_time].attrs['long_name'] = 'time averaged over %s sols'%(nday)
 
             # Create New File, set time dimension as unlimitted
-            fullnameOUT = fullnameIN[:-3]+ext+'.nc'
+            base_name = os.path.splitext(fullnameIN)[0]
+            fullnameOUT = f"{base_name}{ext}.nc"
             DS_average.to_netcdf(fullnameOUT,unlimited_dims=model.dim_time,format='NETCDF4_CLASSIC')
 
 

bin/MarsInterp.py

@@ -102,10 +102,12 @@ def wrapper(*args, **kwargs):
 
 parser.add_argument('-t', '--interp_type', type=str, default='pstd',
     help=(
-        f"Interpolation interp_: ``pstd``, ``zstd``, or ``zagl``.\n"
+        f"Interpolation to standard pressure (pstd), standard altitude "
+        f"(zstd), or altitude above ground level (zagl).\nWorks on "
+        f"'daily', 'average', and 'diurn' files.\n"
         f"{Green}Example:\n"
         f"> MarsInterp 01336.atmos_average.nc\n"
-        f"> MarsInterp 01336.atmos_average.nc -t pstd"
+        f"> MarsInterp 01336.atmos_average.nc -t pstd\n"
         f"{Nclr}\n\n"
     )
 )
@@ -114,8 +116,11 @@ def wrapper(*args, **kwargs):
 
 parser.add_argument('-v', '--vertical_grid', type=str, default=None,
     help=(
-        f"Layer IDs as defined in ``amescap_profile``.\nFor first "
+        f"For use with ``-t``. Specify a custom vertical grid to "
+        f"interpolate to.\n"
+        f"Custom grids defined in ``amescap_profile``.\nFor first "
         f"time use, copy ``amescap_profile`` to your home directory:\n"
+        f"Works on 'daily', 'diurn', and 'average' files.\n"
         f"{Cyan}cp path/to/amesCAP/mars_templates/amescap_profile "
         f"~/.amescap_profile\n"
         f"{Green}Example:\n"
@@ -126,8 +131,9 @@ def wrapper(*args, **kwargs):
 
 parser.add_argument('-incl', '--include', nargs='+',
     help=(
-        f"Only include the listed variables. Dimensions and 1D "
-        f"variables are always included.\n"
+        f"Only include the listed variables in the action. Dimensions "
+        f"and 1D variables are always included.\n"
+        f"Works on 'daily', 'diurn', and 'average' files.\n"
         f"{Green}Example:\n"
         f"> MarsInterp 01336.atmos_daily.nc -incl temp ps ts"
         f"{Nclr}\n\n"
@@ -153,8 +159,8 @@ def wrapper(*args, **kwargs):
         f"CAP to create a new file with the extension name specified "
         f"here.\n"
         f"{Green}Example:\n"
-        f"> MarsInterp 00334.atmos_average.nc -t pstd -ext _dflt_levs\n"
-        f"{Blue}(Produces 00334.atmos_average_dflt_levs.nc and "
+        f"> MarsInterp 00334.atmos_average.nc -t pstd -ext _my_pstd\n"
+        f"{Blue}(Produces 00334.atmos_average_my_pstd.nc and "
         f"preserves all other files)"
         f"{Nclr}\n\n"
     )