Overfelt/higher order tendency

.github/DISCUSSION_TEMPLATE/feature-request.yml

@@ -0,0 +1,54 @@
+title: "[Brief Title for Feature] "
+labels: ["Feature Request"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This template will help you fill out the New Feature Overview Document. You do not have to fill
+        out everything and will be able to edit the resulting page later.  NOTE. E3SM is not accepting new
+        feature requests for EAM or MPAS-Ocean.
+  - type: textarea
+    id: high-level-description
+    attributes:
+      label: High Level Description
+      description: "Include: High-level description of code changes and/or new code, an overview of the design, infrastructure changes if needed.  You can also attach figures or papers."
+      placeholder: "add text, figures, papers"
+    validations:
+      required: false
+  - type: textarea
+    id: improvements
+    attributes:
+      label: Expected Improvements
+      description: "Expected improvements and how these will be demonstrated"
+      placeholder: "add text"
+    validations:
+      required: false
+  - type: dropdown
+    id: performance
+    attributes:
+      label: Expected impact on performance
+      description: "Choose an option.  Use box below for future explanation"
+      options:
+        - Significant improvement
+        - Slight improvement
+        - No significant change
+        - Slight decline
+        - Significant decline
+    validations:
+      required: true
+  - type: textarea
+    id: perf-explanation
+    attributes:
+      label: Performance explanation
+      description: "Add more details on performance changes"
+      placeholder: "add text"
+    validations:
+      required: false
+  - type: textarea
+    id: publications
+    attributes:
+      label: Planned Publications
+      description: "If relevant and known, describe papers that will be published"
+      placeholder: "add text"
+    validations:
+      required: false

.github/actions/test-all-eamxx/action.yml

@@ -41,6 +41,13 @@ inputs:
     description: 'Semicolon-separated list of key=value pairs for CMake to pass to test-all-eamxx'
     required: false
     default: ''
+  kokkos_develop:
+    description: 'Whether or not this is a test using Kokkos develop branch'
+    required: false
+    default: 'false'
+    valid_values:
+      - 'true'
+      - 'false'
 
 runs:
   using: "composite"
@@ -77,6 +84,18 @@ runs:
         echo "  generate     : ${{ inputs.generate }}"
         echo "  submit       : ${{ inputs.submit }}"
         echo "  cmake-configs: ${{ inputs.cmake-configs }}"
+        echo "  kokkos dev   : ${{ inputs.kokkos_develop }}"
+      shell: sh
+    - name: Checkout Kokkos develop branch
+      if: ${{ inputs.kokkos_develop == 'true' }}
+      working-directory: externals/ekat/extern/kokkos
+      run: |
+        echo "Check out up-to-date Kokkos develop branch"
+        if ! git remote | grep -q upstream; then
+          git remote add upstream https://github.com/kokkos/kokkos.git
+        fi
+        git fetch upstream
+        git checkout upstream/develop
       shell: sh
     - name: Run test-all-eamxx
       working-directory: components/eamxx
@@ -98,6 +117,14 @@ runs:
           cmd+=" -c $config"
         done
 
+        # Add suffix for CDASH build name, and necessary cmake vars if this is a kokkos-develop test
+        if [ "${{ inputs.kokkos_develop }}" = "true" ]; then
+          cmd+=" --build-name-suffix kokkos_develop"
+          cmd+=" -c CMAKE_CXX_STANDARD=20"
+          cmd+=" -c Kokkos_ENABLE_IMPL_VIEW_LEGACY=ON"
+          cmd+=" -c Kokkos_ENABLE_DEPRECATED_CODE_4=ON"
+        fi
+
         # Print the full command, then run it
         echo "test-all-eamxx call: $cmd"
         $cmd

.github/workflows/e3sm-gh-ci-cime-tests.yml

@@ -41,8 +41,8 @@ jobs:
           - ERS_Ld5_P4.ne4pg2_oQU480.F2010.ghci-oci_gnu.eam-wcprod_F2010
           - SMS_D_Ln5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu
           - ERS_Ld5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod
-          - ERS_P4.ne4pg2_oQU480.A.ghci-oci_gnu
-          - ERS_P4.ne4pg2_oQU480.I1850ELMCN.ghci-oci_gnu.elm-qian_1948
+          - ERS.f19_g16_rx1.A.ghci-oci_gnu
+          - ERS.f19_g16.I1850ELM.ghci-oci_gnu.elm-betr
     container:
       image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.1
 

.github/workflows/e3sm-gh-md-linter.yml

@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v5
       with:
         fetch-depth: 0
-    - uses: tj-actions/changed-files@v46
+    - uses: tj-actions/changed-files@v47
       id: changed-files
       with:
         files: '**/*.md'

.github/workflows/eamxx-gh-clang-format.yml

@@ -3,11 +3,12 @@ name: "eamxx-format"
 # if .{cpp,hpp} files are touched in a PR, lint them!
 
 on:
-  # pull_request:
-  #   branches: ["master"]
-  #   paths:
-  #     - 'components/eamxx/**/*.cpp'
-  #     - 'components/eamxx/**/*.hpp'
+  pull_request:
+    branches: ["master"]
+    paths:
+      - 'components/eamxx/**/*.cpp'
+      - 'components/eamxx/**/*.hpp'
+      - .github/workflows/eamxx-gh-clang-format.yml
 
   # Manual run
   workflow_dispatch:
@@ -16,50 +17,75 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
   cancel-in-progress: true
 
+# RECIPE:
+# 0. install clang-format via conda. This allows us to pick virtually any clang-format
+#    version we want, without having to rely on what's available on the runner
+# 1a. runs clang-format on the files changed by this PR, modifying them in-place
+# 1b. in case of workflow_disaptch, run clang-format on ALL eamxx cpp/hpp files
+# 2. any files that required formatting are returned by a 'git diff --name-only'
+#    and written to the job summary along with an informational message
+# if (formatting required) {
+#   3. runs a full diff to generate a patch file
+#   4. uploads the patch as an artifact
+# }
+# 5. return the pass/fail status of the job
+
 jobs:
   clang-format-linter:
-    if: ${{ github.repository == 'E3SM-Project/E3SM' }}
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v5
       with:
         fetch-depth: 0
-    - uses: tj-actions/changed-files@v46
+    - uses: tj-actions/changed-files@v47
       id: changed-files
       with:
         files: 'components/eamxx/**/*.{cpp,hpp}'
         separator: " "
-    # 1. runs clang-format on the files changed by this PR, modifying them in-place
-    # 2. any files that required formatting are returned by a 'git diff --name-only'
-    #    and written to the job summary along with an informational message
-    # if (formatting required) {
-    #   3. runs a full diff to generate a patch file
-    #   4. uploads the patch as an artifact
-    # }
-    # 5. return the pass/fail status of the job
-    # NOTE: different versions of clang-format are likely to give different
-    #       results, so we keep an eye on this
-    - name: get version clang-format-18
-      run: |
-        ver=$(clang-format-18 --version)
-        if [ "${ver}" = "18.1.3" ]; then
-          echo "Running clang-format version ${ver}."
-        else
-          echo "Warning! clang-format version has changed! (default 18.1.3)"
-          echo "Please inform developers by opening an issue."
-          echo "Running clang-format with non-standard version ${ver}."
-        fi
-    - name: run formatter looping over files
+    - name: setup micromamba env
+      uses: mamba-org/setup-micromamba@v2
+      with:
+        init-shell: bash
+        environment-name: clang-format
+        create-args:
+          clang-format=20.1.8
+    - name: run formatter looping over PR-changed files, then diff -  PR trigger
+      # Note: this (json) variable contains a handful of sub-fields, but its
+      # existence encapsulates all of the PR-related triggers
+      # this is the best option because github.event_name could be something
+      # like "synchronization," which falls under the umbrella of PR triggers
+      if: ${{ github.event.pull_request }}
+      shell: bash -leo pipefail {0}
       run: |
         flist="${{ steps.changed-files.outputs.all_changed_files }}"
         run_cfmt () {
-          clang-format-18 -i --style=file:"components/eamxx/.clang-format" $1
+        clang-format -i --style=file:"components/eamxx/.clang-format" $1
+        }
+        clang-format --version
+        for f in ${flist[@]}; do
+          echo "--Running clang-format on ${f}--"
+          run_cfmt "${f}"
+        done
+        git diff --name-only > diffiles.txt
+    - name: run formatter looping over all files, then diff - manual trigger
+      # Note: different from above, this trigger does not result in a variable
+      # github.event.workflow_dispatch, but this will always be the event_name
+      if: ${{ github.event_name == 'workflow_dispatch' }}
+      shell: bash -leo pipefail {0}
+      run: |
+        # when it's a manual trigger, we'll choose to consider all cpp/hpp files
+        flist=($(find components/eamxx -depth -name "*.?pp"))
+        run_cfmt () {
+          clang-format -i --style=file:"components/eamxx/.clang-format" $1
         }
+        clang-format --version
         for f in ${flist[@]}; do
           echo "--Running clang-format on ${f}--"
           run_cfmt "${f}"
         done
         git diff --name-only > diffiles.txt
+    - name: examine diff and write to summary
+      run: |
         if [[ $(wc -l < diffiles.txt) -gt 0 ]]; then
           # we don't want to return a nonzero exit code in this step because
           # it'll stop the job, so we store it in the ENV
@@ -82,22 +108,43 @@ jobs:
                "execute \`git apply <patch-file>\` from your local E3SM " \
                "branch." >> "${GITHUB_STEP_SUMMARY}"
           echo "Then commit the changes and push to your remote branch." >> "${GITHUB_STEP_SUMMARY}"
+          echo "" >> "${GITHUB_STEP_SUMMARY}"
+          echo "For diffs with ***fewer than 100 lines***, we also include the text of " \
+               "the patch file in a collapsed section, below."  >> "${GITHUB_STEP_SUMMARY}"
         else
           echo "status_fail=false" >> "${GITHUB_ENV}"
           echo "## Status: \`clang-format\` Check Passed!" > "${GITHUB_STEP_SUMMARY}"
         fi
     - name: diff format changes
-      run: git diff > "components/eamxx/clang-format-patch.diff"
+      run: |
+        d_str=$(date +"%Y%m%d%H%M%S")
+        fname="clang-format-patch_${{ github.event.pull_request.number || github.event_name }}-${d_str}.diff"
+        echo "fname=${fname}" >> "${GITHUB_ENV}"
+        git diff > "components/eamxx/${fname}"
+        if [[ ${status_fail} = true ]] && [[ $(wc -l < "components/eamxx/${fname}") -lt 100 ]]; then
+          echo "<details>" >> "${GITHUB_STEP_SUMMARY}"
+          echo "" >> "${GITHUB_STEP_SUMMARY}"
+          echo "<summary>" >> "${GITHUB_STEP_SUMMARY}"
+          echo "<b>Contents of</b> <tt>clang-format-patch.diff</tt>" >> "${GITHUB_STEP_SUMMARY}"
+          echo "</summary>" >> "${GITHUB_STEP_SUMMARY}"
+          echo "To apply this version of the patch, save the text to a local file, " \
+               "and <tt>git apply &lt;patch-file&gt;</tt>, etc., as above." >> "${GITHUB_STEP_SUMMARY}"
+          echo "" >> "${GITHUB_STEP_SUMMARY}"
+          echo "\`\`\`" >> "${GITHUB_STEP_SUMMARY}"
+          cat "components/eamxx/${fname}" >> "${GITHUB_STEP_SUMMARY}"
+          echo "\`\`\`" >> "${GITHUB_STEP_SUMMARY}"
+          echo "" >> "${GITHUB_STEP_SUMMARY}"
+          echo "</details>" >> "${GITHUB_STEP_SUMMARY}"
+        fi
     - name: upload diff patch
-      if: ${{ env.status_fail }}
+      if: ${{ env.status_fail == 'true' }}
       uses: actions/upload-artifact@v4
       with:
-        name: clang-format-patch_${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}.diff
-        path: |
-          components/eamxx/clang-format-patch.diff
+        name: ${{ env.fname }}
+        path: "components/eamxx/${{ env.fname }}"
     - name: return pass-fail status
       run: |
-        if [[ ${status_fail} ]]; then
+        if [[ ${status_fail} = true ]]; then
           exit 1
         else
           exit 0

.github/workflows/eamxx-sa-testing.yml

@@ -18,6 +18,7 @@ on:
       - 'externals/haero'
       - 'externals/scorpio'
       # second, no to these
+      - '!components/eamxx/.clang-format'
       - '!components/eamxx/docs/**'
       - '!components/eamxx/mkdocs.yml'
       - '!components/eamxx/cime_config/**'
@@ -41,10 +42,15 @@ on:
         description: 'Force cdash submission'
         required: true
         type: boolean
+      use_kokkos_dev:
+        description: 'Run with Kokkos develop branch'
+        required: true
+        type: boolean
 
   # Add schedule trigger for nightly runs at midnight MT (Standard Time)
   schedule:
     - cron: '0 7 * * *'  # Runs at 7 AM UTC, which is midnight MT during Standard Time
+    - cron: '0 7 * * 1'  # Weekly run every Monday at 7 AM UTC (using Kokkos develop branch)
 
 concurrency:
   # Two runs are in the same group if they are testing the same git ref
@@ -59,6 +65,8 @@ env:
   generate: ${{ github.event_name == 'workflow_dispatch' && inputs.bless }}
   # Do EKAT testing for nightlies or (TODO: if EKAT label is present)
   ekat: ${{ github.event_name == 'schedule' }}
+  # For the weekly run, we test using the kokkos develop branch
+  kokkos_dev_test: ${{ github.event_name == 'schedule' && github.event.schedule == '0 7 * * 1' || (github.event_name == 'workflow_dispatch' && inputs.use_kokkos_dev) }}
 
 jobs:
   gcc-openmp:
@@ -94,6 +102,7 @@ jobs:
           submit: ${{ env.submit }}
           cmake-configs: Kokkos_ENABLE_OPENMP=ON
           ekat: ${{ env.ekat }}
+          kokkos_develop: ${{ env.kokkos_dev_test }}
   gcc-cuda:
     if: |
       ${{
@@ -162,3 +171,4 @@ jobs:
           submit: ${{ env.submit }}
           cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }};SCREAM_SMALL_KERNELS=${{ matrix.test.SK }}
           ekat: ${{ env.ekat }}
+          kokkos_develop: ${{ env.kokkos_dev_test }}