Adding a new XRISM-Xtend getting started notebook

.circleci/config.yml

@@ -1,12 +1,27 @@
 version: 2.1
 
+parameters:
+  notebooks_to_build:
+    type: string
+    default: ""
+
 jobs:
   pr-build-docs:
+    # Allows parameters to be passed into the job when
+    #  it is triggered through the API.
+    # The notebooks to build parameter will be a comma-separated list of relative
+    #  notebook paths to files that should be built. Every other notebook will be
+    #  ignored. Default value of "" means that all notebooks will be built.
+    parameters:
+      notebooks_to_build:
+        type: string
+        default: ""
+    # We're using a docker container as a run environment - particularly the NASA Fornax
+    #  high-energy image that we maintain as part of the Fornax initiative
     docker:
-      # The Fornax-Hea images can definitely be used here, just we're having some issues with what supporting files
-      #  software are included at the moment, so it is easier to use a different image and just install HEASoft
-      #  at the moment
-      - image: ghcr.io/nasa-fornax/fornax-images/fornax-hea:20251030_1555
+      # We make use of the NASA Fornax high-energy astrophysics image, which has
+      #  pre-existing environments for most of the missions we support.
+      - image: ghcr.io/nasa-fornax/fornax-images/fornax-hea:update-addHEASoft6.36
     # This sets the amount of compute that we'll use to run the on-PR builds/tests - this particular setup corresponds
     #  to 2 CPUs and 4GB of RAM as of 16th September 2025
     resource_class: medium
@@ -40,38 +55,38 @@ jobs:
       - restore_cache:
           name: Restoring Chandra CalDB cache
           keys:
-          - ciao-caldb-
+            - ciao-caldb-
 
       # Some missions (e.g. XMM, Chandra, eROSITA) require that their calibration files are available locally. We
       #  don't want to download them every single time, so we're going to cache them for future runs.
-      - run:
-          name: Acquiring/validating XMM-CCFs
-          no_output_timeout: 30m
-          command: |
-            # If we can't see the xmm_ccf directory, or the version file we generate doesn't contain the same
-            #  version number as currently defined in the environment variable, we have to download the data
-            if [ ! -d /home/jovyan/xmm-ccf ] || [[ ! "$(<"/home/jovyan/xmm-ccf/xmm-ccf.ver")" == "$CIRCLECI_XMM_CCF_VER" ]]; then
-
-              # Makes the xmm-ccf directory, if it doesn't already exist. This is a different approach to the
-              #  what we do for the Chandra CalDB below, as rsync will update files if there are newer versions
-              #  available, so we don't want to delete the directory.
-              mkdir -p /home/jovyan/xmm-ccf/ccf-files
-
-              # We do delete the version file, if it exists, as if we get to this point then we are either downloading
-              #  the data for the first time, or the version number has changed.
-              [ -f /home/jovyan/xmm-ccf/xmm-ccf.ver ] && rm /home/jovyan/xmm-ccf/xmm-ccf.ver
-
-              # Make a new version file - this goes a level up from the directory where the files will actually
-              #  be stored (ccf-files), because the rsync process will delete the version file
-              echo "${CIRCLECI_XMM_CCF_VER}" > /home/jovyan/xmm-ccf/xmm-ccf.ver
-
-              # We don't include rsync in the Fornax-Hea image, so unfortunately we'll have to install it now. The
-              #  neatest way is to make a new conda environment that just contains rsync
-              micromamba create -n rsync-env -y -c conda-forge rsync
-
-              # Actually rsync the calibration files
-              micromamba run -n rsync-env rsync -v -a --delete --delete-after --force --include='*.CCF' --exclude='*/' sasdev-xmm.esac.esa.int::XMM_VALID_CCF /home/jovyan/xmm-ccf/ccf-files
-            fi
+#      - run:
+#          name: Acquiring/validating XMM-CCFs
+#          no_output_timeout: 30m
+#          command: |
+#            # If we can't see the xmm_ccf directory, or the version file we generate doesn't contain the same
+#            #  version number as currently defined in the environment variable, we have to download the data
+#            if [ ! -d /home/jovyan/xmm-ccf ] || [[ ! "$(<"/home/jovyan/xmm-ccf/xmm-ccf.ver")" == "$CIRCLECI_XMM_CCF_VER" ]]; then
+#
+#              # Makes the xmm-ccf directory, if it doesn't already exist. This is a different approach to the
+#              #  what we do for the Chandra CalDB below, as rsync will update files if there are newer versions
+#              #  available, so we don't want to delete the directory.
+#              mkdir -p /home/jovyan/xmm-ccf/ccf-files
+#
+#              # We do delete the version file, if it exists, as if we get to this point then we are either downloading
+#              #  the data for the first time, or the version number has changed.
+#              [ -f /home/jovyan/xmm-ccf/xmm-ccf.ver ] && rm /home/jovyan/xmm-ccf/xmm-ccf.ver
+#
+#              # Make a new version file - this goes a level up from the directory where the files will actually
+#              #  be stored (ccf-files), because the rsync process will delete the version file
+#              echo "${CIRCLECI_XMM_CCF_VER}" > /home/jovyan/xmm-ccf/xmm-ccf.ver
+#
+#              # We don't include rsync in the Fornax-Hea image, so unfortunately we'll have to install it now. The
+#              #  neatest way is to make a new conda environment that just contains rsync
+#              micromamba create -n rsync-env -y -c conda-forge rsync
+#
+#              # Actually rsync the calibration files
+#              micromamba run -n rsync-env rsync -v -a --delete --delete-after --force --include='*.CCF' --exclude='*/' sasdev-xmm.esac.esa.int::XMM_VALID_CCF /home/jovyan/xmm-ccf/ccf-files
+#            fi
 
       # We also download the Chandra CalDB
       - run:
@@ -89,7 +104,7 @@ jobs:
               mkdir -p /home/jovyan/chandra-caldb
 
               # Download the Chandra CalDB
-              wget https://cxc.cfa.harvard.edu/cdaftp/arcftp/caldb/caldb_${CHANDRA_CALDB_VER}_main.tar.gz
+              wget https://cxc.cfa.harvard.edu/cdaftp/arcftp/ChandraCalDB/tars/caldb_${CHANDRA_CALDB_VER}_main.tar.gz
 
               # Unpack the archive into a specific directory
               tar xzf caldb_${CHANDRA_CALDB_VER}_main.tar.gz -C /home/jovyan/chandra-caldb
@@ -148,8 +163,12 @@ jobs:
       # Now we're going to start building the documentation
       - run:
           name: Build HTML rendering of notebooks
-          no_output_timeout: 30m
+          no_output_timeout: 60m
           command: |
+            # The first command sets the HEASARC_NOTEBOOKS_TO_BUILD environment variable, which tells the build
+            #  process which notebooks to build. It allows granular control over which notebooks are built
+            #  for a given test build - the default value of "" will build all the notebooks.
+            export HEASARC_NOTEBOOKS_TO_BUILD="<< parameters.notebooks_to_build >>"
             micromamba run -n build_docs sphinx-build -b html . _build/html -nWT --keep-going
             sed -E -i.bak '/caption-text/{N; s/.+caption-text.+\n<ul>/<ul>/; P;D;}' _build/html/index.html
             bash -c 'rm _build/html/index.html.bak'
@@ -186,4 +205,5 @@ jobs:
 workflows:
   build-for-PR:
     jobs:
-      - pr-build-docs
+      - pr-build-docs:
+          notebooks_to_build: << pipeline.parameters.notebooks_to_build >>

.github/workflows/trigger-circleci.yml

@@ -19,6 +19,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Wait for 'pre-commit.ci' to pass
         # This action repeatedly polls the GitHub Statuses API for the specific check.
@@ -54,6 +56,68 @@ jobs:
               exit 3
             fi
 
+      - name: Define notebook build list
+        # The CircleCI pipeline can take an argument that defines exactly which notebooks
+        #  should be built. This step sets up the variable that will be passed to the
+        #  pipeline. Either by identifying which notebooks have been altered in the
+        #  triggering PR, or by looking for an override in the PR comments.
+        id: which_notebooks_to_build
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          # Define the exclusion patterns
+          EXCLUDE_PATTERNS=("*notebook_template*" "*pull_request_template*" "*README*" "**/*README*" "*.ipynb_checkpoints*")
+
+          # First, check if there's an override pattern in PR comments
+          OVERRIDE_NOTEBOOKS=$(curl -s \
+            -H "Authorization: token ${GITHUB_TOKEN}" \
+            "https://api.github.com/repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" | \
+            jq -r '.[] | select(.body | contains("@heasarc-notebooks:")) | .body | match("@heasarc-notebooks:\\s*(.+)"; "g") | .captures[0].string' | tail -1)
+
+          if [ -n "$OVERRIDE_NOTEBOOKS" ]; then
+            # Use the override from the comments
+            NOTEBOOKS_TO_BUILD="$OVERRIDE_NOTEBOOKS"
+            echo "Using notebooks specified in PR comment: $NOTEBOOKS_TO_BUILD"
+          else
+            # Get the list of changed files in the PR
+            CHANGED_FILES=$(git diff --name-only origin/main...HEAD)
+
+            # Filter and process the files
+            NOTEBOOKS_TO_BUILD=""
+            while IFS= read -r file; do
+              # Check if file is in tutorials directory and ends with .md
+              if [[ "$file" =~ ^tutorials/.*\.md$ ]]; then
+                # Get relative path from tutorials directory
+                REL_PATH="${file#tutorials/}"
+
+                # Check if file should be excluded
+                SHOULD_EXCLUDE=false
+                for pattern in "${EXCLUDE_PATTERNS[@]}"; do
+                  # Convert glob patterns to match logic
+                  if [[ "$REL_PATH" == *"index"* ]] || [[ "$REL_PATH" =~ $(echo "$pattern" | sed 's/\*/.*/'g) ]]; then
+                    SHOULD_EXCLUDE=true
+                    break
+                  fi
+                done
+
+                if [ "$SHOULD_EXCLUDE" = false ]; then
+                  if [ -z "$NOTEBOOKS_TO_BUILD" ]; then
+                    NOTEBOOKS_TO_BUILD="$REL_PATH"
+                  else
+                    NOTEBOOKS_TO_BUILD="${NOTEBOOKS_TO_BUILD},$REL_PATH"
+                  fi
+                fi
+              fi
+            done <<< "$CHANGED_FILES"
+
+            echo "Notebooks to build based on changed files: $NOTEBOOKS_TO_BUILD"
+          fi
+
+          # Export the variable for the next step
+          echo "notebooks_to_build=$NOTEBOOKS_TO_BUILD" >> $GITHUB_OUTPUT
+
+
       - name: Trigger CircleCI Pipeline via API
         # Only proceed if the wait step succeeded, and the 'skip-doc-build' label wasn't added to the PR (could be
         #  that the contributor added the label after the PR was opened).
@@ -65,16 +129,24 @@ jobs:
           TARGET_BRANCH: ${{ github.head_ref }}
           PROJECT_SLUG: 'circleci/LdzbTUR6aexSM6vJCVrZk5/GRoaAeYuJNgCdifZviG612'
           CHECKOUT_REPO: 'HEASARC/heasarc-tutorials'
+          NOTEBOOKS_TO_BUILD: ${{ steps.which_notebooks_to_build.outputs.notebooks_to_build }}
+
         run: |
           echo "pre-commit.ci checks passed on branch ${TARGET_BRANCH}. Triggering CircleCI."
+          echo "Notebooks to build: ${NOTEBOOKS_TO_BUILD}"
 
           # Post request to CircleCI API v2 to trigger a new pipeline
+          # We pass arguments to the pipeline via the 'parameters' field, which
+          #  allows us to control exactly which notebooks are to built.
           curl -X POST \
             --url "https://circleci.com/api/v2/project/${PROJECT_SLUG}/pipeline/run" \
             --header "Content-Type: application/json" \
             --header "Circle-Token: ${CIRCLE_TOKEN}" \
             --data '{
               "definition_id": "f5bf0ced-00cd-4643-aa02-c2f10e445243",
               "config": {"branch": "'"${TARGET_BRANCH}"'"},
-              "checkout": {"branch": "'"${TARGET_BRANCH}"'"}
-            }'
+              "checkout": {"branch": "'"${TARGET_BRANCH}"'"},
+              "parameters": {
+                "notebooks_to_build": "'"${NOTEBOOKS_TO_BUILD}"'"
+              }
+             }'