ENH: switching to JB2 infrastructure

.circleci/config.yml

@@ -1,46 +1,63 @@
 version: 2.1
 
-jobs:
 
+jobs:
   build-docs:
     docker:
-      - image: cimg/python:3.12
-
+      - image: cimg/python:3.13
     steps:
-      - attach_workspace:
-          at: ~/
-
       - checkout
 
-      - run:
-          name: Install CI dependencies
-          command: python -m pip install --upgrade tox
+      - restore_cache:
+          keys:
+            - node-cache-v1
 
       - restore_cache:
           keys:
-              - jupyter_ch
+            - pip-cache-{{ checksum "site_requirements.txt" }}
+
+      - run:
+          name: Install Node.js
+          command: |
+            curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash -
+            sudo apt-get install -y nodejs
+
+      - run:
+          name: Install mystmd
+          command: |
+            npm install mystmd
 
       - run:
-          name: Build HTML rendering of notebooks
-          no_output_timeout: 30m
+          name: Install Python dependencies
           command: |
-            python -m tox -e py312-buildhtml
+            python -m pip install --upgrade pip tox
 
       - save_cache:
-          key: jupyter_ch
+          key: node-cache-v1
           paths:
-            - _build/.jupyter_cache
+            - node_modules
 
-      - store_artifacts:
-          path: _build/html
+      - run:
+          name: Build documentation
+          no_output_timeout: 60m
+          environment:
+            # Ensure this is same as store_artifacts path below
+            DOCS_PATH: _build/html
+          command: |
+            export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH"
 
-      - persist_to_workspace:
-          root: _build
+            python -m tox -e py313-buildhtml
+
+      - save_cache:
+          key: pip-cache-{{ checksum "site_requirements.txt" }}
           paths:
-            - html
+            - ~/.cache/pip
+
+      - store_artifacts:
+          path: _build/html
 
 workflows:
   version: 2
-  default:
+  build-and-docs:
     jobs:
       - build-docs

ignore_circleci_testing

@@ -0,0 +1,2 @@
+tutorials/parallelize/Parallelize_Convolution
+tutorials/parquet-catalog-demos/wise-allwise-catalog-demo

ignore_testing

@@ -0,0 +1,2 @@
+tutorials/cloud_access/euclid-cloud-access.md
+tutorials/roman_simulations/roman_hlss_number_density.md

index.md

@@ -1,135 +1,11 @@
-# Caltech/IPAC--IRSA Python Notebook Tutorials
+# Caltech/IPAC—IRSA Python Notebook Tutorials
 
 
 These Python Jupyter Notebook tutorials demonstrate access methods and techniques for working with data served by the [NASA/IPAC Infrared Science Archive (IRSA)](https://irsa.ipac.caltech.edu).
 They cover topics like querying IRSA, working with catalogs in Parquet format, visualizing with Firefly, and general other techniques.
 
 
-## Accessing IRSA's on-premises holdings using VO protocols
 
-```{toctree}
----
-maxdepth: 1
-caption: VO on-prem data access
----
-tutorials/irsa-sia-examples/sia_2mass_allsky
-tutorials/irsa-sia-examples/sia_allwise_atlas
-tutorials/irsa-sia-examples/sia_cosmos
-tutorials/irsa-sia-examples/siav2_seip
-tutorials/cosmodc2/cosmoDC2_TAP_access.md
-
-```
-
-## Accessing IRSA's cloud holdings
-
-These notebooks demonstrate how to access the IRSA-curated datasets that available in Amazon Web Services (AWS) S3 cloud storage buckets.
-
-
-```{toctree}
----
-maxdepth: 1
-caption: Cloud data access
----
-
-tutorials/cloud_access/cloud-access-intro
-tutorials/parquet-catalog-demos/wise-allwise-catalog-demo
-tutorials/parquet-catalog-demos/neowise-source-table-strategies
-tutorials/parquet-catalog-demos/neowise-source-table-lightcurves
-tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey
-tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey
-tutorials/cloud_access/euclid-cloud-access
-
-```
-
-## Accessing Euclid data
-
-### Euclid Early Release Observation
-
-```{toctree}
----
-maxdepth: 1
-caption: Euclid Early Release Observations
----
-
-tutorials/euclid_access/Euclid_ERO
-
-```
-
-### Euclid Quick Release 1
-
-```{toctree}
----
-maxdepth: 1
-caption: Euclid Quick Release 1
----
-
-tutorials/euclid_access/1_Euclid_intro_MER_images
-tutorials/euclid_access/2_Euclid_intro_MER_catalog
-tutorials/euclid_access/3_Euclid_intro_1D_spectra
-tutorials/euclid_access/4_Euclid_intro_PHZ_catalog
-tutorials/euclid_access/5_Euclid_intro_SPE_catalog
-tutorials/cloud_access/euclid-cloud-access
-
-```
-
-## Accessing SPHEREx data
-
-```{toctree}
----
-maxdepth: 1
-caption: SPHEREx Data
----
-
-tutorials/spherex/spherex_intro
-
-```
-
-## Interactive visualization in Python with Firefly
-
-These notebooks demonstrate how to use the Firefly visualization tools from Python.
-[Firefly](https://github.com/Caltech-IPAC/firefly) is an open-source toolkit based on IVOA standards and designed to enable astronomical data archive access, exploratory data analysis, and visualization.
-
-It is used in archive user interfaces at [IRSA](https://irsa.ipac.caltech.edu), the [NASA Exoplanet Science Institute (NExScI)](https://nexsci.caltech.edu/), the [NASA/IPAC Extragalactic Database (NED)](https://ned.ipac.caltech.edu/), and the [Vera C. Rubin Observatory](https://www.lsst.org/).
-
-```{toctree}
----
-maxdepth: 1
-caption: Visualizations with Firefly
----
-
-tutorials/firefly/SEDs_in_Firefly
-tutorials/firefly/NEOWISE_light_curve_demo
-tutorials/firefly/OpenUniverse2024Preview_Firefly
-
-```
-
-## Simulated Data
-
-```{toctree}
----
-maxdepth: 1
-caption: Simulated Data
----
-
-tutorials/roman_simulations/roman_hlss_number_density.md
-
-```
-
-## Generally useful techniques
-
-These notebooks  cover miscellaneous topics that users might find useful in their analysis of IRSA-curated data.
-
-```{toctree}
----
-maxdepth: 1
-caption: Generic techniques
----
-
-tutorials/parallelize/Parallelize_Convolution
-
-```
-
-***
 
 ## About these notebooks
 

myst.yml

@@ -0,0 +1,16 @@
+# See docs at: https://mystmd.org/guide/frontmatter
+version: 1
+project:
+  title: IRSA Tutorials
+  subject: IRSA Tutorials
+  description: IRSA Python Notebook tutorials to demostrate accessing data served by the NASA/IPAC Infrared Science Archive (IRSA)
+  keywords: [astronomy]
+  authors: [IRSA Scientists and Developers]
+  github: https://github.com/Caltech-IPAC/irsa-tutorials/
+extends:
+  - toc.yml
+site:
+  template: book-theme
+  options:
+    favicon: _static/irsa-favicon.ico
+    logo: _static/irsa_logo.png

toc.yml

@@ -0,0 +1,61 @@
+version: 1
+project:
+  toc:
+    - title: IRSA Tutorials
+      file: index.md
+    - title: SPHEREx
+      children:
+        - file: tutorials/spherex/spherex_intro.md
+    - title: Euclid
+      children:
+        - title: Euclid Early Release Observations
+          children:
+            - file: tutorials/euclid_access/Euclid_ERO.md
+        - title: Euclid Quick Release 1
+          children:
+            - file: tutorials/euclid_access/1_Euclid_intro_MER_images.md
+            - file: tutorials/euclid_access/2_Euclid_intro_MER_catalog.md
+            - file: tutorials/euclid_access/3_Euclid_intro_1D_spectra.md
+            - file: tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md
+            - file: tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md
+            - file: tutorials/cloud_access/euclid-cloud-access.md
+    - title: WISE
+      children:
+        - title: AllWISE
+          children:
+            - file: tutorials/irsa-sia-examples/sia_allwise_atlas.md
+            - file: tutorials/parquet-catalog-demos/wise-allwise-catalog-demo.md
+        - title: NEOWISE
+          children:
+            - file: tutorials/firefly/NEOWISE_light_curve_demo.md
+            - file: tutorials/parquet-catalog-demos/neowise-source-table-strategies.md
+            - file: tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md
+    - title: Spitzer
+      children:
+        - title: Spitzer Enhanced Imaging Products (SEIP)
+          children:
+            - file: tutorials/irsa-sia-examples/siav2_seip.md
+    - title: 2MASS
+      children:
+        - file: tutorials/irsa-sia-examples/sia_2mass_allsky.md
+    - title: COSMOS
+      children:
+        - file: tutorials/irsa-sia-examples/sia_cosmos.md
+    - title: Simulated Data
+      children:
+        - title: Roman HLSS Galaxy Mock Catalog
+          children:
+            - file: tutorials/roman_simulations/roman_hlss_number_density.md
+        - title: OpenUniverse 2024
+          children:
+            - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey.md
+            - file: tutorials/firefly/OpenUniverse2024Preview_Firefly.md
+            - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey.md
+        - title: CosmoDC2
+          children:
+            - file: tutorials/cosmodc2/cosmoDC2_TAP_access.md
+    - title: Special Topics
+      children:
+        - file: tutorials/cloud_access/cloud-access-intro.md
+        - file: tutorials/firefly/SEDs_in_Firefly.md
+        - file: tutorials/parallelize/Parallelize_Convolution.md

tox.ini

@@ -12,7 +12,7 @@ setenv =
     FIREFLY_URL = https://irsa.ipac.caltech.edu/irsaviewer # fallback url for firefly notebooks if not defined in code
     devdeps: PIP_EXTRA_INDEX_URL =  https://pypi.anaconda.org/scientific-python-nightly-wheels/simple https://pypi.anaconda.org/astropy/simple
 
-passenv = CI, CIRCLECI, GITHUB_EVENT_NAME, GITHUB_EVENT_PATH
+passenv = CI, CIRCLECI, GITHUB_EVENT_NAME, GITHUB_EVENT_PATH, BASE_URL
 
 deps =
     # We use these files to specify all the dependencies, and below we override
@@ -38,26 +38,27 @@ deps =
     devdeps: git+https://github.com/astropy/pyvo.git#egg=pyvo
     devdeps: git+https://github.com/astropy/astroquery.git#egg=astroquery
 
-allowlist_externals = bash, sed
+allowlist_externals = bash, npx
 
 commands =
     pip freeze
 
     # Ignore testing the tutorials listed in ignore_testing file. We have some OS specific ignores,
     # too due to issues with e.g. multiprocessing and problems in upstream dependency
-    !buildhtml: bash -c 'if python -c "import platform; print(platform.platform())" | grep -i macos; then cat ignore_osx_testing >> ignore_testing; fi'
-    !buildhtml: bash -c 'if python -c "import platform; print(platform.platform())" | grep -i win; then cat ignore_windows_testing >> ignore_testing; fi'
-
-    !buildhtml: bash -c 'if [[ $CI == true ]]; then cat ignore_gha_testing >> ignore_testing; fi'
+    bash -c 'if python -c "import platform; print(platform.platform())" | grep -i macos; then cat ignore_osx_testing >> ignore_testing; fi'
+    bash -c 'if python -c "import platform; print(platform.platform())" | grep -i win; then cat ignore_windows_testing >> ignore_testing; fi'
+    bash -c 'if [[ $CI == true ]]; then cat ignore_gha_testing >> ignore_testing; fi'
+    bash -c 'if [[ $CIRCLECI == true ]]; then cat ignore_circleci_testing >> ignore_testing; fi'
 
     # We only want to run CI in PRs for the notebooks we touched
     !buildhtml: bash -c 'if [[ $GITHUB_EVENT_NAME == pull_request && -z "$(grep force_run:all_tests ${GITHUB_EVENT_PATH})" ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | grep .md; else find tutorials -name "*.md"; fi | grep -vf ignore_testing | xargs jupytext --to notebook '
 
     !buildhtml: pytest --nbval-lax -vv --suppress-no-test-exit-code --durations=10 tutorials
-    buildhtml: sphinx-build -b html . _build/html -nWT --keep-going
-    # SED magic to remove the toctree captions from the rendered index page while keeping them in the sidebar TOC
-    buildhtml: sed -E -i.bak '/caption-text/{N; s/.+caption-text.+\n<ul>/<ul>/; P;D;}' _build/html/index.html
-    buildhtml: bash -c 'rm _build/html/index.html.bak'
+
+    # sed -i needs a bit of hacky conditional on ubuntu to cover the case of an empty ignore
+    buildhtml: bash -c "find tutorials -name '*md' | grep -f ignore_testing | sort | uniq > ignore_execute; if [ -s ignore_execute ]; then cat ignore_execute | xargs -n 1 sed -i -e 's|name: python3|name: python3\nskip_execution: true|g';fi"
+
+    buildhtml: npx myst build --execute --html
 
 pip_pre =
     predeps: true

tutorials/spherex/spherex_intro.md

@@ -7,9 +7,9 @@ jupytext:
     format_version: 0.13
     jupytext_version: 1.17.2
 kernelspec:
-  name: python3
   display_name: Python 3 (ipykernel)
   language: python
+  name: python3
 ---
 
 # Introduction to SPHEREx Spectral Images