Merge pull request #119 from bsipocz/ENH_switch_to_JB2

troyraen · web-flow · commit 8901b1e52f8c · 2025-09-26T10:58:57.000-07:00
ENH: switching to JB2 infrastructure
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,46 +1,46 @@
 version: 2.1
 
-jobs:
 
+jobs:
   build-docs:
     docker:
       - image: cimg/python:3.12
-
     steps:
-      - attach_workspace:
-          at: ~/
-
       - checkout
 
       - run:
-          name: Install CI dependencies
-          command: python -m pip install --upgrade tox
+          name: Install Node.js
+          command: |
+            curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash -
+            sudo apt-get install -y nodejs
 
-      - restore_cache:
-          keys:
-              - jupyter_ch
+      - run:
+          name: Install mystmd
+          command: |
+            npm install mystmd
 
       - run:
-          name: Build HTML rendering of notebooks
-          no_output_timeout: 30m
+          name: Install Python dependencies
           command: |
+            python -m pip install --upgrade pip tox
+
+      - run:
+          name: Build documentation
+          no_output_timeout: 60m
+          environment:
+            # Ensure this is same as store_artifacts path below
+            DOCS_PATH: _build/html
+          command: |
+            export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH"
+            export OMP_NUM_THREADS=1
             python -m tox -e py312-buildhtml
 
-      - save_cache:
-          key: jupyter_ch
-          paths:
-            - _build/.jupyter_cache
 
       - store_artifacts:
           path: _build/html
 
-      - persist_to_workspace:
-          root: _build
-          paths:
-            - html
-
 workflows:
   version: 2
-  default:
+  build-and-docs:
     jobs:
       - build-docs
diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml
@@ -24,13 +24,13 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
 
       - name: Install dependencies
         run: python -m pip install --upgrade tox
 
       - name: Execute notebooks while building HTMLs
-        run: tox -e py311-buildhtml
+        run: tox -e py312-buildhtml
 
       - name: Publish
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e  # v4.0.0
diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml
@@ -68,10 +68,10 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
 
       - name: Install dependencies
         run: python -m pip install --upgrade tox
 
       - name: Execute notebooks as testing
-        run: tox -e py311-buildhtml
+        run: tox -e py312-buildhtml
diff --git a/ignore_osx_testing b/ignore_osx_testing
diff --git a/ignore_tutorials/ignore_all_testing b/ignore_tutorials/ignore_all_testing
diff --git a/ignore_tutorials/ignore_circleci_testing b/ignore_tutorials/ignore_circleci_testing
@@ -0,0 +1,13 @@
+# ignore these 10 now as excluding execution for these is a combination that passes JB2 building on CircleCI.
+# Note that the first 2 are excluded for all kinds of testing; while the parallelize causes issues for GHA JB2 builds and non linux testing, too.
+#
+tutorials/parquet-catalog-demos/neowise-source-table-lightcurves
+tutorials/parquet-catalog-demos/neowise-source-table-strategies
+tutorials/parallelize/Parallelize_Convolution
+tutorials/roman_simulations/roman_hlss_number_density
+tutorials/cosmodc2/cosmoDC2_TAP_access
+tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey
+tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey
+tutorials/parquet-catalog-demos/wise-allwise-catalog-demo
+tutorials/cloud_access/euclid-cloud-access
+tutorials/cloud_access/cloud-access-intro
diff --git a/ignore_tutorials/ignore_gha_testing b/ignore_tutorials/ignore_gha_testing
@@ -1,2 +1,3 @@
+# Both NEOWISE parquet notebooks work with large data that doesn't work within CircleCI or GHA resource limits
 tutorials/parquet-catalog-demos/neowise-source-table-lightcurves
 tutorials/parquet-catalog-demos/neowise-source-table-strategies
diff --git a/ignore_tutorials/ignore_osx_testing b/ignore_tutorials/ignore_osx_testing
@@ -0,0 +1,2 @@
+# The way the notebooks use the multiprocessing module is known to not work on non-Linux
+tutorials/parallelize/Parallelize_Convolution
diff --git a/ignore_tutorials/ignore_rendering b/ignore_tutorials/ignore_rendering
@@ -0,0 +1,2 @@
+# Runs out of memory on GHA using the JB2 stack
+tutorials/parallelize/Parallelize_Convolution
diff --git a/ignore_tutorials/ignore_windows_testing b/ignore_tutorials/ignore_windows_testing
@@ -0,0 +1,2 @@
+# The way the notebooks use the multiprocessing module is known to not work on non-Linux
+tutorials/parallelize/Parallelize_Convolution
diff --git a/ignore_windows_testing b/ignore_windows_testing
diff --git a/index.md b/index.md
@@ -1,135 +1,11 @@
-# Caltech/IPAC--IRSA Python Notebook Tutorials
+# Caltech/IPAC—IRSA Python Notebook Tutorials
 
 
 These Python Jupyter Notebook tutorials demonstrate access methods and techniques for working with data served by the [NASA/IPAC Infrared Science Archive (IRSA)](https://irsa.ipac.caltech.edu).
 They cover topics like querying IRSA, working with catalogs in Parquet format, visualizing with Firefly, and general other techniques.
 
 
-## Accessing IRSA's on-premises holdings using VO protocols
 
-```{toctree}
----
-maxdepth: 1
-caption: VO on-prem data access
----
-tutorials/irsa-sia-examples/sia_2mass_allsky
-tutorials/irsa-sia-examples/sia_allwise_atlas
-tutorials/irsa-sia-examples/sia_cosmos
-tutorials/irsa-sia-examples/siav2_seip
-tutorials/cosmodc2/cosmoDC2_TAP_access.md
-
-```
-
-## Accessing IRSA's cloud holdings
-
-These notebooks demonstrate how to access the IRSA-curated datasets that available in Amazon Web Services (AWS) S3 cloud storage buckets.
-
-
-```{toctree}
----
-maxdepth: 1
-caption: Cloud data access
----
-
-tutorials/cloud_access/cloud-access-intro
-tutorials/parquet-catalog-demos/wise-allwise-catalog-demo
-tutorials/parquet-catalog-demos/neowise-source-table-strategies
-tutorials/parquet-catalog-demos/neowise-source-table-lightcurves
-tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey
-tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey
-tutorials/cloud_access/euclid-cloud-access
-
-```
-
-## Accessing Euclid data
-
-### Euclid Early Release Observation
-
-```{toctree}
----
-maxdepth: 1
-caption: Euclid Early Release Observations
----
-
-tutorials/euclid_access/Euclid_ERO
-
-```
-
-### Euclid Quick Release 1
-
-```{toctree}
----
-maxdepth: 1
-caption: Euclid Quick Release 1
----
-
-tutorials/euclid_access/1_Euclid_intro_MER_images
-tutorials/euclid_access/2_Euclid_intro_MER_catalog
-tutorials/euclid_access/3_Euclid_intro_1D_spectra
-tutorials/euclid_access/4_Euclid_intro_PHZ_catalog
-tutorials/euclid_access/5_Euclid_intro_SPE_catalog
-tutorials/cloud_access/euclid-cloud-access
-
-```
-
-## Accessing SPHEREx data
-
-```{toctree}
----
-maxdepth: 1
-caption: SPHEREx Data
----
-
-tutorials/spherex/spherex_intro
-
-```
-
-## Interactive visualization in Python with Firefly
-
-These notebooks demonstrate how to use the Firefly visualization tools from Python.
-[Firefly](https://github.com/Caltech-IPAC/firefly) is an open-source toolkit based on IVOA standards and designed to enable astronomical data archive access, exploratory data analysis, and visualization.
-
-It is used in archive user interfaces at [IRSA](https://irsa.ipac.caltech.edu), the [NASA Exoplanet Science Institute (NExScI)](https://nexsci.caltech.edu/), the [NASA/IPAC Extragalactic Database (NED)](https://ned.ipac.caltech.edu/), and the [Vera C. Rubin Observatory](https://www.lsst.org/).
-
-```{toctree}
----
-maxdepth: 1
-caption: Visualizations with Firefly
----
-
-tutorials/firefly/SEDs_in_Firefly
-tutorials/firefly/NEOWISE_light_curve_demo
-tutorials/firefly/OpenUniverse2024Preview_Firefly
-
-```
-
-## Simulated Data
-
-```{toctree}
----
-maxdepth: 1
-caption: Simulated Data
----
-
-tutorials/roman_simulations/roman_hlss_number_density.md
-
-```
-
-## Generally useful techniques
-
-These notebooks  cover miscellaneous topics that users might find useful in their analysis of IRSA-curated data.
-
-```{toctree}
----
-maxdepth: 1
-caption: Generic techniques
----
-
-tutorials/parallelize/Parallelize_Convolution
-
-```
-
-***
 
 ## About these notebooks
 
diff --git a/myst.yml b/myst.yml
@@ -0,0 +1,19 @@
+# See docs at: https://mystmd.org/guide/frontmatter
+version: 1
+project:
+  title: IRSA Tutorials
+  subject: IRSA Tutorials
+  description: IRSA Python Notebook tutorials to demostrate accessing data served by the NASA/IPAC Infrared Science Archive (IRSA)
+  keywords: [astronomy]
+  authors: [IRSA Scientists and Developers]
+  github: https://github.com/Caltech-IPAC/irsa-tutorials/
+  settings:
+    output_matplotlib_strings: remove
+extends:
+  - toc.yml
+site:
+  template: book-theme
+  options:
+    favicon: _static/irsa-favicon.ico
+    logo: _static/irsa_logo.png
+    outline_maxdepth: 1
diff --git a/site_requirements.txt b/site_requirements.txt
@@ -1,5 +1,2 @@
-sphinx
-# To pick up newest scroller and dark-mode features
-myst-nb>=1.3
-sphinx-book-theme
-sphinx-copybutton
+# We need JB2 here, the stable release is not yet available
+jupyter-book>=2.0.0b0
diff --git a/toc.yml b/toc.yml
@@ -0,0 +1,59 @@
+version: 1
+project:
+  toc:
+    - title: IRSA Tutorials
+      file: index.md
+    - title: SPHEREx
+      children:
+        - file: tutorials/spherex/spherex_intro.md
+    - title: Euclid
+      children:
+        - title: Euclid Early Release Observations
+          children:
+            - file: tutorials/euclid_access/Euclid_ERO.md
+        - title: Euclid Quick Release 1
+          children:
+            - file: tutorials/euclid_access/1_Euclid_intro_MER_images.md
+            - file: tutorials/euclid_access/2_Euclid_intro_MER_catalog.md
+            - file: tutorials/euclid_access/3_Euclid_intro_1D_spectra.md
+            - file: tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md
+            - file: tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md
+            - file: tutorials/cloud_access/euclid-cloud-access.md
+    - title: WISE
+      children:
+        - title: AllWISE
+          children:
+            - file: tutorials/irsa-sia-examples/sia_allwise_atlas.md
+            - file: tutorials/parquet-catalog-demos/wise-allwise-catalog-demo.md
+        - title: NEOWISE
+          children:
+            - file: tutorials/firefly/NEOWISE_light_curve_demo.md
+            - file: tutorials/parquet-catalog-demos/neowise-source-table-strategies.md
+            - file: tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md
+    - title: Spitzer
+      children:
+        - file: tutorials/irsa-sia-examples/siav2_seip.md
+    - title: 2MASS
+      children:
+        - file: tutorials/irsa-sia-examples/sia_2mass_allsky.md
+    - title: COSMOS
+      children:
+        - file: tutorials/irsa-sia-examples/sia_cosmos.md
+    - title: Simulated Data
+      children:
+        - title: Roman HLSS Galaxy Mock Catalog
+          children:
+            - file: tutorials/roman_simulations/roman_hlss_number_density.md
+        - title: OpenUniverse 2024
+          children:
+            - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey.md
+            - file: tutorials/firefly/OpenUniverse2024Preview_Firefly.md
+            - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey.md
+        - title: CosmoDC2
+          children:
+            - file: tutorials/cosmodc2/cosmoDC2_TAP_access.md
+    - title: Special Topics
+      children:
+        - file: tutorials/cloud_access/cloud-access-intro.md
+        - file: tutorials/firefly/SEDs_in_Firefly.md
+        - file: tutorials/parallelize/Parallelize_Convolution.md
diff --git a/tox.ini b/tox.ini
diff --git a/tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md b/tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md
diff --git a/tutorials/parquet-catalog-demos/neowise-source-table-strategies.md b/tutorials/parquet-catalog-demos/neowise-source-table-strategies.md

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
	`1`	`+# Both NEOWISE parquet notebooks work with large data that doesn't work within CircleCI or GHA resource limits`
`1`	`2`	`tutorials/parquet-catalog-demos/neowise-source-table-lightcurves`
`2`	`3`	`tutorials/parquet-catalog-demos/neowise-source-table-strategies`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# The way the notebooks use the multiprocessing module is known to not work on non-Linux`
	`2`	`+tutorials/parallelize/Parallelize_Convolution`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# Runs out of memory on GHA using the JB2 stack`
	`2`	`+tutorials/parallelize/Parallelize_Convolution`