diff --git a/.circleci/config.yml b/.circleci/config.yml index bdf639e8..603fdd12 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,46 +1,63 @@ version: 2.1 -jobs: +jobs: build-docs: docker: - - image: cimg/python:3.12 - + - image: cimg/python:3.13 steps: - - attach_workspace: - at: ~/ - - checkout - - run: - name: Install CI dependencies - command: python -m pip install --upgrade tox + - restore_cache: + keys: + - node-cache-v1 - restore_cache: keys: - - jupyter_ch + - pip-cache-{{ checksum "site_requirements.txt" }} + + - run: + name: Install Node.js + command: | + curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - + sudo apt-get install -y nodejs + + - run: + name: Install mystmd + command: | + npm install mystmd - run: - name: Build HTML rendering of notebooks - no_output_timeout: 30m + name: Install Python dependencies command: | - python -m tox -e py312-buildhtml + python -m pip install --upgrade pip tox - save_cache: - key: jupyter_ch + key: node-cache-v1 paths: - - _build/.jupyter_cache + - node_modules - - store_artifacts: - path: _build/html + - run: + name: Build documentation + no_output_timeout: 60m + environment: + # Ensure this is same as store_artifacts path below + DOCS_PATH: _build/html + command: | + export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH" - - persist_to_workspace: - root: _build + python -m tox -e py313-buildhtml + + - save_cache: + key: pip-cache-{{ checksum "site_requirements.txt" }} paths: - - html + - ~/.cache/pip + + - store_artifacts: + path: _build/html workflows: version: 2 - default: + build-and-docs: jobs: - build-docs diff --git a/ignore_circleci_testing b/ignore_circleci_testing new file mode 100644 index 00000000..09605002 --- /dev/null +++ b/ignore_circleci_testing @@ -0,0 +1,2 @@ +tutorials/parallelize/Parallelize_Convolution +tutorials/parquet-catalog-demos/wise-allwise-catalog-demo diff --git a/ignore_testing b/ignore_testing index e69de29b..ff623959 100644 --- a/ignore_testing +++ b/ignore_testing @@ -0,0 +1,2 @@ +tutorials/cloud_access/euclid-cloud-access.md +tutorials/roman_simulations/roman_hlss_number_density.md diff --git a/index.md b/index.md index 807a00bb..e4dea84d 100644 --- a/index.md +++ b/index.md @@ -1,135 +1,11 @@ -# Caltech/IPAC--IRSA Python Notebook Tutorials +# Caltech/IPAC—IRSA Python Notebook Tutorials These Python Jupyter Notebook tutorials demonstrate access methods and techniques for working with data served by the [NASA/IPAC Infrared Science Archive (IRSA)](https://irsa.ipac.caltech.edu). They cover topics like querying IRSA, working with catalogs in Parquet format, visualizing with Firefly, and general other techniques. -## Accessing IRSA's on-premises holdings using VO protocols -```{toctree} ---- -maxdepth: 1 -caption: VO on-prem data access ---- -tutorials/irsa-sia-examples/sia_2mass_allsky -tutorials/irsa-sia-examples/sia_allwise_atlas -tutorials/irsa-sia-examples/sia_cosmos -tutorials/irsa-sia-examples/siav2_seip -tutorials/cosmodc2/cosmoDC2_TAP_access.md - -``` - -## Accessing IRSA's cloud holdings - -These notebooks demonstrate how to access the IRSA-curated datasets that available in Amazon Web Services (AWS) S3 cloud storage buckets. - - -```{toctree} ---- -maxdepth: 1 -caption: Cloud data access ---- - -tutorials/cloud_access/cloud-access-intro -tutorials/parquet-catalog-demos/wise-allwise-catalog-demo -tutorials/parquet-catalog-demos/neowise-source-table-strategies -tutorials/parquet-catalog-demos/neowise-source-table-lightcurves -tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey -tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey -tutorials/cloud_access/euclid-cloud-access - -``` - -## Accessing Euclid data - -### Euclid Early Release Observation - -```{toctree} ---- -maxdepth: 1 -caption: Euclid Early Release Observations ---- - -tutorials/euclid_access/Euclid_ERO - -``` - -### Euclid Quick Release 1 - -```{toctree} ---- -maxdepth: 1 -caption: Euclid Quick Release 1 ---- - -tutorials/euclid_access/1_Euclid_intro_MER_images -tutorials/euclid_access/2_Euclid_intro_MER_catalog -tutorials/euclid_access/3_Euclid_intro_1D_spectra -tutorials/euclid_access/4_Euclid_intro_PHZ_catalog -tutorials/euclid_access/5_Euclid_intro_SPE_catalog -tutorials/cloud_access/euclid-cloud-access - -``` - -## Accessing SPHEREx data - -```{toctree} ---- -maxdepth: 1 -caption: SPHEREx Data ---- - -tutorials/spherex/spherex_intro - -``` - -## Interactive visualization in Python with Firefly - -These notebooks demonstrate how to use the Firefly visualization tools from Python. -[Firefly](https://github.com/Caltech-IPAC/firefly) is an open-source toolkit based on IVOA standards and designed to enable astronomical data archive access, exploratory data analysis, and visualization. - -It is used in archive user interfaces at [IRSA](https://irsa.ipac.caltech.edu), the [NASA Exoplanet Science Institute (NExScI)](https://nexsci.caltech.edu/), the [NASA/IPAC Extragalactic Database (NED)](https://ned.ipac.caltech.edu/), and the [Vera C. Rubin Observatory](https://www.lsst.org/). - -```{toctree} ---- -maxdepth: 1 -caption: Visualizations with Firefly ---- - -tutorials/firefly/SEDs_in_Firefly -tutorials/firefly/NEOWISE_light_curve_demo -tutorials/firefly/OpenUniverse2024Preview_Firefly - -``` - -## Simulated Data - -```{toctree} ---- -maxdepth: 1 -caption: Simulated Data ---- - -tutorials/roman_simulations/roman_hlss_number_density.md - -``` - -## Generally useful techniques - -These notebooks cover miscellaneous topics that users might find useful in their analysis of IRSA-curated data. - -```{toctree} ---- -maxdepth: 1 -caption: Generic techniques ---- - -tutorials/parallelize/Parallelize_Convolution - -``` - -*** ## About these notebooks diff --git a/myst.yml b/myst.yml new file mode 100644 index 00000000..4df49601 --- /dev/null +++ b/myst.yml @@ -0,0 +1,16 @@ +# See docs at: https://mystmd.org/guide/frontmatter +version: 1 +project: + title: IRSA Tutorials + subject: IRSA Tutorials + description: IRSA Python Notebook tutorials to demostrate accessing data served by the NASA/IPAC Infrared Science Archive (IRSA) + keywords: [astronomy] + authors: [IRSA Scientists and Developers] + github: https://github.com/Caltech-IPAC/irsa-tutorials/ +extends: + - toc.yml +site: + template: book-theme + options: + favicon: _static/irsa-favicon.ico + logo: _static/irsa_logo.png diff --git a/toc.yml b/toc.yml new file mode 100644 index 00000000..cac05583 --- /dev/null +++ b/toc.yml @@ -0,0 +1,61 @@ +version: 1 +project: + toc: + - title: IRSA Tutorials + file: index.md + - title: SPHEREx + children: + - file: tutorials/spherex/spherex_intro.md + - title: Euclid + children: + - title: Euclid Early Release Observations + children: + - file: tutorials/euclid_access/Euclid_ERO.md + - title: Euclid Quick Release 1 + children: + - file: tutorials/euclid_access/1_Euclid_intro_MER_images.md + - file: tutorials/euclid_access/2_Euclid_intro_MER_catalog.md + - file: tutorials/euclid_access/3_Euclid_intro_1D_spectra.md + - file: tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md + - file: tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md + - file: tutorials/cloud_access/euclid-cloud-access.md + - title: WISE + children: + - title: AllWISE + children: + - file: tutorials/irsa-sia-examples/sia_allwise_atlas.md + - file: tutorials/parquet-catalog-demos/wise-allwise-catalog-demo.md + - title: NEOWISE + children: + - file: tutorials/firefly/NEOWISE_light_curve_demo.md + - file: tutorials/parquet-catalog-demos/neowise-source-table-strategies.md + - file: tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md + - title: Spitzer + children: + - title: Spitzer Enhanced Imaging Products (SEIP) + children: + - file: tutorials/irsa-sia-examples/siav2_seip.md + - title: 2MASS + children: + - file: tutorials/irsa-sia-examples/sia_2mass_allsky.md + - title: COSMOS + children: + - file: tutorials/irsa-sia-examples/sia_cosmos.md + - title: Simulated Data + children: + - title: Roman HLSS Galaxy Mock Catalog + children: + - file: tutorials/roman_simulations/roman_hlss_number_density.md + - title: OpenUniverse 2024 + children: + - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey.md + - file: tutorials/firefly/OpenUniverse2024Preview_Firefly.md + - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey.md + - title: CosmoDC2 + children: + - file: tutorials/cosmodc2/cosmoDC2_TAP_access.md + - title: Special Topics + children: + - file: tutorials/cloud_access/cloud-access-intro.md + - file: tutorials/firefly/SEDs_in_Firefly.md + - file: tutorials/parallelize/Parallelize_Convolution.md diff --git a/tox.ini b/tox.ini index f84e03ce..0fb91807 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ setenv = FIREFLY_URL = https://irsa.ipac.caltech.edu/irsaviewer # fallback url for firefly notebooks if not defined in code devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple https://pypi.anaconda.org/astropy/simple -passenv = CI, CIRCLECI, GITHUB_EVENT_NAME, GITHUB_EVENT_PATH +passenv = CI, CIRCLECI, GITHUB_EVENT_NAME, GITHUB_EVENT_PATH, BASE_URL deps = # We use these files to specify all the dependencies, and below we override @@ -38,26 +38,27 @@ deps = devdeps: git+https://github.com/astropy/pyvo.git#egg=pyvo devdeps: git+https://github.com/astropy/astroquery.git#egg=astroquery -allowlist_externals = bash, sed +allowlist_externals = bash, npx commands = pip freeze # Ignore testing the tutorials listed in ignore_testing file. We have some OS specific ignores, # too due to issues with e.g. multiprocessing and problems in upstream dependency - !buildhtml: bash -c 'if python -c "import platform; print(platform.platform())" | grep -i macos; then cat ignore_osx_testing >> ignore_testing; fi' - !buildhtml: bash -c 'if python -c "import platform; print(platform.platform())" | grep -i win; then cat ignore_windows_testing >> ignore_testing; fi' - - !buildhtml: bash -c 'if [[ $CI == true ]]; then cat ignore_gha_testing >> ignore_testing; fi' + bash -c 'if python -c "import platform; print(platform.platform())" | grep -i macos; then cat ignore_osx_testing >> ignore_testing; fi' + bash -c 'if python -c "import platform; print(platform.platform())" | grep -i win; then cat ignore_windows_testing >> ignore_testing; fi' + bash -c 'if [[ $CI == true ]]; then cat ignore_gha_testing >> ignore_testing; fi' + bash -c 'if [[ $CIRCLECI == true ]]; then cat ignore_circleci_testing >> ignore_testing; fi' # We only want to run CI in PRs for the notebooks we touched !buildhtml: bash -c 'if [[ $GITHUB_EVENT_NAME == pull_request && -z "$(grep force_run:all_tests ${GITHUB_EVENT_PATH})" ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | grep .md; else find tutorials -name "*.md"; fi | grep -vf ignore_testing | xargs jupytext --to notebook ' !buildhtml: pytest --nbval-lax -vv --suppress-no-test-exit-code --durations=10 tutorials - buildhtml: sphinx-build -b html . _build/html -nWT --keep-going - # SED magic to remove the toctree captions from the rendered index page while keeping them in the sidebar TOC - buildhtml: sed -E -i.bak '/caption-text/{N; s/.+caption-text.+\n