MAINT: reprganize ignore testing files

Author: bsipocz

ignore_circleci_testing

@@ -0,0 +1,13 @@
+# ignore these 10 now as excluding execution for these is a combination that passes JB2 building on CircleCI.
+# Note that the first 2 are excluded for all kinds of testing; while the parallelize causes issues for GHA JB2 builds and non linux testing, too.
+#
+tutorials/parquet-catalog-demos/neowise-source-table-lightcurves
+tutorials/parquet-catalog-demos/neowise-source-table-strategies
+tutorials/parallelize/Parallelize_Convolution
+tutorials/roman_simulations/roman_hlss_number_density
+tutorials/cosmodc2/cosmoDC2_TAP_access
+tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey
+tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey
+tutorials/parquet-catalog-demos/wise-allwise-catalog-demo
+tutorials/cloud_access/euclid-cloud-access
+tutorials/cloud_access/cloud-access-intro

ignore_osx_testing

@@ -1,2 +1,3 @@
+# Both NEOWISE parquet notebooks work with large data that doesn't work within CircleCI or GHA resource limits
 tutorials/parquet-catalog-demos/neowise-source-table-lightcurves
 tutorials/parquet-catalog-demos/neowise-source-table-strategies

ignore_testing renamed to ignore_tutorials/ignore_all_testing

@@ -0,0 +1,2 @@
+# The way the notebooks use the multiprocessing module is known to not work on non-Linux
+tutorials/parallelize/Parallelize_Convolution

ignore_tutorials/ignore_circleci_testing

@@ -1,2 +1,2 @@
 # Runs out of memory on GHA using the JB2 stack
-tutorials/parallelize/Parallelize_Convolution.md
+tutorials/parallelize/Parallelize_Convolution

ignore_gha_testing renamed to ignore_tutorials/ignore_gha_testing

@@ -0,0 +1,2 @@
+# The way the notebooks use the multiprocessing module is known to not work on non-Linux
+tutorials/parallelize/Parallelize_Convolution

ignore_tutorials/ignore_osx_testing

@@ -43,17 +43,19 @@ allowlist_externals = bash, jupyter-book
 commands =
     pip freeze
 
-    # Ignore testing the tutorials listed in ignore_testing file. We have some OS specific ignores,
-    # too due to issues with e.g. multiprocessing and problems in upstream dependency
-    bash -c 'if python -c "import platform; print(platform.platform())" | grep -i macos; then cat ignore_osx_testing >> ignore_testing; fi'
-    bash -c 'if python -c "import platform; print(platform.platform())" | grep -i win; then cat ignore_windows_testing >> ignore_testing; fi'
-    bash -c 'if [[ $CI == true ]]; then cat ignore_gha_testing >> ignore_testing; fi'
-    bash -c 'if [[ $CIRCLECI == true ]]; then cat ignore_circleci_testing >> ignore_testing; fi'
+    # Ignore some tutorials for various types of testing. The individual lists are stored in the `ignore_tutorials`
+    # directory and are documented about the specific reasons of them included in the ignores. These files are kept under version control.
+    # The files `ignore_testing` and `ignore_execution` are generated during the CI jobs and are not under version control.
+    bash -c 'cat ignore_tutorials/ignore_all_testing >> ignore_testing'
+    bash -c 'if python -c "import platform; print(platform.platform())" | grep -i macos; then cat ignore_tutorials/ignore_osx_testing >> ignore_testing; fi'
+    bash -c 'if python -c "import platform; print(platform.platform())" | grep -i win; then cat ignore_tutorials/ignore_windows_testing >> ignore_testing; fi'
+    bash -c 'if [[ $CI == true ]]; then cat ignore_tutorials/ignore_gha_testing >> ignore_testing; fi'
+    bash -c 'if [[ $CIRCLECI == true ]]; then cat ignore_tutorials/ignore_circleci_testing >> ignore_testing; fi'
 
-    buildhtml: bash -c "cat ignore_rendenring >> ignore_testing"
+    buildhtml: bash -c 'cat ignore_tutorials/ignore_rendenring >> ignore_testing'
 
     # We only want to run CI in PRs for the notebooks we touched
-    !buildhtml: bash -c 'if [[ $GITHUB_EVENT_NAME == pull_request && -z "$(grep force_run:all_tests ${GITHUB_EVENT_PATH})" ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | grep .md; else find tutorials -name "*.md"; fi | grep -vf ignore_testing | xargs jupytext --to notebook '
+    !buildhtml: bash -c 'if [[ $GITHUB_EVENT_NAME == pull_request && -z "$(grep force_run:all_tests ${GITHUB_EVENT_PATH})" ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | grep .md; else find tutorials -name "*.md"; fi | grep -vf ignore_testing/ignore_testing | xargs jupytext --to notebook '
 
     !buildhtml: pytest --nbval-lax -vv --suppress-no-test-exit-code --durations=10 tutorials