Merge remote-tracking branch 'upstream/develop' into v1.4.6

Author: martinkilbinger

.github/workflows/deploy-image.yml

@@ -0,0 +1,52 @@
+name: Create and publish a Docker image
+
+on:
+  push:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  BRANCH: ${{ github.ref }}
+
+jobs:
+  build-and-push-image:
+    runs-on: 
+      - ubuntu-latest
+      # - macos-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          load: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Test
+        run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sp_validation"
+      
+      - name: Push
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -145,7 +145,7 @@ notebooks/validation*.py
 notebooks/main_set_up.py
 notebooks/analyse_shear_cat/tutorial_UNIONS_SP_v1.0.py
 
-notebooks/cosmo_val/output
+notebooks/cosmo_val/output*
 notebooks/cosmo_val/cosmo_val.ipynb
 notebooks/cosmo_val/match_LF_SP.ipynb
 

Dockerfile

@@ -0,0 +1,18 @@
+# Development image with more bells and whistles
+FROM ghcr.io/cosmostat/shapepipe:develop
+
+RUN apt-get update -y --quiet --fix-missing && \
+    apt-get dist-upgrade -y --quiet --fix-missing && \
+    apt-get install -y --quiet \
+        libgsl-dev \
+        htop \
+        tmux
+
+RUN pip install --no-cache-dir \ 
+    snakemake
+
+WORKDIR /sp_validation
+COPY . /sp_validation
+
+# Install sp_validation
+RUN pip install --no-cache-dir -e .

README.md

@@ -33,6 +33,32 @@ tasks that can be performed by `sp_validation` are:
   a shear validation run and performes cosmology inference using the two-point
   correlation function.
 
+## Container Installation (Recommended)
+
+The easiest way to install sp_validation is via a container. Docker images are automatically built and pushed to the [Github Container Registry (GHCR)](ghcr.io/cosmostat/sp_validation) for each release. This image can be installed and run on most systems (including clusters) with just a few lines of code.
+
+We recommend running the image with **Apptainer** (formerly Singularity) which is installed on most HPC clusters. To simply run the image, use the following command:
+
+```bash
+# build writeable "sandbox" container in the current directory
+# ./sp_validation will be a directory that functions like a vm
+apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop
+
+# open a shell in the container
+apptainer shell --writable sp_validation 
+# and confirm that the installation was successful
+python -c "import sp_validation"
+```
+
+You can also run the image with **Docker**:
+
+```bash
+docker run --rm -it ghcr.io/cosmostat/sp_validation:pyproject_docker python -c "import sp_validation"  
+```
+
+We do not currently build images for Apple Silicon/amr64; however the amd64 images should work on these systems, albeit with reduced performance.
+
+
 
 ## Flow chart
 

cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/.gitignore

@@ -0,0 +1,3 @@
+results/*
+plots/*
+.snakemake/*

cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/config.yaml

@@ -0,0 +1,19 @@
+version: SP_v1.4.5
+plot_style: config/paper.mplstyle
+
+ini_dirs:
+  - /n23data1/n06data/lgoh/scratch/UNIONS/cosmo_inference/cosmosis_config/pipeline/
+  - /home/guerrini/sp_validation/cosmo_inference/cosmosis_config/
+
+
+# chain: SP_v1.4.5_C_sc_10_150_new_ia
+chain: SP_v1.4.5_A_sc_10_60
+
+npatch: 256
+min_sep: 0.1
+max_sep: 250
+nbins: 20
+
+min_sep_int: 0.08
+max_sep_int: 300
+nbins_int: 100

cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/paper.mplstyle

@@ -0,0 +1,13 @@
+figure.titlesize : 14
+figure.titleweight : bold
+figure.autolayout : True
+axes.titlesize : 12
+legend.fontsize : 8
+lines.linewidth : 1.5
+lines.markersize : 4
+axes.labelsize : 10
+xtick.labelsize : 10
+ytick.labelsize : 10
+text.usetex : True
+font.family : serif
+font.size : 12

cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/Snakefile

@@ -0,0 +1,60 @@
+configfile: "config/config.yaml"
+
+
+def find_cosmosis_inputs():
+    chain = config["chain"]
+    for ini_dir in config["ini_dirs"]:
+        ini_file = Path(ini_dir) / f"cosmosis_pipeline_{chain}.ini"
+        if os.path.exists(ini_file):
+            return ini_file
+    raise FileNotFoundError(
+        f"Cosmosis input file not found for chain {chain} in specified directories."
+    )
+
+
+rule get_bestfit_model:
+    input:
+        inference_ini=find_cosmosis_inputs(),
+    output:
+        bestfit_values_ini=f"results/{config['chain']}/bestfit_values.ini",
+        cosmosis_ini=f"results/{config['chain']}/cosmosis.ini",
+        xi_shear=f"results/{config['chain']}/bestfit_xi_shear.txt",
+        xi_sys=f"results/{config['chain']}/bestfit_xi_sys.txt",
+    params:
+        bestfit_dir=f"results/{config['chain']}",
+    script:
+        "scripts/get_bestfit_model.py"
+
+
+rule paper_plots:
+    input:
+        bestfit_xi_shear=f"results/{config['chain']}/bestfit_xi_shear.txt",
+        bestfit_xi_sys=f"results/{config['chain']}/bestfit_xi_sys.txt",
+        xi_reporting="/n17data/cdaley/unions/sp_validation/notebooks/cosmo_val/output/{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt".format(
+            **config
+        ),
+        xi_integration="/n17data/cdaley/unions/sp_validation/notebooks/cosmo_val/output/{version}_xi_minsep={min_sep_int}_maxsep={max_sep_int}_nbins={nbins_int}_npatch={npatch}.txt".format(
+            **config
+        ),
+    output:
+        xis="/n17data/cdaley/unions/pure_eb/results/paper_plots/{version}_eb_with_bestfit_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_xis.pdf".format(
+            **config
+        ),
+        ptes="/n17data/cdaley/unions/pure_eb/results/paper_plots/{version}_eb_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_ptes.pdf".format(
+            **config
+        ),
+        cov="/n17data/cdaley/unions/pure_eb/results/paper_plots/{version}_eb_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_cov.pdf".format(
+            **config
+        ),
+    params:
+        version=config["version"],
+        min_sep=lambda params: float(config["min_sep"]),
+        max_sep=lambda params: int(config["max_sep"]),
+        nbins=lambda params: int(config["nbins"]),
+        min_sep_int=lambda params: float(config["min_sep_int"]),
+        max_sep_int=lambda params: int(config["max_sep_int"]),
+        nbins_int=lambda params: int(config["nbins_int"]),
+        npatch=lambda params: int(config["npatch"]),
+    threads: 1
+    script:
+        "paper_plots.py"