Merge pull request #2636 from Trusted-AI/lizaitang-main_great_score

Lizaitang main great score

Author: beat-buesser

.github/actions/deepspeech-v3/Dockerfile

@@ -1,5 +1,5 @@
 # Get base image
-FROM python:3.10.13-slim-bookworm
+FROM python:3.10.17-slim-bookworm
 
 # Set to install things in non-interactive mode
 ENV DEBIAN_FRONTEND noninteractive
@@ -11,7 +11,6 @@ RUN apt-get update \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         libavcodec-extra \
         ffmpeg \
         curl \
@@ -25,10 +24,12 @@ RUN apt-get update \
      && rm -r /var/lib/apt/lists/*
 
 # Install necessary libraries for deepspeech v3
-RUN pip install --ignore-installed PyYAML torch==2.1.1 tensorflow==2.14.1 torchaudio==2.1.1 pytorch-lightning==2.1.2 scikit-learn==1.3.2
+RUN pip install --ignore-installed torch==2.1.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cpu
+RUN pip install --ignore-installed --no-deps pytorch-lightning==2.1.2
+RUN pip install --ignore-installed PyYAML tensorflow==2.14.1 scikit-learn==1.3.2
 RUN pip install --no-build-isolation fairscale==0.4.13
 
-RUN git clone https://github.com/SeanNaren/deepspeech.pytorch.git && cd deepspeech.pytorch && sed -i '/^sklearn/d' requirements.txt && pip install -r requirements.txt && pip install -e .
+RUN git clone https://github.com/SeanNaren/deepspeech.pytorch.git && cd deepspeech.pytorch && sed -i '/^sklearn/d;/^torch/d;/^torchaudio/d' requirements.txt && pip install -r requirements.txt && pip install -e .
 
 RUN pip install numba==0.56.4 pytest-cov==4.1.0 pydub==0.25.1
 

.github/actions/espresso/Dockerfile

@@ -1,18 +1,18 @@
 # Get base image
-FROM python:3.7.17-slim-bookworm
-
-# Set to install things in non-interactive mode
-ENV DEBIAN_FRONTEND noninteractive
+FROM python:3.10.17-slim-bookworm
 
 # Install system wide software
 RUN apt-get update \
      && apt-get install -y \
         build-essential \
+        gfortran \
+        libblas-dev \
+        liblapack-dev \
+        libatlas-base-dev \
         libgl1-mesa-glx \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         libavcodec-extra \
         ffmpeg \
         curl \
@@ -24,7 +24,10 @@ RUN apt-get update \
      && rm -r /var/lib/apt/lists/*
 
 # Install necessary libraries for espresso
-RUN pip install torch==1.6.0 torchaudio==0.6.0 pytest-cov~=4.1.0 scipy==1.2 scikit-learn==0.24 protobuf==3.20 kaldiio==2.18.0 tensorflow==2.9
+RUN pip install --upgrade pip setuptools wheel
+RUN pip install --ignore-installed torch==1.11.0 torchaudio==0.11.0 --index-url https://download.pytorch.org/whl/cpu
+RUN pip install numpy==1.21.6 scipy==1.9.3
+RUN pip install pytest-cov~=4.1.0 scikit-learn==1.6.1 protobuf==3.20 kaldiio==2.18.0 tensorflow==2.9
 RUN pip install --no-build-isolation fairscale==0.3.6
 
 RUN git clone https://github.com/beat-buesser/espresso

.github/actions/goturn/Dockerfile

@@ -1,5 +1,5 @@
 # Get base image
-FROM python:3.8.19-slim-bookworm
+FROM python:3.10.17-slim-bookworm
 
 ENV MONO_TLS_PROVIDER=legacy
 
@@ -11,7 +11,6 @@ RUN apt-get update \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         libavcodec-extra \
         ffmpeg \
         curl \
@@ -26,12 +25,14 @@ RUN apt-get update \
      && rm -r /var/lib/apt/lists/*
 
 # Install necessary libraries for goturn
-RUN pip install torch==1.9.1 torchvision==0.10.1 tensorflow==2.9 scikit-learn==0.24 pytest~=6.2.5 pytest-cov~=4.1.0
+RUN pip install --ignore-installed torch==1.11.0 torchaudio==0.11.0 torchvision==0.12.0 --index-url https://download.pytorch.org/whl/cpu
+RUN pip install --ignore-installed --no-deps pytorch_lightning==0.7.1 torch_lr_finder==0.2.1 torchsummary==1.5.1
+RUN pip install tensorflow==2.9 scikit-learn==1.6.1 pytest~=6.2.5 pytest-cov~=4.1.0 matplotlib==3.9.4
 
 RUN git clone https://github.com/nrupatunga/goturn-pytorch.git /tmp/goturn-pytorch
 #RUN cd /tmp/goturn-pytorch && pip install -r requirements.txt
-RUN pip install loguru==0.5.3 torchsummary==1.5.1 tqdm==4.62.3 pytorch_lightning==0.7.1 imutils==0.5.3 torch_lr_finder==0.2.1 numpy==1.21.6 Pillow==8.0.1 visdom==0.1.8.9 gdown==5.1.0
-RUN pip3 install opencv_python==4.9.0.80 --no-build-isolation
+RUN pip install loguru==0.5.3 tqdm==4.62.3 imutils==0.5.3 numpy==1.23.5 Pillow==8.0.1 visdom==0.1.8.9 gdown==5.1.0
+RUN pip install opencv_python==4.9.0.80 --no-build-isolation
 
 ENV PYTHONPATH "${PYTHONPATH}:/tmp/goturn-pytorch/src"
 ENV PYTHONPATH "${PYTHONPATH}:/tmp/goturn-pytorch/src/scripts"

.github/actions/tfv2-faster-rcnn/Dockerfile

@@ -1,8 +1,5 @@
-# Get base from a tensorflow image
-FROM tensorflow/tensorflow:2.13.0
-
-# Set to install things in non-interactive mode
-ENV DEBIAN_FRONTEND noninteractive
+# Get base image
+FROM python:3.8.20-slim-bookworm
 
 # Install system wide softwares
 RUN apt-get update \
@@ -11,36 +8,28 @@ RUN apt-get update \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         libavcodec-extra \
         ffmpeg \
         curl \
         libsndfile-dev \
         libsndfile1 \
         wget \
         unzip \
+        software-properties-common \
+        gnupg2 \
+        lsb-release \
      && apt-get clean all \
      && rm -r /var/lib/apt/lists/*
 
-RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-RUN bash Miniconda3-latest-Linux-x86_64.sh -b -p /miniconda
-RUN /miniconda/bin/conda install --yes \
-    astropy \
-    matplotlib \
-    pandas \
-    scikit-learn \
-    scikit-image
+RUN pip install --upgrade pip setuptools wheel
 
 # Install necessary libraries for tensorflow faster rcnn
 RUN mkdir TensorFlow
 RUN cd TensorFlow && git clone https://github.com/tensorflow/models
 RUN cd TensorFlow && wget https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protoc-3.17.3-linux-x86_64.zip
 RUN cd TensorFlow && unzip protoc-3.17.3-linux-x86_64.zip -d protobuf
 RUN cd TensorFlow/models/research && /TensorFlow/protobuf/bin/protoc object_detection/protos/*.proto --python_out=.
-RUN cd TensorFlow/models/research && cp object_detection/packages/tf2/setup.py .
+RUN cd TensorFlow/models/research && cp object_detection/packages/tf2/setup.py . && sed -i "/^'tensorflow_io',$/d; /^'keras',$/d" setup.py
 RUN cd TensorFlow/models/research && python -m pip install .
 
-RUN pip install tqdm
-RUN pip install requests
-RUN pip install numba==0.50.0
-RUN pip install pytest-cov
+RUN pip install numpy matplotlib tqdm requests llvmlite numba==0.56.4 pytest-cov "pytest<8.0.0" "exceptiongroup<1.1.0" tensorflow==2.6.0 keras==2.6.0

.github/actions/yolo/Dockerfile

@@ -1,5 +1,5 @@
 # Get base image
-FROM python:3.9.19-slim
+FROM python:3.10.17-slim-bookworm
 
 # Install system wide software
 RUN apt-get update \
@@ -8,7 +8,6 @@ RUN apt-get update \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         cmake \
         libavcodec-extra \
         ffmpeg \
@@ -20,7 +19,8 @@ RUN apt-get update \
      && apt-get clean all \
      && rm -r /var/lib/apt/lists/*
 
-RUN pip install pytorchyolo==1.8.0 tensorflow==2.14.1 scikit-learn==1.4.2 pytest-cov~=4.1.0 torch==1.12.1
+RUN pip install --ignore-installed torch==1.12.1 --index-url https://download.pytorch.org/whl/cpu
+RUN pip install pytorchyolo==1.8.0 tensorflow==2.14.1 scikit-learn==1.4.2 pytest-cov~=4.1.0
 
 RUN cd /tmp/ && git clone https://github.com/eriklindernoren/PyTorch-YOLOv3.git && cd ./PyTorch-YOLOv3/weights && ./download_weights.sh
 

art/evaluations/great_score/__init__.py

@@ -0,0 +1,5 @@
+"""
+This module implements the evaluation of GREAT Score.
+"""
+
+from art.evaluations.great_score.great_score import GreatScorePyTorch

art/evaluations/great_score/great_score.py

@@ -0,0 +1,58 @@
+# MIT License
+#
+# Copyright (C) The Adversarial Robustness Toolbox (ART) Authors 2024
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import numpy as np
+
+from art.evaluations.evaluation import Evaluation
+
+
+class GreatScorePyTorch(Evaluation):
+
+    def __init__(self, classifier):
+        """
+        Calculate the GREAT score and accuracy for given samples using the specified classifier.
+
+        :param classifier: ART Classifier of the model to evaluate.
+        """
+        self.classifier = classifier
+        super().__init__()
+
+    def evaluate(self, x: np.ndarray, y: np.ndarray) -> tuple[list[float], float]:
+        """
+        Calculate the GREAT score and accuracy for given samples using the specified model.
+
+        :param x: Input samples (images) as a numpy array.
+        :param y: True labels for the samples.
+
+        :return: great_score, accuracy
+        """
+        # Calculate accuracy
+        y_pred: np.ndarray = self.classifier.predict(x=x)
+        correct_predictions: np.ndarray = np.argmax(y_pred, axis=1) == np.argmax(y, axis=1)
+        accuracy = np.mean(correct_predictions)
+
+        # Calculate the GREAT score
+        k = 2
+        top2_indices = np.argpartition(y_pred, -k, axis=1)[:, -k:]  # (batch_size, 2), unsorted
+        top2_values = np.take_along_axis(y_pred, top2_indices, axis=1)  # corresponding values
+        # Now sort these top-2 values and indices in descending order
+        sorted_order = np.argsort(top2_values, axis=1)[:, ::-1]
+        top2_values_sorted = np.take_along_axis(top2_values, sorted_order, axis=1)
+        great_scores = np.where(correct_predictions, top2_values_sorted[:, 0] - top2_values_sorted[:, 1], 0)
+        average_great_score = np.mean(great_scores)
+
+        return average_great_score, accuracy

setup.py

@@ -119,9 +119,9 @@ def get_version(rel_path):
         "Intended Audience :: Education",
         "Intended Audience :: Science/Research",
         "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Topic :: Software Development :: Libraries",
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Topic :: Scientific/Engineering :: Artificial Intelligence",

tests/evaluations/test_great_score.py

@@ -0,0 +1,37 @@
+# MIT License
+#
+# Copyright (C) The Adversarial Robustness Toolbox (ART) Authors 2024
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import pytest
+
+from art.evaluations.great_score import GreatScorePyTorch
+
+from tests.utils import ARTTestException
+
+
+@pytest.mark.only_with_platform("pytorch")
+def test_great_score(art_warning, image_dl_estimator, get_mnist_dataset):
+    try:
+        classifier, _ = image_dl_estimator(from_logits=False)
+        (_, _), (x_test, y_test) = get_mnist_dataset
+
+        great_score = GreatScorePyTorch(classifier=classifier)
+        score, accuracy = great_score.evaluate(x=x_test, y=y_test)
+        assert score == pytest.approx(0.03501499, rel=1e-6)
+        assert accuracy == pytest.approx(0.2487, rel=1e-4)
+
+    except ARTTestException as e:
+        art_warning(e)