Merge pull request #2678 from Trusted-AI/dev_1.20.0

Update to ART 1.20.0

Author: beat-buesser

.github/ISSUE_TEMPLATE/bug_report.md

@@ -24,4 +24,4 @@ If applicable, add screenshots to help explain your problem.
  - OS
  - Python version
  - ART version or commit number
- - TensorFlow / Keras / PyTorch / MXNet version
+ - TensorFlow / Keras / PyTorch version

.github/actions/deepspeech-v3/Dockerfile

@@ -1,5 +1,5 @@
 # Get base image
-FROM python:3.10.13-slim-bookworm
+FROM python:3.10.17-slim-bookworm
 
 # Set to install things in non-interactive mode
 ENV DEBIAN_FRONTEND noninteractive
@@ -11,7 +11,6 @@ RUN apt-get update \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         libavcodec-extra \
         ffmpeg \
         curl \
@@ -25,10 +24,12 @@ RUN apt-get update \
      && rm -r /var/lib/apt/lists/*
 
 # Install necessary libraries for deepspeech v3
-RUN pip install --ignore-installed PyYAML torch==2.1.1 tensorflow==2.14.1 torchaudio==2.1.1 pytorch-lightning==2.1.2 scikit-learn==1.3.2
+RUN pip install --ignore-installed torch==2.1.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cpu
+RUN pip install --ignore-installed --no-deps pytorch-lightning==2.1.2
+RUN pip install --ignore-installed PyYAML tensorflow==2.14.1 scikit-learn==1.3.2
 RUN pip install --no-build-isolation fairscale==0.4.13
 
-RUN git clone https://github.com/SeanNaren/deepspeech.pytorch.git && cd deepspeech.pytorch && sed -i '/^sklearn/d' requirements.txt && pip install -r requirements.txt && pip install -e .
+RUN git clone https://github.com/SeanNaren/deepspeech.pytorch.git && cd deepspeech.pytorch && sed -i '/^sklearn/d;/^torch/d;/^torchaudio/d' requirements.txt && pip install -r requirements.txt && pip install -e .
 
 RUN pip install numba==0.56.4 pytest-cov==4.1.0 pydub==0.25.1
 

.github/actions/espresso/Dockerfile

@@ -1,18 +1,18 @@
 # Get base image
-FROM python:3.7.17-slim-bookworm
-
-# Set to install things in non-interactive mode
-ENV DEBIAN_FRONTEND noninteractive
+FROM python:3.10.17-slim-bookworm
 
 # Install system wide software
 RUN apt-get update \
      && apt-get install -y \
         build-essential \
+        gfortran \
+        libblas-dev \
+        liblapack-dev \
+        libatlas-base-dev \
         libgl1-mesa-glx \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         libavcodec-extra \
         ffmpeg \
         curl \
@@ -24,7 +24,10 @@ RUN apt-get update \
      && rm -r /var/lib/apt/lists/*
 
 # Install necessary libraries for espresso
-RUN pip install torch==1.6.0 torchaudio==0.6.0 pytest-cov~=4.1.0 scipy==1.2 scikit-learn==0.24 protobuf==3.20 kaldiio==2.18.0 tensorflow==2.9
+RUN pip install --upgrade pip setuptools wheel
+RUN pip install --ignore-installed torch==1.11.0 torchaudio==0.11.0 --index-url https://download.pytorch.org/whl/cpu
+RUN pip install numpy==1.21.6 scipy==1.9.3
+RUN pip install pytest-cov~=4.1.0 scikit-learn==1.6.1 protobuf==3.20 kaldiio==2.18.0 tensorflow==2.9
 RUN pip install --no-build-isolation fairscale==0.3.6
 
 RUN git clone https://github.com/beat-buesser/espresso

.github/actions/goturn/Dockerfile

@@ -1,5 +1,5 @@
 # Get base image
-FROM python:3.8.19-slim-bookworm
+FROM python:3.10.17-slim-bookworm
 
 ENV MONO_TLS_PROVIDER=legacy
 
@@ -11,7 +11,6 @@ RUN apt-get update \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         libavcodec-extra \
         ffmpeg \
         curl \
@@ -26,12 +25,14 @@ RUN apt-get update \
      && rm -r /var/lib/apt/lists/*
 
 # Install necessary libraries for goturn
-RUN pip install torch==1.9.1 torchvision==0.10.1 tensorflow==2.9 scikit-learn==0.24 pytest~=6.2.5 pytest-cov~=4.1.0
+RUN pip install --ignore-installed torch==1.11.0 torchaudio==0.11.0 torchvision==0.12.0 --index-url https://download.pytorch.org/whl/cpu
+RUN pip install --ignore-installed --no-deps pytorch_lightning==0.7.1 torch_lr_finder==0.2.1 torchsummary==1.5.1
+RUN pip install tensorflow==2.9 scikit-learn==1.6.1 pytest~=6.2.5 pytest-cov~=4.1.0 matplotlib==3.9.4
 
 RUN git clone https://github.com/nrupatunga/goturn-pytorch.git /tmp/goturn-pytorch
 #RUN cd /tmp/goturn-pytorch && pip install -r requirements.txt
-RUN pip install loguru==0.5.3 torchsummary==1.5.1 tqdm==4.62.3 pytorch_lightning==0.7.1 imutils==0.5.3 torch_lr_finder==0.2.1 numpy==1.21.6 Pillow==8.0.1 visdom==0.1.8.9 gdown==5.1.0
-RUN pip3 install opencv_python==4.9.0.80 --no-build-isolation
+RUN pip install loguru==0.5.3 tqdm==4.62.3 imutils==0.5.3 numpy==1.23.5 Pillow==8.0.1 visdom==0.1.8.9 gdown==5.1.0
+RUN pip install opencv_python==4.9.0.80 --no-build-isolation
 
 ENV PYTHONPATH "${PYTHONPATH}:/tmp/goturn-pytorch/src"
 ENV PYTHONPATH "${PYTHONPATH}:/tmp/goturn-pytorch/src/scripts"

.github/actions/tf-faster-rcnn/Dockerfile

@@ -1,10 +1,5 @@
-# Get base from a tensorflow image
-FROM tensorflow/tensorflow:2.13.0
-
-ENV MONO_TLS_PROVIDER=legacy
-
-# Set to install things in non-interactive mode
-ENV DEBIAN_FRONTEND noninteractive
+# Get base image
+FROM python:3.8.20-slim-bookworm
 
 # Install system wide softwares
 RUN apt-get update \
@@ -13,24 +8,28 @@ RUN apt-get update \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         libavcodec-extra \
         ffmpeg \
         curl \
         libsndfile-dev \
         libsndfile1 \
         wget \
         unzip \
+        software-properties-common \
+        gnupg2 \
+        lsb-release \
      && apt-get clean all \
      && rm -r /var/lib/apt/lists/*
 
+RUN pip install --upgrade pip setuptools wheel
+
 # Install necessary libraries for tensorflow faster rcnn
 RUN mkdir TensorFlow
 RUN cd TensorFlow && git clone https://github.com/tensorflow/models
 RUN cd TensorFlow && wget https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protoc-3.17.3-linux-x86_64.zip
 RUN cd TensorFlow && unzip protoc-3.17.3-linux-x86_64.zip -d protobuf
 RUN cd TensorFlow/models/research && /TensorFlow/protobuf/bin/protoc object_detection/protos/*.proto --python_out=.
-RUN cd TensorFlow/models/research && cp object_detection/packages/tf2/setup.py .
+RUN cd TensorFlow/models/research && cp object_detection/packages/tf2/setup.py . && sed -i "/^'tensorflow_io',$/d; /^'keras',$/d" setup.py
 RUN cd TensorFlow/models/research && python -m pip install .
 
-RUN pip install tqdm requests numba==0.50 pytest-cov numpy==1.23.5
+RUN pip install numpy matplotlib tqdm requests llvmlite numba==0.56.4 pytest-cov "pytest<8.0.0" "exceptiongroup<1.1.0" tensorflow==2.6.0 keras==2.6.0

.github/actions/tf-faster-rcnn/action.yml

@@ -1,5 +1,5 @@
 # Get base image
-FROM python:3.9.19-slim
+FROM python:3.10.17-slim-bookworm
 
 # Install system wide software
 RUN apt-get update \
@@ -8,7 +8,6 @@ RUN apt-get update \
         libx11-xcb1 \
         git \
         gcc \
-        mono-mcs \
         cmake \
         libavcodec-extra \
         ffmpeg \
@@ -20,7 +19,8 @@ RUN apt-get update \
      && apt-get clean all \
      && rm -r /var/lib/apt/lists/*
 
-RUN pip install pytorchyolo==1.8.0 tensorflow==2.14.1 scikit-learn==1.4.2 pytest-cov~=4.1.0 torch==1.12.1
+RUN pip install --ignore-installed torch==1.12.1 --index-url https://download.pytorch.org/whl/cpu
+RUN pip install pytorchyolo==1.8.0 tensorflow==2.14.1 scikit-learn==1.4.2 pytest-cov~=4.1.0
 
 RUN cd /tmp/ && git clone https://github.com/eriklindernoren/PyTorch-YOLOv3.git && cd ./PyTorch-YOLOv3/weights && ./download_weights.sh
 

.github/actions/tf-faster-rcnn/run.sh

@@ -51,9 +51,9 @@ jobs:
           sudo apt-get update
           sudo apt-get -y -q install ffmpeg libavcodec-extra
           python -m pip install --upgrade pip setuptools wheel
-          pip3 install -r requirements_test.txt
-          pip install tensorflow==2.14.0
-          pip install keras==2.14.0
+          pip install -q -r <(sed '/^tensorflow/d;/^keras/d;/^torch/d;/^torchvision/d;/^torchaudio/d;/^transformers/d' requirements_test.txt)
+          pip install tensorflow==2.18.1
+          pip install keras==3.10.0
           pip install torch==${{ matrix.torch }} --index-url https://download.pytorch.org/whl/cpu
           pip install torchvision==${{ matrix.torchvision }} --index-url https://download.pytorch.org/whl/cpu
           pip install torchaudio==${{ matrix.torchaudio }} --index-url https://download.pytorch.org/whl/cpu