Merge pull request #1318 from Trusted-AI/development_goturn

Add estimator for object tracker GOTURN in PyTorch

Author: beat-buesser

.github/actions/goturn/Dockerfile

@@ -0,0 +1,51 @@
+# Get base from a pytorch image
+FROM pytorch/pytorch:1.6.0-cuda10.1-cudnn7-runtime
+
+# Set to install things in non-interactive mode
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install system wide softwares
+RUN apt-get update \
+     && apt-get install -y \
+        libgl1-mesa-glx \
+        libx11-xcb1 \
+        git \
+        gcc \
+        mono-mcs \
+        libavcodec-extra \
+        ffmpeg \
+        curl \
+        libsndfile-dev \
+        libsndfile1 \
+     && apt-get install -y libsm6 libxext6 \
+     && apt-get install -y libxrender-dev \
+     && apt-get clean all \
+     && rm -r /var/lib/apt/lists/*
+
+RUN /opt/conda/bin/conda install --yes \
+    astropy \
+    matplotlib \
+    pandas \
+    scikit-learn \
+    scikit-image
+
+# Install necessary libraries for goturn
+RUN pip install torch==1.4
+RUN pip install tensorflow==2.1.4
+RUN pip install torchaudio==0.5.0
+RUN pip install pytest
+RUN pip install numba
+RUN pip install scikit-learn==0.20
+RUN pip install pytest-cov
+RUN pip install gdown
+
+RUN git clone https://github.com/nrupatunga/goturn-pytorch.git /tmp/goturn-pytorch
+RUN cd /tmp/goturn-pytorch && pip install -r requirements.txt
+
+RUN pip install numpy==1.20.3
+
+ENV PYTHONPATH "${PYTHONPATH}:/tmp/goturn-pytorch/src"
+ENV PYTHONPATH "${PYTHONPATH}:/tmp/goturn-pytorch/src/scripts"
+
+RUN mkdir /tmp/goturn-pytorch/src/goturn/models/checkpoints
+RUN cd /tmp/goturn-pytorch/src/goturn/models/checkpoints && gdown https://drive.google.com/uc?id=1GouImhqpcoDtV_eLrD2wra-qr3vkAMY4

.github/actions/goturn/action.yml

@@ -0,0 +1,7 @@
+name: 'Test GOTURN'
+description: 'Run tests for GOTURN'
+runs:
+  using: 'composite'
+  steps:
+    - run: $GITHUB_ACTION_PATH/run.sh
+      shell: bash

.github/actions/goturn/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+exit_code=0
+
+pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/object_tracking/test_pytorch_goturn.py --framework=pytorch --durations=0
+if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/object_tracking/test_pytorch_goturn tests"; fi
+
+exit ${exit_code}

.github/workflows/ci-goturn.yml

@@ -0,0 +1,32 @@
+name: CI PyTorchGoturn
+on:
+  # Run on manual trigger
+  workflow_dispatch:
+
+  # Run on pull requests
+  pull_request:
+    paths-ignore:
+      - '*.md'
+
+  # Run when pushing to main or dev branches
+  push:
+    branches:
+      - main
+      - dev*
+
+  # Run scheduled CI flow daily
+  schedule:
+    - cron: '0 8 * * 0'
+
+jobs:
+  test_pytorch_goturn:
+    name: PyTorchGoturn
+    runs-on: ubuntu-latest
+    container: adversarialrobustnesstoolbox/art_testing_envs:goturn
+    steps:
+      - name: Checkout Repo
+        uses: actions/[email protected]
+      - name: Run Test Action
+        uses: ./.github/actions/goturn
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]

art/attacks/evasion/fast_gradient.py

@@ -472,7 +472,11 @@ def _apply_perturbation(
         batch = batch + perturbation_step
         if self.estimator.clip_values is not None:
             clip_min, clip_max = self.estimator.clip_values
-            batch = np.clip(batch, clip_min, clip_max)
+            if batch.dtype == np.object:
+                for i_obj in range(batch.shape[0]):
+                    batch[i_obj] = np.clip(batch[i_obj], clip_min, clip_max)
+            else:
+                batch = np.clip(batch, clip_min, clip_max)
 
         return batch
 

art/estimators/object_tracking/__init__.py

@@ -0,0 +1,4 @@
+"""
+Module containing estimators for object tracking.
+"""
+from art.estimators.object_tracking.pytorch_goturn import PyTorchGoturn

art/estimators/object_tracking/object_tracker.py

@@ -0,0 +1,47 @@
+# MIT License
+#
+# Copyright (C) The Adversarial Robustness Toolbox (ART) Authors 2021
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""
+This module implements mixin abstract base class for all object trackers in ART.
+"""
+
+from abc import ABC, abstractmethod
+
+from art.estimators.estimator import BaseEstimator
+from art.estimators.classification.classifier import LossGradientsMixin
+
+
+class ObjectTrackerMixin(ABC):
+    """
+    Mix-in Base class for ART object trackers.
+    """
+
+    @property
+    @abstractmethod
+    def native_label_is_pytorch_format(self) -> bool:
+        """
+        Are the native labels in PyTorch format [x1, y1, x2, y2]?
+        """
+        raise NotImplementedError
+
+
+class ObjectTracker(ObjectTrackerMixin, LossGradientsMixin, BaseEstimator, ABC):
+    """
+    Typing variable definition.
+    """
+
+    pass