fixing bug with listing and container_image (#170)

* fixing bug with listing and container_image

currently, the listing parser expects there to be a container_image key in the result,
but I just found a case where it is not there. I am making it option by using .get(
instead of a direct index

* require singularity 3.x or greater for singularity instance list with json
* remove black version pin
* I dont think we need instance.iutils anymore, I dont see it being used and its redundant with list
* test updated singularity orb version
* test if circleci config is valid

Signed-off-by: vsoch <[email protected]>
Co-authored-by: vsoch <[email protected]>

Author: vsoch

.circleci/config.yml

@@ -1,125 +1,118 @@
-# CircleCI build config to test different versions of Singuarity
 version: 2.1
 
-orbs:
-  singularity: singularity/[email protected]
+# Modified from hpcng/singularity
 
 workflows:
   version: 2
-  test:
+  build_and_test:
     jobs:
-      - run-spython-ci-tests: &base_job
+      - test_singularity_python:
           filters:
             branches:
               ignore: master
-          name: "Latest Singularity - Python 3"
-          run-linter: false
-          singularity: 3.5.0
-          singularity-3: true
-      - run-spython-ci-tests:
-          <<: *base_job
-          name: "Singularity 3.2.1 - Python 3"
-          singularity: 3.2.1
-          run-linter: true # run linter once, must be Python 3
-      - run-spython-ci-tests:
-          <<: *base_job
-          name: "Singularity 3.1.0 - Python 3"
-          singularity: 3.1.0
-      - run-spython-ci-tests:
-          <<: *base_job
-          name: "Singularity 2 - Python 3"
-          singularity: 2.6.1
-          singularity-3: false
 
-waitforapt: &waitforapt
-  name: Remove cloud init lock
-  command: |
-      while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 10; done
-      while sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1; do echo 'Waiting for autoupdates to complete...'; sleep 10; done
-      echo 'Waiting for instance to really be ready...'
-      sleep 30
-      sudo rm -rf /var/lib/apt/lists/lock
-      sudo rm /var/lib/dpkg/lock && sudo dpkg --configure -a
+setup_environment: &setup_environment
+  name: Setup environment
+  command: |-
+      echo 'set -x' >> $BASH_ENV
+      echo 'export GOPATH=$HOME/go'           >> $BASH_ENV
+      echo 'export GOROOT=/usr/local/go' >> $BASH_ENV
+      echo 'export GOBIN=$HOME/go/bin'        >> $BASH_ENV
+      echo 'export PATH=$GOBIN:$GOROOT/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin' >> $BASH_ENV
+      env | sed -e 's,^,ENV: ,' | sort
+      test -e $BASH_ENV && sed -e 's,^,BASH_ENV: ,' < $BASH_ENV
 
+update_go: &update_go
+  name: Update Go to 1.14.9
+  working_directory: /tmp
+  command: |-
+      wget https://dl.google.com/go/go1.14.9.linux-amd64.tar.gz
+      sudo rm -rf $GOROOT
+      sudo tar -C /usr/local -xzf go1.14.9.linux-amd64.tar.gz
+      sudo ln -s $GOROOT/bin/go /usr/local/bin/go
 
+fetch_deb_deps: &fetch_deb_deps
+  name: Fetch deps
+  working_directory: /tmp
+  command: |-
+      # https://discuss.circleci.com/t/could-not-get-lock-var-lib-apt-lists-lock/28337/4
+      sudo killall -9 apt-get || true && \
+      sudo apt-get update -y && \
+      sudo apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup
+
+build_singularity: &install_singularity
+  name: Build Singularity
+  working_directory: ~/go/singularity
+  command: |-
+      cd ~/go
+      wget https://github.com/hpcng/singularity/releases/download/v3.7.1/singularity-3.7.1.tar.gz && \
+      tar -xzvf singularity-3.7.1.tar.gz && \
+      cd singularity
+      ./mconfig -p /usr/local && \
+      make -C builddir && \
+      sudo make -C builddir install
+      
 install_spython: &install_spython
   name: install spython
-  command: |
+  command: |-
+      export PATH=~/conda/Python3/bin:$PATH
+      which python
       pip uninstall spython --yes || echo "Not installed"
       python --version
       python setup.py install
 
-commands:
-  install_dependencies:
-    steps:
-      - run: |
-          PYTHON_VERSION=3
-          CONDA_PATH="$HOME/conda/Python${PYTHON_VERSION}"
-          echo 'export PATH="'"$CONDA_PATH"'/bin:$PATH"' >> "$BASH_ENV"
-          source "$BASH_ENV"
-          if [ ! -d "$CONDA_PATH" ]; then
-              CONDA_SCRIPT=Miniconda${PYTHON_VERSION}-latest-Linux-x86_64.sh
-              wget https://repo.anaconda.com/miniconda/$CONDA_SCRIPT
-              /bin/bash $CONDA_SCRIPT -b -p $CONDA_PATH
-          else
-              echo "Miniconda is already installed, continuing to build."
-          fi
-          python --version
-          [ $(python -c'import sys;print(sys.version_info.major)') -eq $PYTHON_VERSION ]
+install_dependencies: &install_dependencies
+  name: install dependencies
+  command: |-
+      PYTHON_VERSION=3
+      CONDA_PATH="$HOME/conda/Python${PYTHON_VERSION}"
+      echo 'export PATH="'"$CONDA_PATH"'/bin:$PATH"' >> "$BASH_ENV"
+      source "$BASH_ENV"
+      if [ ! -d "$CONDA_PATH" ]; then
+          CONDA_SCRIPT=Miniconda${PYTHON_VERSION}-latest-Linux-x86_64.sh
+          wget https://repo.anaconda.com/miniconda/$CONDA_SCRIPT
+          /bin/bash $CONDA_SCRIPT -b -p $CONDA_PATH
+      else
+          echo "Miniconda is already installed, continuing to build."
+      fi
+      python --version
+      [ $(python -c'import sys;print(sys.version_info.major)') -eq $PYTHON_VERSION ]
 
-          pip install --upgrade pytest
-          pip install black || true
+      pip install --upgrade pytest
+      pip install black || true
 
 run_linter: &run_linter
   name: run linter
-  command: |
+  command: |-
+      export PATH=~/conda/Python3/bin:$PATH
       cd ~/repo
       black --check spython
 
 test_spython: &test_spython
   name: Test Singularity Python
-  command: pytest ~/repo/spython
+  command: |- 
+      export PATH=~/conda/Python3/bin:$PATH
+      pytest ~/repo/spython
+
 
 jobs:
-  run-spython-ci-tests:
-    parameters:
-        run-linter:
-          type: boolean
-          default: false
-        singularity:
-          type: string
-          description: "Singularity version"
-        singularity-3:
-          type: boolean
-          description: "Set to true for singularity 3, false for singularity 2"
-          default: true
-    machine: true
+  test_singularity_python:
     working_directory: ~/repo
+    machine:
+      image: ubuntu-2004:202008-01
     steps:
       - checkout
       - restore_cache:
           keys: v2-dependencies
-      - install_dependencies
-      - run: *waitforapt
-      - when:
-          condition: << parameters.singularity-3 >>
-          steps:
-            - singularity/install-go:
-                go-version: "1.13"
-            - singularity/debian-install-3:
-                singularity-version: << parameters.singularity >>
-      - unless:
-          condition: << parameters.singularity-3 >>
-          steps:
-            - singularity/debian-install-2:
-                singularity-version: << parameters.singularity >>
+      - run: *install_dependencies
+      - run: *setup_environment
+      - run: *update_go
+      - run: *fetch_deb_deps
+      - run: *install_singularity
       - run: *install_spython
       - save_cache:
           paths:
             - ~/conda
-          key: v2-dependencies
-      - when:
-          condition: << parameters.run-linter >>
-          steps:
-            - run: *run_linter
+          key: v3-dependencies
+      - run: *run_linter
       - run: *test_spython