updated CI/CD

Author: Sandip117

.github/workflows/ci.yml

@@ -4,7 +4,7 @@
 # - on push and PR: run pytest
 # - on push to main: build and push container images as ":latest"
 # - on push to semver tag: build and push container image with tag and
-#                          upload plugin description to https://chrisstore.co
+#                          upload plugin description to https://cube.chrisproject.org/api/v1/
 
 name: build
 
@@ -19,28 +19,21 @@ on:
 jobs:
   test:
     name: Unit tests
-    if: false  # delete this line to enable automatic testing
+    if: False
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
       - name: Build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           build-args: extras_require=dev
           context: .
           load: true
           push: false
           tags: "localhost/local/app:dev"
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
       - name: Run pytest
         run: |
           docker run -v "$GITHUB_WORKSPACE:/app:ro" -w /app localhost/local/app:dev \
@@ -52,105 +45,138 @@ jobs:
     # needs: [ test ]  # uncomment to require passing tests
     runs-on: ubuntu-22.04
 
-    # A local registry helps us reuse the built image between steps
-    services:
-      registry:
-        image: registry:2
-        ports:
-          - 5000:5000
-
     steps:
-      - name: Get git tag
-        id: git_info
-        if: startsWith(github.ref, 'refs/tags/')
-        run: echo "::set-output name=tag::${GITHUB_REF##*/}"
-      - name: Get project info
-        id: determine
-        env:
-          git_tag: ${{ steps.git_info.outputs.tag }}
+      - name: Decide image tags
+        id: info
+        shell: python
         run: |
-          repo="${GITHUB_REPOSITORY,,}"  # to lower case
-          # if build triggered by tag, use tag name
-          tag="${git_tag:-latest}"
+          import os
+          import itertools
 
-          # if tag is a version number prefixed by 'v', remove the 'v'
-          if [[ "$tag" =~ ^v[0-9].* ]]; then
-            tag="${tag:1}"
-          fi
+          def join_tag(t):
+            registry, repo, tag = t
+            return f'{registry}/{repo}:{tag}'.lower()
+
+          registries = ['docker.io', 'ghcr.io']
+          repos = ['${{ github.repository }}']
+          if '${{ github.ref_type }}' == 'branch':
+              tags = ['latest']
+          elif '${{ github.ref_type }}' == 'tag':
+              tag = '${{ github.ref_name }}'
+              version = tag[1:] if tag.startswith('v') else tag
+              tags = ['latest', version]
+          else:
+              tags = []
+
+          if '${{ github.ref_type }}' == 'tag':
+              local_tag = join_tag(('ghcr.io', '${{ github.repository }}', version))
+          else:
+              local_tag = join_tag(('localhost', '${{ github.repository }}', 'latest'))
 
-          dock_image=$repo:$tag
-          echo $dock_image
-          echo "::set-output name=dock_image::$dock_image"
-          echo "::set-output name=repo::$repo"
+          product = itertools.product(registries, repos, tags)
+          tags_csv = ','.join(map(join_tag, product))
+          outputs = {
+              'tags_csv' : tags_csv,
+              'push'     : 'true' if tags_csv else 'false',
+              'local_tag': local_tag
+          }
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as out:
+              for k, v in outputs.items():
+                  out.write(f'{k}={v}\n')
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         # QEMU is used for non-x86_64 builds
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         # buildx adds additional features to docker build
-      - uses: docker/setup-buildx-action@v2
+      - uses: docker/setup-buildx-action@v3
         with:
           driver-opts: network=host
-        # cache slightly improves rebuild time
-      - name: Cache Docker layers
-        uses: actions/cache@v2
+
+        # Here, we want to do the docker build twice:
+        # The first build pushes to our local registry for testing.
+        # The second build pushes to Docker Hub and ghcr.io
+      - name: Build (local only)
+        uses: docker/build-push-action@v3
+        id: docker_build
         with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
+          context: .
+          file: ./Dockerfile
+          tags: ${{ steps.info.outputs.local_tag }}
+          load: true
+          cache-from: type=gha
+      # If you have a directory called examples/incoming/ and examples/outgoing/, then
+      # run your ChRIS plugin with no parameters, and assert that it creates all the files
+      # which are expected. File contents are not compared.
+      - name: Run examples
+        id: run_examples
+        run: |
+          if ! [ -d 'examples/incoming/' ] || ! [ -d 'examples/outgoing/' ]; then
+            echo "No examples."
+            exit 0
+          fi
+        
+          dock_image=${{ steps.info.outputs.local_tag }}
+          output_dir=$(mktemp -d)
+          cmd=$(docker image inspect -f '{{ (index .Config.Cmd 0) }}' $dock_image)
+          docker run --rm -u "$(id -u):$(id -g)" \
+            -v "$PWD/examples/incoming:/incoming:ro" \
+            -v "$output_dir:/outgoing:rw" \
+            $dock_image $cmd /incoming /outgoing
+          
+          for expected_file in $(find examples/outgoing -type f); do
+            fname="${expected_file##*/}"
+            out_path="$output_dir/$fname"
+            printf "Checking output %s exists..." "$out_path"
+            if [ -f "$out_path" ]; then
+              echo "ok"
+            else
+              echo "not found"
+              exit 1
+            fi
+          done
 
       - name: Login to DockerHub
-        id: dockerhub_login
-        uses: docker/login-action@v2
+        if: (github.event_name == 'push' || github.event_name == 'release') && contains(steps.info.outputs.tags_csv, 'docker.io')
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
-          
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        if: (github.event_name == 'push' || github.event_name == 'release') && contains(steps.info.outputs.tags_csv, 'ghcr.io')
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Build and push
-        uses: docker/build-push-action@v3
-        id: docker_build
+        uses: docker/build-push-action@v5
+        if: (github.event_name == 'push' || github.event_name == 'release')
         with:
           context: .
           file: ./Dockerfile
-          tags: |
-            localhost:5000/${{ steps.determine.outputs.dock_image }}
-            docker.io/${{ steps.determine.outputs.dock_image }}
-            ghcr.io/${{ steps.determine.outputs.dock_image }}
+          tags: ${{ steps.info.outputs.tags_csv }}
           # if non-x86_84 architectures are supported, add them here
           platforms: linux/amd64  #,linux/arm64,linux/ppc64le
-          push: true
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          push: ${{ steps.info.outputs.push }}
+          cache-to: type=gha,mode=max
 
-      - name: Get plugin meta
-        id: pluginmeta
-        run: |
-          repo=${{ steps.determine.outputs.repo }}
-          dock_image=${{ steps.determine.outputs.dock_image }}
-          docker run --rm localhost:5000/$dock_image chris_plugin_info > /tmp/description.json
-          jq < /tmp/description.json  # pretty print in log
-          echo "::set-output name=title::$(jq -r '.title' < /tmp/description.json)"
+      - name: Upload ChRIS Plugin
+        id: upload
+        if: github.ref_type == 'tag'
+        uses: FNNDSC/upload-chris-plugin@v1
+        with:
+          dock_image: ${{ steps.info.outputs.local_tag }}
+          username: ${{ secrets.CHRISPROJECT_USERNAME }}
+          password: ${{ secrets.CHRISPROJECT_PASSWORD }}
+          chris_url: https://cube.chrisproject.org/api/v1/
+          compute_names: NERC
 
       - name: Update DockerHub description
-        uses: peter-evans/dockerhub-description@v2
+        if: steps.upload.outcome == 'success'
+        uses: peter-evans/dockerhub-description@v3
         continue-on-error: true  # it is not crucial that this works
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
-          short-description: ${{ steps.pluginmeta.outputs.title }}
-          readme-filepath: ./README.md
-          repository: ${{ steps.determine.outputs.repo }}
-
-      - name: Upload to ChRIS Store
-        if: steps.git_info.outcome != 'skipped'
-        uses: FNNDSC/chrisstore-action@master
-        with:
-          descriptor_file: /tmp/description.json
-          auth: ${{ secrets.CHRIS_STORE_USER }}
+          short-description: ${{ steps.upload.outputs.title }}
+          readme-filepath: ./README.md

requirements.txt

@@ -1,4 +1,4 @@
-chris_plugin==0.1.2
+chris_plugin>=0.3.0
 Faker==6.4.1
 pfdo==5.0.0
 pfdo-run==3.2.6

shexec.py

@@ -7,7 +7,7 @@
 from chris_plugin import chris_plugin, PathMapper
 
 __pkg       = Distribution.from_name(__package__)
-__version__ = '1.2.16'
+__version__ = '1.2.17'
 
 import  os, sys
 os.environ['XDG_CONFIG_HOME'] = '/tmp'