bioc-cloner/.github/workflows/singleclone.yaml at main · almahmoud/bioc-cloner · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
name: x1 Clone Bioconductor packages to SWIFT buckets
on:
  workflow_dispatch: {}

jobs:
  swiftclone:
    strategy:
      fail-fast: false
      matrix:
        worker: [1]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2

      - name: Start k8s locally
        id: k8s
        uses: jupyterhub/action-k3s-helm@v1
        with:
          traefik-enabled: false
          metrics-enabled: false
        #   k3s-version: v1.19.10+k3s1 # releases:  https://github.com/k3s-io/k3s/tags
        continue-on-error: true

      - name: Start k8s locally
        uses: jupyterhub/action-k3s-helm@v1
        with:
          traefik-enabled: false
          metrics-enabled: false
        #   k3s-version: v1.19.10+k3s1 # releases:  https://github.com/k3s-io/k3s/tags
        if: steps.k8s.outcome=='failure'
      - name: Verify function of k8s, kubectl, and helm
        run: |
          kubectl version
          helm version
      - name: Claim list
        id: list
        run: |
          #sleep $(( ${{matrix.worker}}*5 )) &&
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          for i in {1..5}; do bash .github/scripts/claim_list.sh ${{matrix.worker}} && git push && export DONE=1 && break || sleep 1; done && if ! [ -z "$DONE" ]; then echo "::set-output name=path::$(cat tmp${{matrix.worker}})"; else exit 1; fi

      - name: Run list with helm install clone jobs
        run: |
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          #sleep $(( ${{matrix.worker}}*5 )) &&
          for i in {1..5}; do bash .github/scripts/mark_running.sh ${{steps.list.outputs.path}} && git push && export RUNNING=1 && break || sleep 1; done &&\
          if ! [ -z "$RUNNING" ]; then echo "::set-output name=path::$(cat tmp${{matrix.worker}})"; else exit 1; fi &&\
          echo "$RCLONE_CONF" > /tmp/creds &&\
          helm install data-clone ./cloner --set-file extraFileMappings."/root/\.rclone\.conf".content=/tmp/creds --set-file configs."list\.txt"=lists/running/${{steps.list.outputs.path}}
        env:
          RCLONE_CONF: ${{secrets.RCLONE_CONF}}

      - name: Get logs and events
        run: sh -c "while true; do kubectl get events > /tmp/events && tail -n 100 /tmp/events && sleep 5 && kubectl logs $(kubectl get pods | grep -o 'data-clone-[^ ]*')  > /tmp/logs && tail -n 100 /tmp/logs && sleep 5; done;" & kubectl wait --for=condition=complete --timeout=6000s job/data-clone
        continue-on-error: true
      - name: Done list
        run: |
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          until bash .github/scripts/mark_done.sh ${{steps.list.outputs.path}} && git push; do sleep 1 && git reset --hard origin/main; done