domains/.github/workflows/sync.yaml at 40e10c28a1487605ba2176254afcf835eb6e5afa · pkgforge-security/domains · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
name: ♻️↕️ Sync ↕️♻️
concurrency:
  group: "${{ github.workflow }}-${{ github.ref }}"
  cancel-in-progress: true

on:
  workflow_dispatch:
  schedule:
   - cron: "0 1 * * *" #UTC 01:00 AM --> 06:45 AM NPT Everyday
   - cron: "0 13 * * *" #UTC 01:00 PM --> 06:45 PM NPT Everyday
jobs:
  sync:
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 320
    permissions:
      attestations: write
      contents: write
      id-token: write
      issues: write
      packages: write
      statuses: read
    strategy:
      max-parallel: 3
      fail-fast: false
      matrix:
        include:
          - runner: "ubuntu-latest"
            source: "certstream"
            dest: "ghcr"
            script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/sync_certstream_ghcr.sh"
            merge-script: "N/A"

          - runner: "ubuntu-latest"
            source: "certstream"
            dest: "hf"
            script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/sync_certstream_hf.sh"
            merge-script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/merge_certstream_ghcr_hf.sh"

          - runner: "ubuntu-latest"
            source: "trickest"
            dest: "ghcr"
            script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/sync_trickest_ghcr.sh"
            merge-script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/merge_trickest_ghcr.sh"

          - runner: "ubuntu-latest"
            source: "trickest"
            dest: "hf"
            script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/sync_trickest_hf.sh"
            merge-script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/merge_trickest_hf.sh"

          - runner: "ubuntu-latest"
            source: "sni-ip-ranges"
            dest: "ghcr"
            script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/sync_sni-ip-ranges_ghcr.sh"
            merge-script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/merge_sni-ip-ranges_ghcr.sh"

          - runner: "ubuntu-latest"
            source: "sni-ip-ranges"
            dest: "hf"
            script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/sync_sni-ip-ranges_hf.sh"
            merge-script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/merge_sni-ip-ranges_hf.sh"

    steps:
      - name: Install Addons
        run: |
          #presets
          set +x ; set +e
          #-------------#
          export DEBIAN_FRONTEND="noninteractive"
          sudo apt update -y -qq
          sudo apt install 7zip b3sum bc coreutils curl dos2unix fdupes jq git git-lfs moreutils wget util-linux -y -qq
          sudo apt install 7zip b3sum bc coreutils curl dos2unix fdupes jq git git-lfs moreutils wget util-linux -y -qq
          bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/devscripts/refs/heads/main/Linux/install_bins_curl.sh")
        continue-on-error: true

      - name: Debloat Runner
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/devscripts/main/Github/Runners/debloat_ubuntu.sh")
        continue-on-error: true

      - name: Setup Env
        run: |
          #presets
          set +x ; set +e
          #-------------#
          #tmp
          SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}"
          echo "SYSTMP=${SYSTMP}" >> "${GITHUB_ENV}"
          #-------------#
          #Git
          sudo apt-get install git-lfs -y -qq
          echo "GH_PAGER=" >> "${GITHUB_ENV}"
          gh config set prompt disabled
          #-------------#
          ##User-Agent
          USER_AGENT="$(curl -qfsSL 'https://raw.githubusercontent.com/pkgforge/devscripts/refs/heads/main/Misc/User-Agents/ua_firefox_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}"
          echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}"
        continue-on-error: true

      - name: Sync Mirror [${{ matrix.source }} ==> ${{ matrix.dest }}]
        env:
          GHCR_TOKEN: "${{ github.token }}"
          GITHUB_TOKEN: "${{ github.token }}"
          #GITHUB_TOKEN: "${{ secrets.RO_GHTOKEN }}"
          HF_TOKEN: "${{ secrets.HF_TOKEN }}"
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          curl -qfsSL "${{ matrix.script }}" -o "./sync.sh"
          dos2unix --quiet "./sync.sh"
          chmod +x "./sync.sh"
          #export FORCE_PUSH="YES"
          PARALLEL_LIMIT="$(($(nproc)+1))" bash "./sync.sh"
          #PARALLEL_LIMIT="1" bash "./sync.sh"
          wait ; echo
          find "${SYSTMP}/DATA" -type f -size +1990M -exec rm -rvf "{}" \;
          if [[ -d "${{ env.ARTIFACTS_PATH }}" && "${{ env.ARTIFACTS_PATH }}" == /tmp/* ]]; then
            ARTIFACTS_DIR="$(realpath ${{ env.ARTIFACTS_PATH }})"
            echo -e "\n[+] Artifacts ==> ${ARTIFACTS_DIR}\n"
          else
            ARTIFACTS_DIR="$(mktemp -d)"
            touch "${ARTIFACTS_DIR}/.keep"
          fi
          realpath "${ARTIFACTS_DIR}" && ls -lah "${ARTIFACTS_DIR}"
          du -sh "${ARTIFACTS_DIR}"
          echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
        continue-on-error: true

      - name: Upload (LOG) Artifacts [${{ matrix.source }}<==>${{ matrix.dest }}]
        uses: actions/upload-artifact@v4
        with:
            name: ${{ matrix.source }}_${{ matrix.dest }}
            path: |
              /tmp/DATA/**
              ${{ env.ARTIFACTS_DIR }}/**
            compression-level: 0 # no compression, [Default: 6 (GNU Gzip)]
            retention-days: 30
            overwrite: true
        continue-on-error: true

      - name: Attest Build Provenance
        uses: actions/attest-build-provenance@v2.2.3
        with:
          subject-name: "${{ matrix.source }}-${{ matrix.dest }}"
          subject-path: |
            /tmp/DATA/**
            ${{ env.ARTIFACTS_DIR }}/**
          show-summary: true
        continue-on-error: true

      - name: Merge [${{ matrix.source }} ==> ${{ matrix.dest }}]
        if: env.MERGE_DATA == 'YES'
        env:
          GHCR_TOKEN: "${{ github.token }}"
          GITHUB_TOKEN: "${{ github.token }}"
          #GITHUB_TOKEN: "${{ secrets.RO_GHTOKEN }}"
          HF_TOKEN: "${{ secrets.HF_TOKEN }}"
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          if echo "${{ matrix.merge-script }}" | grep -qi "http"; then
             find "${SYSTMP}" -type f -size +10M -exec rm -rf "{}" \; 2>/dev/null
             find "${SYSTMP}" -type d -exec bash -c 'test $(du -sb "{}" | cut -f1 | tr -d "[:space:]") -gt 10485760 && rm -rf "{}"' \; 2>/dev/null
             curl -qfsSL "${{ matrix.merge-script }}" -o "./sync.sh"
             dos2unix --quiet "./sync.sh"
             chmod +x "./sync.sh"
             PARALLEL_LIMIT="$(($(nproc)+1))" bash "./sync.sh"
             wait ; echo
          fi
        continue-on-error: true