Merge branch 'master' into daxamin-offline-error

daxamin · web-flow · commit 5dd47bd49f70 · 2023-11-03T12:46:32.000-04:00
diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml
@@ -47,7 +47,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         repository: github/backup-utils-private
         token: "${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}"
@@ -62,7 +62,7 @@ jobs:
     needs: build
     runs-on:
       group: larger-hosted-public-runners
-      labels: ubuntu-latest-xl
+      labels: ubuntu-latest
     env:
         SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
     steps:
@@ -98,7 +98,7 @@ jobs:
         sudo tar -czvf "${{ inputs.backup-name }}.tar.gz" -C "$HOME/ghe-backup-data/$current" .
 
     - name: Login to Azure
-      if: ${{ inputs.backup-name }} != ""
+      if: "${{ inputs.backup-name != '' }}"
       run: |
         az login \
           --service-principal \
@@ -108,11 +108,11 @@ jobs:
         az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
 
     - name: Upload backup to Azure
-      if: ${{ inputs.backup-name }} != ""
+      if: "${{ inputs.backup-name != '' }}"
       run: |
         az storage blob upload \
         --account-name "${{ secrets.AZURE_ACCOUNT_NAME }}" \
         --container-name "${{ secrets.AZURE_CONTAINER_NAME }}" \
         --name "${{ inputs.backup-name }}.tar.gz" \
         --file "${{ inputs.backup-name }}.tar.gz" \
-        --connection-string "${{ secrets.CONNECTIONSTRING }}"
+        --connection-string "${{ secrets.CONNECTIONSTRING }}"
diff --git a/.github/workflows/backuprestore.yml b/.github/workflows/backuprestore.yml
@@ -28,7 +28,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         repository: github/backup-utils-private
         ref: ${{ inputs.ref }}
@@ -51,13 +51,13 @@ jobs:
       run: docker load -i backup-utils.tar
     - uses: actions/checkout@v3
     - name: Create backup directory
-      run: mkdir $HOME/ghe-backup-data
+      run: mkdir "$HOME"/ghe-backup-data
     - name: set up ssh SSH_KEY
-      run: echo -e "${SSH_KEY}\n" > $HOME/backup
+      run: echo -e "${SSH_KEY}\n" > "$HOME"/backup
     - name: set up ssh key permissions
-      run: chmod 0600 $HOME/backup
+      run: chmod 0600 "$HOME"/backup
     - name: change version
-      run: echo "3.8.0" > $HOME/version
+      run: echo "3.8.0" > "$HOME"/version
     - name: Perform backup
       run: |
         docker run -e "GHE_HOSTNAME=${{ inputs.hostname }}" \
@@ -70,7 +70,7 @@ jobs:
         --rm \
         backup-utils ghe-backup
     - name: Prepare for restore
-      run: ssh -p122 -i $HOME/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -s"
+      run: ssh -p122 -i "$HOME"/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -s"
     - name: Restore data to instance
       run: |
         docker run -e "GHE_HOSTNAME=${{ inputs.hostname }}" \
@@ -83,5 +83,5 @@ jobs:
         --rm \
         backup-utils ghe-restore ${{ inputs.hostname }}
     - name: Reset maintenance mode after restore
-      run: ssh -p122 -i $HOME/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -u"
+      run: ssh -p122 -i "$HOME"/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -u"
   
diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml
@@ -4,10 +4,6 @@ name: Build and Release
 on:
   workflow_dispatch:
     inputs:
-      gh-token:
-        description: 'GitHub Token - used to create a commit in the backup-utils repo'
-        required: true
-        type: string
       version:
         description: 'Version - patch version of the release (e.g. x.y.z)'
         required: true
@@ -21,22 +17,20 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    outputs:
+      rc-app-token: ${{ steps.app-token.outputs.token }}
     steps:
-      # resulting token still gets denied by the backup-utils repo
-      # see: https://github.com/actions/create-github-app-token/pull/46
-      # - uses: timreimherr/create-github-app-token@main
-      #   id: app-token
-      #   with:
-      #     # required
-      #     app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }}
-      #     private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }}
-      #     owner: ${{ github.repository_owner }}
-      #     repositories: backup-utils,backup-utils-private
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }}
+          private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: "backup-utils-private"
       - name: Checkout backup-utils-private
         uses: actions/checkout@v4
         with:
-          token: ${{ github.event.inputs.gh-token }}
-          repository: github/backup-utils-private
+          token: ${{ steps.app-token.outputs.token }}
       - name: Install dependencies
         run: |
           sudo apt-get update -y
@@ -50,14 +44,6 @@ jobs:
       - name: Package deb
         run: |
           ./script/package-deb
-      # many need to remove this once release-notes compilation is automated
-      - name: Rename deb artifact
-        run: |
-            for file in dist/github-backup-utils_*_all.deb; do
-              if [[ -f "$file" ]]; then
-                mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb"
-              fi
-            done
       - name: Upload deb artifact
         uses: actions/upload-artifact@v3
         with:
@@ -79,35 +65,40 @@ jobs:
     outputs:
       commit_hash: ${{ steps.empty-commit.outputs.commit_hash }}
     steps:
-      # resulting token still gets denied by the backup-utils repo
-      # see: https://github.com/actions/create-github-app-token/pull/46
-      # - uses: timreimherr/create-github-app-token@main
-      #   id: app-token
-      #   with:
-      #     app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }}
-      #     private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }}
-      #     owner: ${{ github.repository_owner }}
-      #     repositories: backup-utils,backup-utils-private
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }}
+          private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: "backup-utils,backup-utils-private"
+      - name: Get major-feature from version
+        id: get-major-feature
+        run: |
+          echo "MAJOR_FEATURE=$(echo ${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV"
+      - name: Verify major-feature
+        run: |
+          echo "major_feature: $MAJOR_FEATURE"
       - name: Checkout backup-utils
         uses: actions/checkout@v4
         with:
-          token: ${{ github.event.inputs.gh-token }}
+          token: ${{ steps.app-token.outputs.token }}
           repository: github/backup-utils
-          ref: master
       - name: Create empty commit
         uses: stefanzweifel/git-auto-commit-action@v4
         id: empty-commit
         with:
-          branch: master
+          branch: ${{ env.MAJOR_FEATURE }}-stable
           commit_message: "${{ github.event.inputs.version }} release"
-          commit_user_name: "${{ github.actor }}"
-          commit_user_email: "ghes-releases-team@github.com"
+          commit_user_name: "release-controller[bot]"
+          commit_user_email: "223695+release-controller[bot]@users.noreply.github.com"
           commit_options: "--allow-empty"
+          push_options: "--force"
           skip_dirty_check: true
-      - name: Checkout backup-utils
+      - name: Checkout backup-utils-private for release notes
         uses: actions/checkout@v4
         with:
-          token: ${{ github.event.inputs.gh-token }}
+          token: ${{ steps.app-token.outputs.token }}
           repository: github/backup-utils-private
       - name: Download deb artifact
         uses: actions/download-artifact@v3
@@ -120,20 +111,17 @@ jobs:
       - name: Create Release
         uses: ncipollo/release-action@v1
         with:
-          token: ${{ github.event.inputs.gh-token }}
+          token: ${{ steps.app-token.outputs.token }}
+          owner: github
           repo: backup-utils
           name: |
             GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }}
           artifacts: |
-            github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \
+            github-backup-utils-v${{ github.event.inputs.version }}.tar.gz,
             github-backup-utils_${{ github.event.inputs.version }}_all.deb
           tag: v${{ github.event.inputs.version }}
-          commit: ${{ steps.empty-commit.outputs.commit_hash }}
+          commit: ${{ env.MAJOR_FEATURE }}-stable
           bodyFile: release-notes/${{ github.event.inputs.version }}.md
           draft: ${{ github.event.inputs.draft }}
           allowUpdates: true
-          artifactContentType: "raw"
-
-
-
-
+          artifactContentType: "raw"
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -1,4 +1,4 @@
-name: Docker Image CI
+name: Docker Image Build CI
 
 on:
   push:
@@ -13,16 +13,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build the Ubuntu Docker image
-      run: docker build . --file Dockerfile --tag backup-utils:${GITHUB_RUN_ID}
+      run: docker build . --file Dockerfile --tag backup-utils:"${GITHUB_RUN_ID}"
     - name: Build the Alpine Docker image
-      run: docker build . --file Dockerfile.alpine --tag backup-utils-alpine:${GITHUB_RUN_ID}
+      run: docker build . --file Dockerfile.alpine --tag backup-utils-alpine:"${GITHUB_RUN_ID}"
     - name: Run tests in Ubuntu Docker image
       run: |
-        docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version
-        docker run backup-utils:${GITHUB_RUN_ID} rsync --version
+        docker run backup-utils:"${GITHUB_RUN_ID}" ghe-backup --version
+        docker run backup-utils:"${GITHUB_RUN_ID}" rsync --version
     - name: Run tests in Alpine Docker image
       run: |
-        docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version
-        docker run backup-utils-alpine:${GITHUB_RUN_ID} rsync --version
+        docker run backup-utils-alpine:"${GITHUB_RUN_ID}" ghe-backup --version
+        docker run backup-utils-alpine:"${GITHUB_RUN_ID}" rsync --version
diff --git a/.github/workflows/restore.yml b/.github/workflows/restore.yml
@@ -51,7 +51,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         repository: github/backup-utils-private
         ref: ${{ inputs.ref }}
@@ -69,7 +69,7 @@ jobs:
     needs: build
     runs-on:
       group: larger-hosted-public-runners
-      labels: ubuntu-latest-xl
+      labels: ubuntu-latest
     env:
         SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
     steps:
diff --git a/bin/ghe-backup b/bin/ghe-backup
@@ -231,9 +231,11 @@ fi
 
 # Output system information of the backup host
 
-# If /etc/issue.net exists, use it to get the OS version
-if [ -f /etc/issue.net ]; then
-  echo "Running on: $(cat /etc/issue.net)"
+# If /etc/os-release exists, use it to get the OS version
+if [ -f /etc/os-release ]; then
+  OS_NAME=$(grep '^NAME' /etc/os-release | cut -d'"' -f2)
+  VERSION_ID=$(grep '^VERSION_ID' /etc/os-release | cut -d'"' -f2)
+  echo "Running on: $OS_NAME $VERSION_ID"
 else
   echo "Running on: Unknown OS"
 fi
diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config
@@ -661,17 +661,14 @@ prompt_for_confirmation(){
 
 #initialize progress tracking by clearing out the temp files used to track
 init-progress() {
-  if [ -d /tmp/backup-utils-progress ]; then
+
+  if [ -e /tmp/backup-utils-progress ]; then
     rm -rf /tmp/backup-utils-progress/*
   fi
 
-  mkdir -p /tmp/backup-utils-progress
-  chmod -R 777 /tmp/backup-utils-progress
-
-  touch /tmp/backup-utils-progress/total
-  touch /tmp/backup-utils-progress/type
-  touch /tmp/backup-utils-progress/progress
-  touch /tmp/backup-utils-progress/info
+  # shellcheck disable=SC2174 # We are fine with -m only affecting the deepest directory
+  mkdir -m 777 -p /tmp/backup-utils-progress
+  touch /tmp/backup-utils-progress/{total,type,progress,info}
 }
 
 
diff --git a/share/github-backup-utils/ghe-backup-secrets b/share/github-backup-utils/ghe-backup-secrets
@@ -54,7 +54,7 @@ backup-secret() {
   log_info "* Transferring $description ..." 1>&3
   ghe-ssh "$host" -- ghe-config "$setting" > "$file+" || (
     if  [ "$best_effort" = "false" ]; then
-      echo "Warning: $description not set" >&2
+      echo "Info: $description not set. Skipping..." >&2
     fi
   )
   if [ -n "$(cat "$file+")" ]; then
diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh
@@ -1068,9 +1068,26 @@ begin_test "ghe-backup collects information on system where backup-utils is inst
   set -e
 
   output=$(ghe-backup)
-  echo "$output" | grep "Running on: $(cat /etc/issue.net)"
+  OS_NAME=$(grep '^NAME' /etc/os-release | cut -d'"' -f2)
+  VERSION_ID=$(grep '^VERSION_ID' /etc/os-release | cut -d'"' -f2)
+  echo "$output" | grep "Running on: $OS_NAME $VERSION_ID"
   echo "$output" | grep "CPUs: $(nproc)"
   echo "$output" | grep "Memory total/used/free+share/buff/cache:"
 
 )
 end_test
+
+# Check that backup-utils manages track-progress folder correctly
+begin_test "ghe-backup manages progress tracking files properly"
+(
+  set -e
+
+  if [ -e /tmp/backup-utils-progress ]; then
+    rm -rf /tmp/backup-utils-progress/*
+  fi
+
+  output=$(ghe-backup)
+  echo "$output" | grep -v "mkdir: cannot create directory /tmp/backup-utils-progress: File exists"  
+
+)
+end_test

Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ backup-secret() {`
`54`	`54`	`log_info "* Transferring $description ..." 1>&3`
`55`	`55`	`ghe-ssh "$host" -- ghe-config "$setting" > "$file+" \|\| (`
`56`	`56`	`if [ "$best_effort" = "false" ]; then`
`57`		`- echo "Warning: $description not set" >&2`
	`57`	`+ echo "Info: $description not set. Skipping..." >&2`
`58`	`58`	`fi`
`59`	`59`	`)`
`60`	`60`	`if [ -n "$(cat "$file+")" ]; then`