diff --git a/.github/workflows/rustc-pull.yml b/.github/workflows/rustc-pull.yml
index ad570ee45..ea4d7532a 100644
--- a/.github/workflows/rustc-pull.yml
+++ b/.github/workflows/rustc-pull.yml
@@ -9,12 +9,13 @@ on:
 jobs:
   pull:
     if: github.repository == 'rust-lang/rustc-dev-guide'
-    uses: rust-lang/josh-sync/.github/workflows/rustc-pull.yml@main
+    uses: rust-lang/josh-sync/.github/workflows/rustc-pull.yml@ci-gh-app
     with:
+      github-app-id: ${{ vars.APP_CLIENT_ID }}
       zulip-stream-id: 196385
       zulip-bot-email:  "rustc-dev-guide-gha-notif-bot@rust-lang.zulipchat.com"
       pr-base-branch: master
       branch-name: rustc-pull
     secrets:
       zulip-api-token: ${{ secrets.ZULIP_API_TOKEN }}
-      token: ${{ secrets.GITHUB_TOKEN }}
+      github-app-secret: ${{ secrets.APP_PRIVATE_KEY }}
diff --git a/triagebot.toml b/triagebot.toml
index b3f4c2d28..3ac5d57a5 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -62,9 +62,6 @@ allow-unauthenticated = [
 # Documentation at: https://forge.rust-lang.org/triagebot/issue-links.html
 [issue-links]
 
-# Automatically close and reopen PRs made by bots to run CI on them
-[bot-pull-requests]
-
 [behind-upstream]
 days-threshold = 7