perf: Use a more aggressive retry loop for recursiveDelete (#84444)

**Update:** This also fixes a bug pointed out by Graphite where the deletion function could hang forever because the `t` variable used for tracking attempts wasn't properly incremented.

---

https://vercel.slack.com/archives/C04KC8A53T7/p1759360239193629

This is a critical path for `next dev` and `next build` initialization.

On Windows, it's quite common to get an error while trying to recursively delete directories for one of two reasons:

**Temporarily open files:**
1. You open the directory.
2. The user's AV detects the directory as opened and quickly scans the files in that directory.
3. You try to delete a file, but on Windows, files cannot be deleted when open.

**Deletion is not instant/atomic:** It can take some time after deleting files for the directory to become empty.

For this reason, we perform two retries with a 100ms delay after the first attempt and a 200ms delay after the second attempt.

* Node does not use retries by default, but has an option to do so with a `100ms * attemptNumber` (linear backoff) delay, similar to what we do: https://github.com/nodejs/node/blob/b5638e1765d3d1241d07457e767f27e3ee6b9dd2/lib/internal/fs/utils.js#L774-L779
* Node used to use 3 retries by default, but that was changed in v14: nodejs/node@7e85f06
* Rust tries to delete 64 times in a loop without sleeping, but with a call to `thread::yield_now`: https://github.com/rust-lang/rust/blob/4da69dfff1929cc79872b3d05ab7112d84753dba/library/std/src/sys/fs/windows/remove_dir_all.rs#L154-L171

In the vein of reducing time spent sleeping in the critical initialization path (@timneutkens), I think we should be more aggressive, I'm thinking using a 10ms delay (arbitrary number), but with an exponential backoff, so that we still retry over the same 300ms period as we currently do? Thoughts?

The difficulty is that this is on Windows, so it's hard for me to get good numbers about what impact this would have (if any).

Author: bgw

packages/next/src/lib/recursive-delete.ts

@@ -4,7 +4,30 @@ import { join, isAbsolute, dirname } from 'path'
 import isError from './is-error'
 import { wait } from './wait'
 
-const unlinkPath = async (p: string, isDir = false, t = 1): Promise<void> => {
+// We use an exponential backoff. See the unit test for example values.
+//
+// - Node's `fs` module uses a linear backoff, starting with 100ms.
+// - Rust tries 64 times with only a `thread::yield_now` in between.
+//
+// We want something more aggressive, as `recursiveDelete` is in the critical
+// path of `next dev` and `next build` startup.
+const INITIAL_RETRY_MS = 8
+const MAX_RETRY_MS = 64
+const MAX_RETRIES = 6
+
+/**
+ * Used in unit test.
+ * @ignore
+ */
+export function calcBackoffMs(attempt: number): number {
+  return Math.min(INITIAL_RETRY_MS * Math.pow(2, attempt), MAX_RETRY_MS)
+}
+
+const unlinkPath = async (
+  p: string,
+  isDir = false,
+  attempt = 0
+): Promise<void> => {
   try {
     if (isDir) {
       await promises.rmdir(p)
@@ -18,10 +41,12 @@ const unlinkPath = async (p: string, isDir = false, t = 1): Promise<void> => {
         code === 'ENOTEMPTY' ||
         code === 'EPERM' ||
         code === 'EMFILE') &&
-      t < 3
+      attempt < MAX_RETRIES
     ) {
-      await wait(t * 100)
-      return unlinkPath(p, isDir, t++)
+      // retrying is unlikely to succeed on POSIX platforms, but Windows can
+      // fail due to temporarily-open files or due to
+      await wait(calcBackoffMs(attempt))
+      return unlinkPath(p, isDir, attempt + 1)
     }
 
     if (code === 'ENOENT') {

test/unit/recursive-delete.test.ts

@@ -1,6 +1,6 @@
 /* eslint-env jest */
 import fs from 'fs-extra'
-import { recursiveDelete } from 'next/dist/lib/recursive-delete'
+import { recursiveDelete, calcBackoffMs } from 'next/dist/lib/recursive-delete'
 import { recursiveReadDir } from 'next/dist/lib/recursive-readdir'
 import { recursiveCopy } from 'next/dist/lib/recursive-copy'
 import { join } from 'path'
@@ -48,3 +48,12 @@ describe('recursiveDelete', () => {
     }
   })
 })
+
+describe('calcBackoffMs', () => {
+  it('returns expected values', () => {
+    let backoffValuesMs = Array.from({ length: 6 }, (_, attempt) =>
+      calcBackoffMs(attempt)
+    )
+    expect(backoffValuesMs).toEqual([8, 16, 32, 64, 64, 64])
+  })
+})