[version-4-4] chore: backport/version-4-4/pr-7502 (#7549)

* [version-4-4] chore: backport/version-4-4/pr-7502

* ci: adding missing robots.txt or updating noindex in docusaurus.config.js

---------

Co-authored-by: achuribooks <182707758+achuribooks@users.noreply.github.com>

Author: achuribooks

docusaurus.config.js

@@ -119,13 +119,20 @@ const config = {
     locales: ["en"],
   },
   future: {
+    v4: {
+      removeLegacyPostBuildHeadAttribute: true,
+      useCssCascadeLayers: true, // Slightly decreases padding of admonitions when set to `true`.
+    },
     experimental_faster: {
+      ssgWorkerThreads: false,
+      // Set to 'false' as Netlify builds fail with this enabled.
       swcJsLoader: false,
       // Set to 'false' as Netlify builds fail with this enabled.
       swcJsMinimizer: true,
       swcHtmlMinimizer: true,
       lightningCssMinimizer: true,
       rspackBundler: true,
+      rspackPersistentCache: true,
       mdxCrossCompilerCache: true,
     },
   },