feat: Add configurable search engine indexing behavior [internal] (#1642)

If the `NO_INDEX` environment variable is used during build, the
[`noIndex` Docusaurus
config](https://docusaurus.io/docs/api/docusaurus-config#noIndex) will
be used and the docs will be built so that they're not indexable by
search engines.

Author: fnesveda

apify-docs-theme/src/config.js

@@ -10,6 +10,8 @@ if (process.env.LOCALHOST) {
     absoluteUrl = process.env.APIFY_DOCS_ABSOLUTE_URL;
 }
 
+const noIndex = ['true', '1'].includes(process.env.NO_INDEX ?? '');
+
 const themeConfig = {
     docs: {
         versionPersistence: 'localStorage',
@@ -313,4 +315,5 @@ module.exports = {
     themeConfig,
     plugins,
     absoluteUrl,
+    noIndex,
 };

docusaurus.config.js

@@ -12,6 +12,7 @@ module.exports = {
     title: 'Apify Documentation',
     tagline: 'Apify Documentation',
     url: config.absoluteUrl,
+    noIndex: config.noIndex,
     baseUrl: '/',
     trailingSlash: false,
     organizationName: 'apify',