Added robot.txt to disallow indexing non production version of the website

Author: ZibanPirate

.github/workflows/cd.deploy.stg.yml

@@ -63,6 +63,7 @@ jobs:
 
         env:
           CI: true
+          STAGE: "staging"
           SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
           SSH_ADDRESS_STG: ${{ secrets.SSH_ADDRESS_STG }}
           FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}

.github/workflows/cd.deploy.yml

@@ -70,6 +70,7 @@ jobs:
 
         env:
           CI: true
+          STAGE: "production"
           SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
           SSH_ADDRESS_PRD: ${{ secrets.SSH_ADDRESS_PRD }}
           FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}

web/package.json

@@ -82,6 +82,7 @@
     "postcss-preset-env": "^7.0.1",
     "pug": "^3.0.0",
     "pug-loader": "^2.4.0",
+    "robotstxt-webpack-plugin": "^7.0.0",
     "sass": "^1.30.0",
     "sass-loader": "^10.1.0",
     "sitemap": "^6.3.5",

web/webpack.config.ts

@@ -1,4 +1,5 @@
 import { fsConfig } from "@dzcode.io/utils/dist/config";
+import { Environment } from "@dzcode.io/utils/dist/config/environment";
 import CssMinimizerPlugin from "css-minimizer-webpack-plugin";
 import { readFileSync } from "fs";
 import glob from "glob";
@@ -10,9 +11,14 @@ import { Configuration as WPC } from "webpack";
 import { BundleAnalyzerPlugin } from "webpack-bundle-analyzer";
 import { Configuration as WPDSC } from "webpack-dev-server";
 
+const RobotstxtPlugin = require("robotstxt-webpack-plugin"); // eslint-disable-line @typescript-eslint/no-var-requires
+
+// @TODO-ZM: to combine isProduction and isDevelopment and stage
+
 // setting up project configurations and some env variables
 const ANALYZE = process.env.ANALYZE === "true";
 const isProduction = process.env.NODE_ENV === "production" || ANALYZE;
+const stage = (process.env.STAGE as Environment) || "production";
 const isDevelopment = process.env.NODE_ENV === "development" && !ANALYZE;
 const { web } = fsConfig("development");
 const distFolder = "./bundle";
@@ -157,6 +163,15 @@ export default {
           new BundleAnalyzerPlugin(),
         ]
       : []),
+    new RobotstxtPlugin({
+      policy: [
+        {
+          userAgent: "*",
+          allow: stage === "production" ? "/" : undefined,
+          disallow: stage !== "production" ? "/" : undefined,
+        },
+      ],
+    }),
     ...apps.reduce<WPC["plugins"][]>(
       (pV, app) => [...pV, ...require(`./src/apps/${app}/entry/webpack.plugins`)],
       [],

yarn.lock

@@ -10624,6 +10624,18 @@ gcp-metadata@^4.2.0:
     gaxios "^4.0.0"
     json-bigint "^1.0.0"
 
+generate-robotstxt@^8.0.0:
+  version "8.0.3"
+  resolved "https://registry.yarnpkg.com/generate-robotstxt/-/generate-robotstxt-8.0.3.tgz#83384090406e760e9d02247412b2b704562c9b58"
+  integrity sha512-iD//oAVKcHOCz9M0IiT3pyUiF2uN1qvL3qaTA8RGLz7NU7l0XVwyzd3rN+tzhB657DNUgrygXt9w8+0zkTMFrg==
+  dependencies:
+    cosmiconfig "^6.0.0"
+    fs-extra "^9.0.0"
+    ip-regex "^4.1.0"
+    is-absolute-url "^3.0.3"
+    meow "^7.0.1"
+    resolve-from "^5.0.0"
+
 gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2:
   version "1.0.0-beta.2"
   resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
@@ -14569,6 +14581,23 @@ memorystream@^0.3.1:
   resolved "https://registry.yarnpkg.com/memorystream/-/memorystream-0.3.1.tgz#86d7090b30ce455d63fbae12dda51a47ddcaf9b2"
   integrity sha1-htcJCzDORV1j+64S3aUaR93K+bI=
 
+meow@^7.0.1:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/meow/-/meow-7.1.1.tgz#7c01595e3d337fcb0ec4e8eed1666ea95903d306"
+  integrity sha512-GWHvA5QOcS412WCo8vwKDlTelGLsCGBVevQB5Kva961rmNfun0PCbv5+xta2kUMFJyR8/oWnn7ddeKdosbAPbA==
+  dependencies:
+    "@types/minimist" "^1.2.0"
+    camelcase-keys "^6.2.2"
+    decamelize-keys "^1.1.0"
+    hard-rejection "^2.1.0"
+    minimist-options "4.1.0"
+    normalize-package-data "^2.5.0"
+    read-pkg-up "^7.0.1"
+    redent "^3.0.0"
+    trim-newlines "^3.0.0"
+    type-fest "^0.13.1"
+    yargs-parser "^18.1.3"
+
 meow@^8.0.0:
   version "8.1.2"
   resolved "https://registry.yarnpkg.com/meow/-/meow-8.1.2.tgz#bcbe45bda0ee1729d350c03cffc8395a36c4e897"
@@ -19338,6 +19367,14 @@ ripemd160@^2.0.0, ripemd160@^2.0.1:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
+robotstxt-webpack-plugin@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/robotstxt-webpack-plugin/-/robotstxt-webpack-plugin-7.0.0.tgz#04b26c9e1cd2ad8ebc5e20918aeeb6ed5d56835e"
+  integrity sha512-Rk0moeHvqc3to6oE2z5lKW9GTJyQC/ysogNy465N2CT2Xvp2/DMPkix8idnpmBLfExrQjml18B8Z/HoAC/yPlg==
+  dependencies:
+    generate-robotstxt "^8.0.0"
+    webpack-sources "^1.4.3"
+
 router-ips@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/router-ips/-/router-ips-1.0.0.tgz#44e00858ebebc0133d58e40b2cd8a1fbb04203f5"
@@ -21409,6 +21446,11 @@ type-fest@^0.12.0:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.12.0.tgz#f57a27ab81c68d136a51fd71467eff94157fa1ee"
   integrity sha512-53RyidyjvkGpnWPMF9bQgFtWp+Sl8O2Rp13VavmJgfAP9WWG6q6TkrKU8iyJdnwnfgHI6k2hTlgqH4aSdjoTbg==
 
+type-fest@^0.13.1:
+  version "0.13.1"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934"
+  integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==
+
 type-fest@^0.16.0:
   version "0.16.0"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.16.0.tgz#3240b891a78b0deae910dbeb86553e552a148860"
@@ -22782,7 +22824,7 @@ yargs-parser@^13.1.2:
     camelcase "^5.0.0"
     decamelize "^1.2.0"
 
-yargs-parser@^18.1.2:
+yargs-parser@^18.1.2, yargs-parser@^18.1.3:
   version "18.1.3"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0"
   integrity sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==