Merge pull request #474 from dzcode-io/473-add-robottxt-to-disallow-indexing-non-production-version-of-the-website

Author: ZibanPirate

.github/workflows/cd.deploy.stg.yml

@@ -14,6 +14,14 @@ jobs:
         node-version: [14.x]
         os: [ubuntu-latest]
 
+    env:
+      CI: true
+      STAGE: staging
+      SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
+      SSH_ADDRESS_STG: ${{ secrets.SSH_ADDRESS_STG }}
+      FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
+      EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }}
+
     steps:
       - name: "Git"
         uses: actions/checkout@v2
@@ -60,10 +68,3 @@ jobs:
         shell: bash
         run: |
           yarn deploy:stg
-
-        env:
-          CI: true
-          SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
-          SSH_ADDRESS_STG: ${{ secrets.SSH_ADDRESS_STG }}
-          FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
-          EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }}

.github/workflows/cd.deploy.yml

@@ -14,6 +14,14 @@ jobs:
         node-version: [14.x]
         os: [ubuntu-latest]
 
+    env:
+      CI: true
+      STAGE: production
+      SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
+      SSH_ADDRESS_PRD: ${{ secrets.SSH_ADDRESS_PRD }}
+      FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
+      EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }}
+
     steps:
       - name: "Git"
         uses: actions/checkout@v2
@@ -67,10 +75,3 @@ jobs:
         shell: bash
         run: |
           yarn deploy
-
-        env:
-          CI: true
-          SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
-          SSH_ADDRESS_PRD: ${{ secrets.SSH_ADDRESS_PRD }}
-          FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
-          EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }}

api/package.json

@@ -19,6 +19,7 @@
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
     "express-rate-limit": "^5.2.6",
+    "express-robots-txt": "^1.0.0",
     "fs-extra": "^10.0.0",
     "helmet": "^4.4.1",
     "make-fetch-happen": "^9.0.2",

api/src/app/index.ts

@@ -19,6 +19,7 @@ import { LoggerMiddleware } from "./middlewares/logger";
 import { SecurityMiddleware } from "./middlewares/security";
 import { SentryErrorHandlerMiddleware } from "./middlewares/sentry-error-handler";
 import { SentryRequestHandlerMiddleware } from "./middlewares/sentry-request-handler";
+const robots = require("express-robots-txt"); // eslint-disable-line @typescript-eslint/no-var-requires
 
 // Use typedi container
 useContainer(Container);
@@ -59,6 +60,8 @@ const app: Application = createExpressServer(routingControllersOptions);
 
 const logger = Container.get(LoggerService);
 
+app.use(robots({ UserAgent: "*", Disallow: "/" }));
+
 // Start it
 app.listen(PORT, () => {
   const commonConfig = fsConfig(NODE_ENV);

web/package.json

@@ -82,6 +82,7 @@
     "postcss-preset-env": "^7.0.1",
     "pug": "^3.0.0",
     "pug-loader": "^2.4.0",
+    "robotstxt-webpack-plugin": "^7.0.0",
     "sass": "^1.30.0",
     "sass-loader": "^10.1.0",
     "sitemap": "^6.3.5",

web/webpack.config.ts

@@ -1,4 +1,5 @@
 import { fsConfig } from "@dzcode.io/utils/dist/config";
+import { Environment } from "@dzcode.io/utils/dist/config/environment";
 import CssMinimizerPlugin from "css-minimizer-webpack-plugin";
 import { readFileSync } from "fs";
 import glob from "glob";
@@ -10,9 +11,14 @@ import { Configuration as WPC } from "webpack";
 import { BundleAnalyzerPlugin } from "webpack-bundle-analyzer";
 import { Configuration as WPDSC } from "webpack-dev-server";
 
+const RobotstxtPlugin = require("robotstxt-webpack-plugin"); // eslint-disable-line @typescript-eslint/no-var-requires
+
+// @TODO-ZM: to combine isProduction and isDevelopment and stage
+
 // setting up project configurations and some env variables
 const ANALYZE = process.env.ANALYZE === "true";
 const isProduction = process.env.NODE_ENV === "production" || ANALYZE;
+const stage = (process.env.STAGE as Environment) || "production";
 const isDevelopment = process.env.NODE_ENV === "development" && !ANALYZE;
 const { web } = fsConfig("development");
 const distFolder = "./bundle";
@@ -157,6 +163,15 @@ export default {
           new BundleAnalyzerPlugin(),
         ]
       : []),
+    new RobotstxtPlugin({
+      policy: [
+        {
+          userAgent: "*",
+          allow: stage === "production" ? "/" : undefined,
+          disallow: stage !== "production" ? "/" : undefined,
+        },
+      ],
+    }),
     ...apps.reduce<WPC["plugins"][]>(
       (pV, app) => [...pV, ...require(`./src/apps/${app}/entry/webpack.plugins`)],
       [],

yarn.lock

@@ -9711,6 +9711,11 @@ express-rate-limit@^5.2.6:
   resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-5.5.1.tgz#110c23f6a65dfa96ab468eda95e71697bc6987a2"
   integrity sha512-MTjE2eIbHv5DyfuFz4zLYWxpqVhEhkTiwFGuB74Q9CSou2WHO52nlE5y3Zlg6SIsiYUIPj6ifFxnkPz6O3sIUg==
 
+express-robots-txt@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/express-robots-txt/-/express-robots-txt-1.0.0.tgz#354dde6f241c3d4ed39f74cfd0604dd92daea497"
+  integrity sha512-ZzXTJgvHY6N8MIPzWidIrOGngkNJ8nrSP9Ox0nztVkQxvpwcX8egJvhM6tBUojf9xDmnSc7J5RlWQJJK91N3aA==
+
 express-session@^1.17.1:
   version "1.17.2"
   resolved "https://registry.yarnpkg.com/express-session/-/express-session-1.17.2.tgz#397020374f9bf7997f891b85ea338767b30d0efd"
@@ -10624,6 +10629,18 @@ gcp-metadata@^4.2.0:
     gaxios "^4.0.0"
     json-bigint "^1.0.0"
 
+generate-robotstxt@^8.0.0:
+  version "8.0.3"
+  resolved "https://registry.yarnpkg.com/generate-robotstxt/-/generate-robotstxt-8.0.3.tgz#83384090406e760e9d02247412b2b704562c9b58"
+  integrity sha512-iD//oAVKcHOCz9M0IiT3pyUiF2uN1qvL3qaTA8RGLz7NU7l0XVwyzd3rN+tzhB657DNUgrygXt9w8+0zkTMFrg==
+  dependencies:
+    cosmiconfig "^6.0.0"
+    fs-extra "^9.0.0"
+    ip-regex "^4.1.0"
+    is-absolute-url "^3.0.3"
+    meow "^7.0.1"
+    resolve-from "^5.0.0"
+
 gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2:
   version "1.0.0-beta.2"
   resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
@@ -14569,6 +14586,23 @@ memorystream@^0.3.1:
   resolved "https://registry.yarnpkg.com/memorystream/-/memorystream-0.3.1.tgz#86d7090b30ce455d63fbae12dda51a47ddcaf9b2"
   integrity sha1-htcJCzDORV1j+64S3aUaR93K+bI=
 
+meow@^7.0.1:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/meow/-/meow-7.1.1.tgz#7c01595e3d337fcb0ec4e8eed1666ea95903d306"
+  integrity sha512-GWHvA5QOcS412WCo8vwKDlTelGLsCGBVevQB5Kva961rmNfun0PCbv5+xta2kUMFJyR8/oWnn7ddeKdosbAPbA==
+  dependencies:
+    "@types/minimist" "^1.2.0"
+    camelcase-keys "^6.2.2"
+    decamelize-keys "^1.1.0"
+    hard-rejection "^2.1.0"
+    minimist-options "4.1.0"
+    normalize-package-data "^2.5.0"
+    read-pkg-up "^7.0.1"
+    redent "^3.0.0"
+    trim-newlines "^3.0.0"
+    type-fest "^0.13.1"
+    yargs-parser "^18.1.3"
+
 meow@^8.0.0:
   version "8.1.2"
   resolved "https://registry.yarnpkg.com/meow/-/meow-8.1.2.tgz#bcbe45bda0ee1729d350c03cffc8395a36c4e897"
@@ -19338,6 +19372,14 @@ ripemd160@^2.0.0, ripemd160@^2.0.1:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
+robotstxt-webpack-plugin@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/robotstxt-webpack-plugin/-/robotstxt-webpack-plugin-7.0.0.tgz#04b26c9e1cd2ad8ebc5e20918aeeb6ed5d56835e"
+  integrity sha512-Rk0moeHvqc3to6oE2z5lKW9GTJyQC/ysogNy465N2CT2Xvp2/DMPkix8idnpmBLfExrQjml18B8Z/HoAC/yPlg==
+  dependencies:
+    generate-robotstxt "^8.0.0"
+    webpack-sources "^1.4.3"
+
 router-ips@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/router-ips/-/router-ips-1.0.0.tgz#44e00858ebebc0133d58e40b2cd8a1fbb04203f5"
@@ -21409,6 +21451,11 @@ type-fest@^0.12.0:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.12.0.tgz#f57a27ab81c68d136a51fd71467eff94157fa1ee"
   integrity sha512-53RyidyjvkGpnWPMF9bQgFtWp+Sl8O2Rp13VavmJgfAP9WWG6q6TkrKU8iyJdnwnfgHI6k2hTlgqH4aSdjoTbg==
 
+type-fest@^0.13.1:
+  version "0.13.1"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934"
+  integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==
+
 type-fest@^0.16.0:
   version "0.16.0"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.16.0.tgz#3240b891a78b0deae910dbeb86553e552a148860"
@@ -22782,7 +22829,7 @@ yargs-parser@^13.1.2:
     camelcase "^5.0.0"
     decamelize "^1.2.0"
 
-yargs-parser@^18.1.2:
+yargs-parser@^18.1.2, yargs-parser@^18.1.3:
   version "18.1.3"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0"
   integrity sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==