Feat: seo improvements v2 (#868)

* seo improvements

* seo style adjustment + refactor

* seo prevent qa + dev from being crawled

Author: Alessandro100

.github/workflows/web-app-deployer.yml

@@ -266,6 +266,13 @@ jobs:
         working-directory: functions
         run: npx firebase deploy --only functions
 
+      - name: Set robots.txt
+        working-directory: web-app
+        run: |
+          if [ "${{ inputs.FIREBASE_PROJECT }}" != "prod" ]; then
+            mv public/robots.staging.txt public/robots.txt
+          fi
+
       - name: Build
         working-directory: web-app
         run: yarn build:${FIREBASE_PROJECT}

web-app/public/robots.staging.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

web-app/public/robots.txt

@@ -1,3 +1,3 @@
-# https://www.robotstxt.org/robotstxt.html
 User-agent: *
-Disallow:
+Allow: /
+Sitemap: https://mobilitydatabase.org/sitemap.xml