Generate sitemap and add to robots.txt

Signed-off-by: Ruby Iris Juric <[email protected]>

Author: Sorixelle

Gemfile

@@ -18,6 +18,7 @@ ruby file: '.ruby-version'
 gem 'slugize', '>= 0.0.3'
 gem 'jekyll', '>= 3.0.3'
 gem 'jekyll-paginate'
+gem 'jekyll-sitemap'
 gem 'bundler', '>= 1.7.9'
 gem 'nokogiri', '>= 1.6.3.1'
 gem 'algolia', '>= 3.22.0'

Gemfile.lock

@@ -98,6 +98,8 @@ GEM
     jekyll-paginate (1.1.0)
     jekyll-sass-converter (3.1.0)
       sass-embedded (~> 1.75)
+    jekyll-sitemap (1.4.0)
+      jekyll (>= 3.7, < 5.0)
     jekyll-watch (2.2.1)
       listen (~> 3.0)
     json (2.13.0)
@@ -262,6 +264,7 @@ DEPENDENCIES
   htmlentities (>= 4.3.2)
   jekyll (>= 3.0.3)
   jekyll-paginate
+  jekyll-sitemap
   newrelic_rpm (>= 3.9.8.273)
   nokogiri (>= 1.6.3.1)
   pygments.rb

_config.yml

@@ -27,7 +27,9 @@ plugins_dir: plugins/
 
 debug: true
 
-plugins: [jekyll-paginate]
+plugins:
+  - jekyll-paginate
+  - jekyll-sitemap
 
 # Blog Options
 permalink: none
@@ -138,4 +140,11 @@ defaults:
     values:
       layout: sdk/changelog
       generate_toc: true
-
+  - scope:
+      path: developer.pebble.com
+    values:
+      sitemap: false
+  - scope:
+      path: developer.getpebble.com
+    values:
+      sitemap: false

plugins/generator_redirects.rb

@@ -71,6 +71,7 @@ def initialize(site, base, dir, name, redirect_to)
       process(@name)
       read_yaml(File.join(base, '_layouts', 'utils'), 'redirect_permanent.html')
       data['redirect_to'] = redirect_to.delete_suffix('index.html')
+      data['sitemap'] = false
     end
   end
 end

source/robots.txt

@@ -1,2 +1,4 @@
 User-agent: *
-Allow: /
+Allow: /
+
+Sitemap: https://developer.rebble.io/sitemap.xml