Add SEO optimization files: robots.txt and sitemap.xml

Author: calderbuild

robots.txt

@@ -0,0 +1,26 @@
+User-agent: *
+Allow: /
+
+# Disallow crawling of admin and private directories
+Disallow: /.git/
+Disallow: /.github/
+Disallow: /_site/
+Disallow: /node_modules/
+Disallow: /.sass-cache/
+Disallow: /.jekyll-cache/
+Disallow: /.bundle/
+Disallow: /vendor/
+
+# Allow crawling of important files
+Allow: /css/
+Allow: /js/
+Allow: /img/
+Allow: /fonts/
+Allow: /feed.xml
+Allow: /sitemap.xml
+
+# Sitemap location
+Sitemap: https://jasonrobert.me/sitemap.xml
+
+# Crawl delay (optional - be respectful to search engines)
+Crawl-delay: 1

sitemap.xml

@@ -0,0 +1,59 @@
+---
+layout: null
+---
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <!-- Homepage -->
+  <url>
+    <loc>{{ site.url }}/</loc>
+    <lastmod>{{ site.time | date_to_xmlschema }}</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>1.0</priority>
+  </url>
+  
+  <!-- About page -->
+  <url>
+    <loc>{{ site.url }}/about/</loc>
+    <lastmod>{{ site.time | date_to_xmlschema }}</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.8</priority>
+  </url>
+  
+  <!-- Archive page -->
+  <url>
+    <loc>{{ site.url }}/archive/</loc>
+    <lastmod>{{ site.time | date_to_xmlschema }}</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>0.7</priority>
+  </url>
+  
+  <!-- Portfolio page -->
+  <url>
+    <loc>{{ site.url }}/portfolio/</loc>
+    <lastmod>{{ site.time | date_to_xmlschema }}</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.6</priority>
+  </url>
+  
+  <!-- Blog posts -->
+  {% for post in site.posts %}
+  <url>
+    <loc>{{ site.url }}{{ post.url }}</loc>
+    <lastmod>{{ post.date | date_to_xmlschema }}</lastmod>
+    <changefreq>monthly</changefreq>
+    <priority>0.9</priority>
+  </url>
+  {% endfor %}
+  
+  <!-- Pages -->
+  {% for page in site.pages %}
+    {% unless page.url contains '/404' or page.url contains '/feed' or page.url contains '/sitemap' %}
+    <url>
+      <loc>{{ site.url }}{{ page.url }}</loc>
+      <lastmod>{{ site.time | date_to_xmlschema }}</lastmod>
+      <changefreq>monthly</changefreq>
+      <priority>0.5</priority>
+    </url>
+    {% endunless %}
+  {% endfor %}
+</urlset>