add robots.txt to guide crawlers

Author: floriankozikowski

doc/conf.py

@@ -209,7 +209,7 @@
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-# html_extra_path = []
+html_extra_path = ['robots.txt']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

doc/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Sitemap: https://contrib.scikit-learn.org/skglm/sitemap.xml