FIX: robots.txt

jklymak · jklymak · commit 3fb5acf20bf9 · 2021-01-25T21:19:04.000-08:00
diff --git a/robots.txt b/robots.txt
@@ -5,32 +5,27 @@
 User-agent: *
 
 # do not search root directory by default.
-Disallow: /
-Allow: /_sitemap/
-
-# but allow some subdirectories:
-Allow: /stable/
-
-# versions should be searched.  Note sitemap.xml downweights them...
-Allow: /5.*/
-Allow: /4.*/
-Allow: /3.*/
-Allow: /2.*/
-Allow: /1.*/
-
-# packages that should still be searched:
-Allow: /cmocean/
-Allow: /mpl-probscale/
-Allow: /mpl_toolkits/
-Allow: /xkcd/
-Allow: /cycler/
-Allow: /matplotblog/
-Allow: /pytest-mpl/
-Allow: /basemap/
-Allow: /mpl-bench/
-Allow: /mpl-altair/
-Allow: /devdocs/
-
+Disallow: /users/
+Disallow: /tutorials/
+Disallow: /thirdpartypackages/
+Disallow: /resources/
+Disallow: /glossary/
+Disallow: /gallery/
+Disallow: /faq/
+Disallow: /examples/
+Disallow: /devel/
+Disallow: /api/
+Disallow: /_static/
+Disallow: /_sources/
+Disallow: /_modules/
+Disallow: /_images/
+Disallow: /_downloads/
+Disallow: /citing.html
+Disallow: /contents.html
+Disallow: /genindex.html
+Disallow: /index.html
+Disallow: /py-modindex.html
+Disallow: /search.html
 
 # tell robots this is sitemap
 Sitemap: https://matplotlib.org/_sitemap/sitemap.xml
