Add older docs to exclusion list on robots.txt

malfet · web-flow · commit 8b283f17c78b · 2021-11-03T10:26:01.000-07:00
All HTML files in those folders are already covered by noindex, but not .txt and others
diff --git a/robots.txt b/robots.txt
@@ -1,9 +1,16 @@
 User-agent: *
 Disallow: /docs/0.1.12/
 Disallow: /docs/0.2.0/
-Disallow: /docs/0.3.0/
-Disallow: /docs/0.3.1/
-Disallow: /docs/0.4.0/
-Disallow: /docs/0.4.1/
-Disallow: /docs/1.0.0/
-Disallow: /docs/1.0.1/
+Disallow: /docs/0.3.*
+Disallow: /docs/0.4.*
+Disallow: /docs/1.0.*
+Disallow: /docs/1.1.0/
+Disallow: /docs/1.2.0/
+Disallow: /docs/1.3.*
+Disallow: /docs/1.4.0/
+Disallow: /docs/1.5.*
+Disallow: /docs/1.6.0/
+Disallow: /docs/1.7.*
+Disallow: /docs/1.8.*
+Disallow: /docs/1.9.*
+Disallow: /docs/master/
