DEV: Add site description to crawler homepage view (#32845)

pmusaraj · web-flow · commit 2b6efa56f35d · 2025-05-22T08:33:59.000+10:00
In some cases, Google crawlers don't output the meta description but
rather they output the first bit of text in the UI. Sometimes that is a
mix of table headings, topic titles and excerpts, which don't reflect
the site's mission. Adding the description to the homepage header might
help.

Internal ticket t/154372
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
@@ -80,6 +80,7 @@ def use_crawler_layout?
             CrawlerDetection.crawler?(request.user_agent, request.headers["HTTP_VIA"])
         )
   end
+  helper_method :use_crawler_layout?
 
   def perform_refresh_session
     refresh_session(current_user) unless @readonly_mode
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
@@ -288,6 +288,9 @@ def description_content
     )
   end
 
+  def is_crawler_homepage?
+    request.path == "/" && use_crawler_layout?
+  end
   # Creates open graph and twitter card meta data
   def crawlable_meta_data(opts = nil)
     opts ||= {}
diff --git a/app/views/layouts/_noscript_header.html.erb b/app/views/layouts/_noscript_header.html.erb
@@ -1,5 +1,6 @@
 <header>
-  <a href="<%= path "/" %>">
-    <%=SiteSetting.title%>
-  </a>
+  <a href="<%= path "/" %>"><%=SiteSetting.title%></a>
+  <% if is_crawler_homepage? %>
+    <p><%= SiteSetting.site_description %></p>
+  <% end %>
 </header>
diff --git a/spec/requests/home_page_controller_spec.rb b/spec/requests/home_page_controller_spec.rb
@@ -1,8 +1,13 @@
 # frozen_string_literal: true
 
 RSpec.describe HomePageController do
-  describe "#custom" do
+  describe "homepage" do
     context "with crawler view" do
+      before do
+        SiteSetting.site_description = "This is a test description"
+        SiteSetting.has_login_hint = false
+      end
+
       it "should display the menu by default" do
         get "/custom", headers: { "HTTP_USER_AGENT" => "Googlebot" }
 
@@ -43,6 +48,27 @@ def enabled?
           DiscoursePluginRegistry.reset!
         end
       end
+
+      it "should display the site description on the homepage" do
+        get "/", headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+        expect(response.status).to eq(200)
+        expect(response.body).to include("<p>This is a test description</p>")
+        expect(response.body).to include(
+          "<meta name=\"description\" content=\"This is a test description\">",
+        )
+      end
+
+      it "should not display the site description on another route" do
+        get "/top", headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+        expect(response.status).to eq(200)
+        expect(response.body).not_to include("<p>This is a test description</p>")
+        # but still includes the meta tag
+        expect(response.body).to include(
+          "<meta name=\"description\" content=\"This is a test description\">",
+        )
+      end
     end
   end
 end

Original file line number	Diff line number	Diff line change
`@@ -80,6 +80,7 @@ def use_crawler_layout?`
`80`	`80`	`CrawlerDetection.crawler?(request.user_agent, request.headers["HTTP_VIA"])`
`81`	`81`	`)`
`82`	`82`	`end`
	`83`	`+ helper_method :use_crawler_layout?`
`83`	`84`
`84`	`85`	`def perform_refresh_session`
`85`	`86`	`refresh_session(current_user) unless @readonly_mode`
Original file line number	Diff line number	Diff line change
`@@ -288,6 +288,9 @@ def description_content`
`288`	`288`	`)`
`289`	`289`	`end`
`290`	`290`
	`291`	`+ def is_crawler_homepage?`
	`292`	`+ request.path == "/" && use_crawler_layout?`
	`293`	`+ end`
`291`	`294`	`# Creates open graph and twitter card meta data`
`292`	`295`	`def crawlable_meta_data(opts = nil)`
`293`	`296`	`opts \|\|= {}`