From b755b3feadd1f7935c8ad14e04bc227757ae1b26 Mon Sep 17 00:00:00 2001
From: Jeremy Daer <jeremy@37signals.com>
Date: Tue, 2 Dec 2025 13:35:58 -0800
Subject: [PATCH] Robots, begone (#1812)

* robots.txt: "Please, don't come in." If a page is directly linked, the
  URL can still appear in search results, though.
* X-Robots-Tag: "If you're here, forget what you saw." Works even if the
  crawler ignores robots.txt or reaches a page via external link. Can
  remove already-indexed pages.
* Public boards may not be indexed. They're meant for "anyone with the
  link" private sharing, not worldwide publishing.
---
 app/controllers/application_controller.rb     |  1 +
 .../concerns/block_search_engine_indexing.rb  | 14 ++++++++++
 public/robots.txt                             |  3 +-
 .../block_search_engine_indexing_test.rb      | 28 +++++++++++++++++++
 4 files changed, 45 insertions(+), 1 deletion(-)
 create mode 100644 app/controllers/concerns/block_search_engine_indexing.rb
 create mode 100644 test/controllers/concerns/block_search_engine_indexing_test.rb

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 7f1cb6565..87c3ab8dd 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -1,6 +1,7 @@
 class ApplicationController < ActionController::Base
   include Authentication
   include Authorization
+  include BlockSearchEngineIndexing
   include CurrentRequest, CurrentTimezone, SetPlatform
   include RequestForgeryProtection
   include TurboFlash, ViewTransitions
diff --git a/app/controllers/concerns/block_search_engine_indexing.rb b/app/controllers/concerns/block_search_engine_indexing.rb
new file mode 100644
index 000000000..65bb50d2b
--- /dev/null
+++ b/app/controllers/concerns/block_search_engine_indexing.rb
@@ -0,0 +1,14 @@
+# Tell crawlers like Googlebot to drop pages entirely from search results, even
+# if other sites link to it
+module BlockSearchEngineIndexing
+  extend ActiveSupport::Concern
+
+  included do
+    after_action :block_search_engine_indexing
+  end
+
+  private
+    def block_search_engine_indexing
+      headers["X-Robots-Tag"] = "none"
+    end
+end
diff --git a/public/robots.txt b/public/robots.txt
index c19f78ab6..c6742d8a8 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1 +1,2 @@
-# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
+User-Agent: *
+Disallow: /
diff --git a/test/controllers/concerns/block_search_engine_indexing_test.rb b/test/controllers/concerns/block_search_engine_indexing_test.rb
new file mode 100644
index 000000000..fd2789754
--- /dev/null
+++ b/test/controllers/concerns/block_search_engine_indexing_test.rb
@@ -0,0 +1,28 @@
+require "test_helper"
+
+class BlockSearchEngineIndexingTest < ActionDispatch::IntegrationTest
+  test "sets X-Robots-Tag header to none on authenticated requests" do
+    sign_in_as :david
+
+    get board_path(boards(:writebook))
+    assert_response :success
+    assert_equal "none", response.headers["X-Robots-Tag"]
+  end
+
+  test "sets X-Robots-Tag header to none on unauthenticated requests" do
+    untenanted do
+      get new_session_path
+    end
+
+    assert_response :success
+    assert_equal "none", response.headers["X-Robots-Tag"]
+  end
+
+  test "sets X-Robots-Tag header to none on public board pages" do
+    boards(:writebook).publish
+
+    get public_board_path(boards(:writebook).publication.key)
+    assert_response :success
+    assert_equal "none", response.headers["X-Robots-Tag"]
+  end
+end