From 466b215ef83dfe64aea26bec3e7bdea3626cb321 Mon Sep 17 00:00:00 2001
From: Mike Dalessio <mike@37signals.com>
Date: Thu, 16 Apr 2026 19:40:02 -0400
Subject: [PATCH] Add SearchReindexJob to repopulate the search index (#2850)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Rebuilds search records for every published Card and every Comment on a
published Card. Intended for manual invocation after a search-index loss
event — not a recurring job.

Uses ActiveJob::Continuable so the work survives worker restarts and
resumes from the last-processed id. The job is idempotent.

Removes script/maintenance/reindex_stale_search_records.rb, which was
written against a different hypothesis (a date-cutoff backfill) and is
superseded by this unconditional repair job.
---
 app/jobs/search_reindex_job.rb                | 60 ++++++++++++++
 .../reindex_stale_search_records.rb           | 39 ---------
 test/jobs/search_reindex_job_test.rb          | 82 +++++++++++++++++++
 3 files changed, 142 insertions(+), 39 deletions(-)
 create mode 100644 app/jobs/search_reindex_job.rb
 delete mode 100644 script/maintenance/reindex_stale_search_records.rb
 create mode 100644 test/jobs/search_reindex_job_test.rb

diff --git a/app/jobs/search_reindex_job.rb b/app/jobs/search_reindex_job.rb
new file mode 100644
index 000000000..84fadaa8c
--- /dev/null
+++ b/app/jobs/search_reindex_job.rb
@@ -0,0 +1,60 @@
+# Repair task — reindexes every searchable Card and Comment in the database.
+# Intended to be invoked manually after a search-index loss event. Not a recurring job.
+#
+# Idempotent: Searchable#reindex upserts by (searchable_type, searchable_id).
+# Continuable: if the worker is interrupted, the job resumes from the last-processed id.
+#
+# `rich_text_limit` is a safety valve against unbounded bodies: records whose rich text
+# exceeds the limit (in bytes) are filtered out at the SQL level, so they never materialize
+# in Ruby. Without it, a single pathological body can stall the batch query or OOM the
+# worker during preload, which has happened in practice.
+#
+# Usage:
+#   SearchReindexJob.perform_later
+#   SearchReindexJob.perform_later(batch_size: 500, log_every: 100, rich_text_limit: 50_000)
+class SearchReindexJob < ApplicationJob
+  include ActiveJob::Continuable
+
+  queue_as :backend
+
+  def perform(batch_size: 100, log_every: 1000, rich_text_limit: 100_000)
+    step :cards do |step|
+      processed = 0
+      Card.published
+          .left_joins(:rich_text_description)
+          .where("action_text_rich_texts.body IS NULL OR OCTET_LENGTH(action_text_rich_texts.body) <= ?", rich_text_limit)
+          .includes(:rich_text_description)
+          .find_each(start: step.cursor, batch_size: batch_size) do |card|
+        safely_reindex(card)
+        processed += 1
+        log_progress(:cards, processed, card.id) if (processed % log_every).zero?
+        step.advance! from: card.id
+      end
+    end
+
+    step :comments do |step|
+      processed = 0
+      Comment.joins(:card).merge(Card.published)
+             .left_joins(:rich_text_body)
+             .where("action_text_rich_texts.body IS NULL OR OCTET_LENGTH(action_text_rich_texts.body) <= ?", rich_text_limit)
+             .includes(:rich_text_body, :card)
+             .find_each(start: step.cursor, batch_size: batch_size) do |comment|
+        safely_reindex(comment)
+        processed += 1
+        log_progress(:comments, processed, comment.id) if (processed % log_every).zero?
+        step.advance! from: comment.id
+      end
+    end
+  end
+
+  private
+    def safely_reindex(record)
+      record.reindex
+    rescue StandardError => e
+      Rails.error.report(e, context: { record_type: record.class.name, record_id: record.id })
+    end
+
+    def log_progress(step_name, count, cursor)
+      Rails.logger.info { "SearchReindexJob: step=#{step_name} processed=#{count} cursor=#{cursor}" }
+    end
+end
diff --git a/script/maintenance/reindex_stale_search_records.rb b/script/maintenance/reindex_stale_search_records.rb
deleted file mode 100644
index fd848f67d..000000000
--- a/script/maintenance/reindex_stale_search_records.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# Reindex cards and comments that may be missing from the search index.
-#
-# The data import path (Account::DataTransfer) uses insert_all! which bypasses
-# ActiveRecord callbacks, so imported cards and comments never get search records
-# created. This script reindexes cards and comments created before a cutoff date.
-#
-# Usage:
-#   bin/rails runner script/maintenance/reindex_stale_search_records.rb              # default cutoff: 2025-11-13
-#   bin/rails runner script/maintenance/reindex_stale_search_records.rb 2026-01-01   # custom cutoff
-
-cutoff = Date.parse(ARGV[0] || "2025-11-13")
-
-puts "Reindexing cards and comments created before #{cutoff}..."
-
-cards = Card.published.where("created_at < ?", cutoff).includes(:rich_text_description)
-card_count = cards.count
-puts "Found #{card_count} cards to reindex"
-
-reindexed_cards = 0
-cards.find_each do |card|
-  card.reindex
-  reindexed_cards += 1
-  print "\rCards: #{reindexed_cards}/#{card_count}" if reindexed_cards % 100 == 0
-end
-puts "\rCards: #{reindexed_cards}/#{card_count}"
-
-comments = Comment.joins(:card).merge(Card.published).where("comments.created_at < ?", cutoff).includes(:rich_text_body, :card)
-comment_count = comments.count
-puts "Found #{comment_count} comments to reindex"
-
-reindexed_comments = 0
-comments.find_each do |comment|
-  comment.reindex
-  reindexed_comments += 1
-  print "\rComments: #{reindexed_comments}/#{comment_count}" if reindexed_comments % 100 == 0
-end
-puts "\rComments: #{reindexed_comments}/#{comment_count}"
-
-puts "Done! Reindexed #{reindexed_cards} cards and #{reindexed_comments} comments."
diff --git a/test/jobs/search_reindex_job_test.rb b/test/jobs/search_reindex_job_test.rb
new file mode 100644
index 000000000..da238e3c5
--- /dev/null
+++ b/test/jobs/search_reindex_job_test.rb
@@ -0,0 +1,82 @@
+require "test_helper"
+
+class SearchReindexJobTest < ActiveJob::TestCase
+  test "reindexes cards and comments after their search records are nuked" do
+    card = cards(:logo)
+    comment = comments(:logo_1)
+
+    card.reindex
+    comment.reindex
+
+    card_shard = Search::Record.for(card.account_id)
+    comment_shard = Search::Record.for(comment.account_id)
+
+    assert card_shard.exists?(searchable_type: "Card", searchable_id: card.id)
+    assert comment_shard.exists?(searchable_type: "Comment", searchable_id: comment.id)
+
+    card_shard.delete_all
+    comment_shard.delete_all unless comment_shard == card_shard
+
+    assert_not card_shard.exists?(searchable_type: "Card", searchable_id: card.id)
+    assert_not comment_shard.exists?(searchable_type: "Comment", searchable_id: comment.id)
+
+    SearchReindexJob.perform_now
+
+    assert card_shard.exists?(searchable_type: "Card", searchable_id: card.id)
+    assert comment_shard.exists?(searchable_type: "Comment", searchable_id: comment.id)
+  end
+
+  test "skips records whose rich text exceeds rich_text_limit" do
+    Current.account = accounts(:"37s")
+    Current.session = Session.new(identity: identities(:david))
+
+    big_card = boards(:writebook).cards.create!(
+      creator: users(:david),
+      title: "too big to index",
+      status: :published,
+      description: "x" * 5_000
+    )
+
+    nuke_search_records
+
+    SearchReindexJob.perform_now(rich_text_limit: 1_000)
+
+    shard = Search::Record.for(big_card.account_id)
+    assert_not shard.exists?(searchable_type: "Card", searchable_id: big_card.id)
+  end
+
+  test "does not index drafted cards or their comments" do
+    Current.account = accounts(:"37s")
+    Current.session = Session.new(identity: identities(:david))
+
+    card = boards(:writebook).cards.create!(
+      creator: users(:david),
+      title: "will be drafted",
+      status: :published
+    )
+    comment = card.comments.create!(creator: users(:david), body: "on a card that will be drafted")
+    card.update!(status: :drafted)
+
+    nuke_search_records
+
+    SearchReindexJob.perform_now
+
+    shard = Search::Record.for(card.account_id)
+    assert_not shard.exists?(searchable_type: "Card", searchable_id: card.id)
+    assert_not shard.exists?(searchable_type: "Comment", searchable_id: comment.id)
+  end
+
+  private
+    def sqlite?
+      ActiveRecord::Base.connection.adapter_name == "SQLite"
+    end
+
+    def nuke_search_records
+      if sqlite?
+        ActiveRecord::Base.connection.execute("DELETE FROM search_records")
+        ActiveRecord::Base.connection.execute("DELETE FROM search_records_fts")
+      else
+        Search::Record::Trilogy::SHARD_CLASSES.each(&:delete_all)
+      end
+    end
+end