Add SearchReindexJob to repopulate the search index (#2850)
Rebuilds search records for every published Card and every Comment on a published Card. Intended for manual invocation after a search-index loss event — not a recurring job. Uses ActiveJob::Continuable so the work survives worker restarts and resumes from the last-processed id. The job is idempotent. Removes script/maintenance/reindex_stale_search_records.rb, which was written against a different hypothesis (a date-cutoff backfill) and is superseded by this unconditional repair job.
This commit is contained in:
Mike Dalessio
@@ -0,0 +1,60 @@
|
||||
# Repair task — reindexes every searchable Card and Comment in the database.
|
||||
# Intended to be invoked manually after a search-index loss event. Not a recurring job.
|
||||
#
|
||||
# Idempotent: Searchable#reindex upserts by (searchable_type, searchable_id).
|
||||
# Continuable: if the worker is interrupted, the job resumes from the last-processed id.
|
||||
#
|
||||
# `rich_text_limit` is a safety valve against unbounded bodies: records whose rich text
|
||||
# exceeds the limit (in bytes) are filtered out at the SQL level, so they never materialize
|
||||
# in Ruby. Without it, a single pathological body can stall the batch query or OOM the
|
||||
# worker during preload, which has happened in practice.
|
||||
#
|
||||
# Usage:
|
||||
# SearchReindexJob.perform_later
|
||||
# SearchReindexJob.perform_later(batch_size: 500, log_every: 100, rich_text_limit: 50_000)
|
||||
class SearchReindexJob < ApplicationJob
|
||||
include ActiveJob::Continuable
|
||||
|
||||
queue_as :backend
|
||||
|
||||
def perform(batch_size: 100, log_every: 1000, rich_text_limit: 100_000)
|
||||
step :cards do |step|
|
||||
processed = 0
|
||||
Card.published
|
||||
.left_joins(:rich_text_description)
|
||||
.where("action_text_rich_texts.body IS NULL OR OCTET_LENGTH(action_text_rich_texts.body) <= ?", rich_text_limit)
|
||||
.includes(:rich_text_description)
|
||||
.find_each(start: step.cursor, batch_size: batch_size) do |card|
|
||||
safely_reindex(card)
|
||||
processed += 1
|
||||
log_progress(:cards, processed, card.id) if (processed % log_every).zero?
|
||||
step.advance! from: card.id
|
||||
end
|
||||
end
|
||||
|
||||
step :comments do |step|
|
||||
processed = 0
|
||||
Comment.joins(:card).merge(Card.published)
|
||||
.left_joins(:rich_text_body)
|
||||
.where("action_text_rich_texts.body IS NULL OR OCTET_LENGTH(action_text_rich_texts.body) <= ?", rich_text_limit)
|
||||
.includes(:rich_text_body, :card)
|
||||
.find_each(start: step.cursor, batch_size: batch_size) do |comment|
|
||||
safely_reindex(comment)
|
||||
processed += 1
|
||||
log_progress(:comments, processed, comment.id) if (processed % log_every).zero?
|
||||
step.advance! from: comment.id
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def safely_reindex(record)
|
||||
record.reindex
|
||||
rescue StandardError => e
|
||||
Rails.error.report(e, context: { record_type: record.class.name, record_id: record.id })
|
||||
end
|
||||
|
||||
def log_progress(step_name, count, cursor)
|
||||
Rails.logger.info { "SearchReindexJob: step=#{step_name} processed=#{count} cursor=#{cursor}" }
|
||||
end
|
||||
end
|
||||
@@ -1,39 +0,0 @@
|
||||
# Reindex cards and comments that may be missing from the search index.
|
||||
#
|
||||
# The data import path (Account::DataTransfer) uses insert_all! which bypasses
|
||||
# ActiveRecord callbacks, so imported cards and comments never get search records
|
||||
# created. This script reindexes cards and comments created before a cutoff date.
|
||||
#
|
||||
# Usage:
|
||||
# bin/rails runner script/maintenance/reindex_stale_search_records.rb # default cutoff: 2025-11-13
|
||||
# bin/rails runner script/maintenance/reindex_stale_search_records.rb 2026-01-01 # custom cutoff
|
||||
|
||||
cutoff = Date.parse(ARGV[0] || "2025-11-13")
|
||||
|
||||
puts "Reindexing cards and comments created before #{cutoff}..."
|
||||
|
||||
cards = Card.published.where("created_at < ?", cutoff).includes(:rich_text_description)
|
||||
card_count = cards.count
|
||||
puts "Found #{card_count} cards to reindex"
|
||||
|
||||
reindexed_cards = 0
|
||||
cards.find_each do |card|
|
||||
card.reindex
|
||||
reindexed_cards += 1
|
||||
print "\rCards: #{reindexed_cards}/#{card_count}" if reindexed_cards % 100 == 0
|
||||
end
|
||||
puts "\rCards: #{reindexed_cards}/#{card_count}"
|
||||
|
||||
comments = Comment.joins(:card).merge(Card.published).where("comments.created_at < ?", cutoff).includes(:rich_text_body, :card)
|
||||
comment_count = comments.count
|
||||
puts "Found #{comment_count} comments to reindex"
|
||||
|
||||
reindexed_comments = 0
|
||||
comments.find_each do |comment|
|
||||
comment.reindex
|
||||
reindexed_comments += 1
|
||||
print "\rComments: #{reindexed_comments}/#{comment_count}" if reindexed_comments % 100 == 0
|
||||
end
|
||||
puts "\rComments: #{reindexed_comments}/#{comment_count}"
|
||||
|
||||
puts "Done! Reindexed #{reindexed_cards} cards and #{reindexed_comments} comments."
|
||||
@@ -0,0 +1,82 @@
|
||||
require "test_helper"
|
||||
|
||||
class SearchReindexJobTest < ActiveJob::TestCase
|
||||
test "reindexes cards and comments after their search records are nuked" do
|
||||
card = cards(:logo)
|
||||
comment = comments(:logo_1)
|
||||
|
||||
card.reindex
|
||||
comment.reindex
|
||||
|
||||
card_shard = Search::Record.for(card.account_id)
|
||||
comment_shard = Search::Record.for(comment.account_id)
|
||||
|
||||
assert card_shard.exists?(searchable_type: "Card", searchable_id: card.id)
|
||||
assert comment_shard.exists?(searchable_type: "Comment", searchable_id: comment.id)
|
||||
|
||||
card_shard.delete_all
|
||||
comment_shard.delete_all unless comment_shard == card_shard
|
||||
|
||||
assert_not card_shard.exists?(searchable_type: "Card", searchable_id: card.id)
|
||||
assert_not comment_shard.exists?(searchable_type: "Comment", searchable_id: comment.id)
|
||||
|
||||
SearchReindexJob.perform_now
|
||||
|
||||
assert card_shard.exists?(searchable_type: "Card", searchable_id: card.id)
|
||||
assert comment_shard.exists?(searchable_type: "Comment", searchable_id: comment.id)
|
||||
end
|
||||
|
||||
test "skips records whose rich text exceeds rich_text_limit" do
|
||||
Current.account = accounts(:"37s")
|
||||
Current.session = Session.new(identity: identities(:david))
|
||||
|
||||
big_card = boards(:writebook).cards.create!(
|
||||
creator: users(:david),
|
||||
title: "too big to index",
|
||||
status: :published,
|
||||
description: "x" * 5_000
|
||||
)
|
||||
|
||||
nuke_search_records
|
||||
|
||||
SearchReindexJob.perform_now(rich_text_limit: 1_000)
|
||||
|
||||
shard = Search::Record.for(big_card.account_id)
|
||||
assert_not shard.exists?(searchable_type: "Card", searchable_id: big_card.id)
|
||||
end
|
||||
|
||||
test "does not index drafted cards or their comments" do
|
||||
Current.account = accounts(:"37s")
|
||||
Current.session = Session.new(identity: identities(:david))
|
||||
|
||||
card = boards(:writebook).cards.create!(
|
||||
creator: users(:david),
|
||||
title: "will be drafted",
|
||||
status: :published
|
||||
)
|
||||
comment = card.comments.create!(creator: users(:david), body: "on a card that will be drafted")
|
||||
card.update!(status: :drafted)
|
||||
|
||||
nuke_search_records
|
||||
|
||||
SearchReindexJob.perform_now
|
||||
|
||||
shard = Search::Record.for(card.account_id)
|
||||
assert_not shard.exists?(searchable_type: "Card", searchable_id: card.id)
|
||||
assert_not shard.exists?(searchable_type: "Comment", searchable_id: comment.id)
|
||||
end
|
||||
|
||||
private
|
||||
def sqlite?
|
||||
ActiveRecord::Base.connection.adapter_name == "SQLite"
|
||||
end
|
||||
|
||||
def nuke_search_records
|
||||
if sqlite?
|
||||
ActiveRecord::Base.connection.execute("DELETE FROM search_records")
|
||||
ActiveRecord::Base.connection.execute("DELETE FROM search_records_fts")
|
||||
else
|
||||
Search::Record::Trilogy::SHARD_CLASSES.each(&:delete_all)
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user