From 28efe28f2451f4f303868c66490397967c0acc38 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 13 Nov 2025 18:08:29 +0000 Subject: [PATCH] Replace Search::Index with Search::Records Lean on ActiveRecord models for searching and strip out the raw SQL. Replaces the search_index_* tables with sharded search_records_* tables as that allows us to use a Search::Record model name. A Class is dynamically created for each record table shard so that we and we can access it via the Search::Record.for_account(account_id) method. --- app/models/card/searchable.rb | 41 +- app/models/comment/searchable.rb | 25 +- app/models/concerns/searchable.rb | 74 ++-- app/models/search.rb | 46 +- app/models/search/record.rb | 90 ++++ app/models/user/searcher.rb | 2 +- app/views/searches/_result.html.erb | 6 +- ...51113190256_create_search_record_shards.rb | 34 ++ db/schema.rb | 418 ++++++++++-------- lib/tasks/search.rake | 6 +- test/controllers/searches_controller_test.rb | 33 +- test/models/card/searchable_test.rb | 17 +- test/models/comment/searchable_test.rb | 56 +-- test/models/search_test.rb | 27 +- test/test_helpers/search_test_helper.rb | 36 ++ 15 files changed, 484 insertions(+), 427 deletions(-) create mode 100644 app/models/search/record.rb create mode 100644 db/migrate/20251113190256_create_search_record_shards.rb create mode 100644 test/test_helpers/search_test_helper.rb diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb index 0c9831eac..81e704e43 100644 --- a/app/models/card/searchable.rb +++ b/app/models/card/searchable.rb @@ -5,37 +5,26 @@ module Card::Searchable include ::Searchable scope :mentioning, ->(query, user:) do - query = Search::Query.wrap(query) + search_records = Search::Record.for_account(user.account_id) - if query.valid? - table_name = Searchable.search_index_table_name(user.account_id) - uuid_type = ActiveRecord::Type.lookup(:uuid, adapter: :trilogy) - serialized_board_ids = user.board_ids.map { |id| uuid_type.serialize(id) } - - joins("INNER JOIN #{table_name} ON #{table_name}.card_id = cards.id AND #{table_name}.board_id = cards.board_id") - .where("#{table_name}.board_id IN (?)", serialized_board_ids) - .where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", query.to_s) - .distinct - else - none - end + joins(search_records.card_join) + .merge(search_records.for_query(query: Search::Query.wrap(query), user: user)) end end - private - def search_title - Search::Stemmer.stem title - end + def search_title + title + end - def search_content - Search::Stemmer.stem description.to_plain_text - end + def search_content + description.to_plain_text + end - def search_card_id - id - end + def search_card_id + id + end - def search_board_id - board_id - end + def search_board_id + board_id + end end diff --git a/app/models/comment/searchable.rb b/app/models/comment/searchable.rb index 4b8846106..626d2be64 100644 --- a/app/models/comment/searchable.rb +++ b/app/models/comment/searchable.rb @@ -5,20 +5,19 @@ module Comment::Searchable include ::Searchable end - private - def search_title - nil - end + def search_title + nil + end - def search_content - Search::Stemmer.stem body.to_plain_text - end + def search_content + body.to_plain_text + end - def search_card_id - card_id - end + def search_card_id + card_id + end - def search_board_id - card.board_id - end + def search_board_id + card.board_id + end end diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 1184131b3..fa080df1d 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -1,10 +1,6 @@ module Searchable extend ActiveSupport::Concern - def self.search_index_table_name(account_id) - "search_index_#{Zlib.crc32(account_id) % 16}" - end - included do after_create_commit :create_in_search_index after_update_commit :update_in_search_index @@ -17,54 +13,40 @@ module Searchable private def create_in_search_index - table_name = Searchable.search_index_table_name(account_id) - uuid_type = ActiveRecord::Type.lookup(:uuid, adapter: :trilogy) - - self.class.connection.execute self.class.sanitize_sql([ - "INSERT INTO #{table_name} (id, searchable_type, searchable_id, card_id, board_id, title, content, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - uuid_type.serialize(ActiveRecord::Type::Uuid.generate), - self.class.name, - uuid_type.serialize(id), - uuid_type.serialize(search_card_id), - uuid_type.serialize(search_board_id), - search_title, - search_content, - created_at - ]) + Search::Record.for_account(account_id).create!(search_record_attributes) end def update_in_search_index - table_name = Searchable.search_index_table_name(account_id) - uuid_type = ActiveRecord::Type.lookup(:uuid, adapter: :trilogy) - - result = self.class.connection.execute(self.class.sanitize_sql([ - "UPDATE #{table_name} SET card_id = ?, board_id = ?, title = ?, content = ?, created_at = ? WHERE searchable_type = ? AND searchable_id = ?", - uuid_type.serialize(search_card_id), - uuid_type.serialize(search_board_id), - search_title, - search_content, - created_at, - self.class.name, - uuid_type.serialize(id) - ])) - - create_in_search_index if result.affected_rows == 0 + Search::Record.for_account(account_id).upsert_all( + [search_record_attributes.merge(id: ActiveRecord::Type::Uuid.generate)], + update_only: [:card_id, :board_id, :title, :content, :created_at] + ) end def remove_from_search_index - table_name = Searchable.search_index_table_name(account_id) - uuid_type = ActiveRecord::Type.lookup(:uuid, adapter: :trilogy) - - self.class.connection.execute self.class.sanitize_sql([ - "DELETE FROM #{table_name} WHERE searchable_type = ? AND searchable_id = ?", - self.class.name, - uuid_type.serialize(id) - ]) + Search::Record.for_account(account_id).where( + searchable_type: self.class.name, + searchable_id: id + ).delete_all end - # Models must implement these methods: - # - search_title: returns title string or nil - # - search_content: returns content string - # - search_card_id: returns the card id (self.id for cards, card_id for comments) - # - search_board_id: returns the board id + def search_record_attributes + { + account_id: account_id, + searchable_type: self.class.name, + searchable_id: id, + card_id: search_card_id, + board_id: search_board_id, + title: Search::Stemmer.stem(search_title), + content: Search::Stemmer.stem(search_content), + created_at: created_at + } + end + + # Models must implement these methods: + # - account_id: returns the account id + # - search_title: returns title string or nil + # - search_content: returns content string + # - search_card_id: returns the card id (self.id for cards, card_id for comments) + # - search_board_id: returns the board id end diff --git a/app/models/search.rb b/app/models/search.rb index b9bae9393..264194f03 100644 --- a/app/models/search.rb +++ b/app/models/search.rb @@ -1,49 +1,9 @@ -class Search - attr_reader :user, :query - +module Search def self.table_name_prefix "search_" end - def initialize(user, query) - @user = user - @query = Query.wrap(query) + def self.results(query:, user:) + Record.for_account(user.account_id).search(query: Query.wrap(query), user: user) end - - def results - if query.valid? && board_ids.any? - perform_search - else - Search::Result.none - end - end - - private - def board_ids - @board_ids ||= user.board_ids - end - - def perform_search - query_string = query.to_s - sanitized_raw_query = Search::Result.connection.quote(query.terms) - table_name = Searchable.search_index_table_name(user.account_id) - uuid_type = ActiveRecord::Type.lookup(:uuid, adapter: :trilogy) - serialized_board_ids = board_ids.map { |id| uuid_type.serialize(id) } - - Search::Result.from(table_name) - .joins("INNER JOIN cards ON #{table_name}.card_id = cards.id") - .joins("INNER JOIN boards ON cards.board_id = boards.id") - .where("#{table_name}.board_id IN (?)", serialized_board_ids) - .where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", query_string) - .select([ - "#{table_name}.card_id as card_id", - "CASE WHEN #{table_name}.searchable_type = 'Comment' THEN #{table_name}.searchable_id ELSE NULL END as comment_id", - "boards.name as board_name", - "cards.creator_id", - "cards.number as number", - "#{table_name}.created_at as created_at", - "#{sanitized_raw_query} AS query" - ].join(",")) - .order("#{table_name}.created_at DESC") - end end diff --git a/app/models/search/record.rb b/app/models/search/record.rb new file mode 100644 index 000000000..0ae85c224 --- /dev/null +++ b/app/models/search/record.rb @@ -0,0 +1,90 @@ +class Search::Record < ApplicationRecord + self.abstract_class = true + + SHARD_COUNT = 16 + + SHARD_CLASSES = SHARD_COUNT.times.map do |shard_id| + Class.new(self) do + self.table_name = "search_records_#{shard_id}" + + def self.name + "Search::Record" + end + end + end.freeze + + belongs_to :searchable, polymorphic: true + belongs_to :card + + # Virtual attributes from search query + attribute :query, :string + + validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true + + class << self + def for_account(account_id) + SHARD_CLASSES[shard_id_for_account(account_id)] + end + + def shard_id_for_account(account_id) + Zlib.crc32(account_id.to_s) % SHARD_COUNT + end + + def card_join + "INNER JOIN #{table_name} ON #{table_name}.card_id = cards.id" + end + end + + scope :for_query, ->(query:, user:) do + if query.valid? && user.board_ids.any? + matching(query.to_s).for_user(user) + else + none + end + end + + scope :matching, ->(query) do + where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", query) + end + + scope :for_user, ->(user) do + where(account_id: user.account_id, board_id: user.board_ids) + end + + scope :search, ->(query:, user:) do + for_query(query: query, user: user) + .includes(:searchable, card: [:board, :creator]) + .select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query") + .order(created_at: :desc) + end + + def source + searchable_type == "Comment" ? searchable : card + end + + def comment + searchable if searchable_type == "Comment" + end + + def card_title + highlight(card.title, show: :full) if card_id + end + + def card_description + highlight(card.description.to_plain_text, show: :snippet) if card_id + end + + def comment_body + highlight(comment.body.to_plain_text, show: :snippet) if comment + end + + private + def highlight(text, show:) + if text.present? && attribute?(:query) + highlighter = Search::Highlighter.new(query) + show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) + else + text + end + end +end diff --git a/app/models/user/searcher.rb b/app/models/user/searcher.rb index 8c3489bd0..951717082 100644 --- a/app/models/user/searcher.rb +++ b/app/models/user/searcher.rb @@ -6,7 +6,7 @@ module User::Searcher end def search(terms) - Search.new(self, terms).results + Search.results(query: terms, user: self) end def remember_search(terms) diff --git a/app/views/searches/_result.html.erb b/app/views/searches/_result.html.erb index 3a7feda6d..9463e9802 100644 --- a/app/views/searches/_result.html.erb +++ b/app/views/searches/_result.html.erb @@ -2,7 +2,7 @@ <%= link_to result.source, class: "search__result", data: { turbo_frame: "_top", action: "bar#reset" } do %>

- # <%= result.number %> <%= result.card_title %> + # <%= result.card.number %> <%= result.card_title %>

<% if result.card_description.present? %> @@ -12,7 +12,7 @@ <% else if result.comment_body.present? %>
- <%= avatar_preview_tag result.creator %> + <%= avatar_preview_tag result.card.creator %>
<%= result.comment_body %>
@@ -20,6 +20,6 @@ <% end %> <% end %>
-
<%= result.board_name %>
+
<%= result.card.board.name %>
<% end %> diff --git a/db/migrate/20251113190256_create_search_record_shards.rb b/db/migrate/20251113190256_create_search_record_shards.rb new file mode 100644 index 000000000..5b6ee89ce --- /dev/null +++ b/db/migrate/20251113190256_create_search_record_shards.rb @@ -0,0 +1,34 @@ +class CreateSearchRecordShards < ActiveRecord::Migration[8.2] + SHARD_COUNT = 16 + + def up + # Create 16 sharded search_records tables + SHARD_COUNT.times do |shard_id| + create_table "search_records_#{shard_id}", id: :uuid do |t| + t.uuid :account_id, null: false + t.string :searchable_type, null: false + t.uuid :searchable_id, null: false + t.uuid :card_id, null: false + t.uuid :board_id, null: false + t.string :title + t.text :content + t.datetime :created_at, null: false + + t.index [:searchable_type, :searchable_id], unique: true + t.index :account_id + t.index [:content, :title], type: :fulltext + end + end + + # Drop old search_index tables + SHARD_COUNT.times do |shard_id| + drop_table "search_index_#{shard_id}", if_exists: true + end + end + + def down + SHARD_COUNT.times do |shard_id| + drop_table "search_records_#{shard_id}" + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 144b2ce33..8b553cca6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2025_11_13_163145) do +ActiveRecord::Schema[8.2].define(version: 2025_11_13_190256) do create_table "accesses", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false @@ -396,198 +396,6 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_13_163145) do t.index ["reacter_id"], name: "index_reactions_on_reacter_id" end - create_table "search_index_0", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si0_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si0_type_id", unique: true - end - - create_table "search_index_1", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si1_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si1_type_id", unique: true - end - - create_table "search_index_10", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si10_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si10_type_id", unique: true - end - - create_table "search_index_11", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si11_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si11_type_id", unique: true - end - - create_table "search_index_12", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si12_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si12_type_id", unique: true - end - - create_table "search_index_13", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si13_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si13_type_id", unique: true - end - - create_table "search_index_14", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si14_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si14_type_id", unique: true - end - - create_table "search_index_15", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si15_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si15_type_id", unique: true - end - - create_table "search_index_2", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si2_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si2_type_id", unique: true - end - - create_table "search_index_3", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si3_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si3_type_id", unique: true - end - - create_table "search_index_4", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si4_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si4_type_id", unique: true - end - - create_table "search_index_5", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si5_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si5_type_id", unique: true - end - - create_table "search_index_6", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si6_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si6_type_id", unique: true - end - - create_table "search_index_7", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si7_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si7_type_id", unique: true - end - - create_table "search_index_8", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si8_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si8_type_id", unique: true - end - - create_table "search_index_9", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["content", "title"], name: "idx_si9_fulltext", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "idx_si9_type_id", unique: true - end - create_table "search_queries", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.uuid "account_id", null: false t.datetime "created_at", null: false @@ -600,6 +408,230 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_13_163145) do t.index ["user_id"], name: "index_search_queries_on_user_id" end + create_table "search_records_0", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_0_on_account_id" + t.index ["content", "title"], name: "index_search_records_0_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_0_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_1", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_1_on_account_id" + t.index ["content", "title"], name: "index_search_records_1_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_1_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_10", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_10_on_account_id" + t.index ["content", "title"], name: "index_search_records_10_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_10_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_11", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_11_on_account_id" + t.index ["content", "title"], name: "index_search_records_11_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_11_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_12", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_12_on_account_id" + t.index ["content", "title"], name: "index_search_records_12_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_12_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_13", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_13_on_account_id" + t.index ["content", "title"], name: "index_search_records_13_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_13_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_14", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_14_on_account_id" + t.index ["content", "title"], name: "index_search_records_14_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_14_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_15", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_15_on_account_id" + t.index ["content", "title"], name: "index_search_records_15_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_15_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_2", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_2_on_account_id" + t.index ["content", "title"], name: "index_search_records_2_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_2_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_3", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_3_on_account_id" + t.index ["content", "title"], name: "index_search_records_3_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_3_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_4", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_4_on_account_id" + t.index ["content", "title"], name: "index_search_records_4_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_4_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_5", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_5_on_account_id" + t.index ["content", "title"], name: "index_search_records_5_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_5_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_6", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_6_on_account_id" + t.index ["content", "title"], name: "index_search_records_6_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_6_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_7", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_7_on_account_id" + t.index ["content", "title"], name: "index_search_records_7_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_7_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_8", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_8_on_account_id" + t.index ["content", "title"], name: "index_search_records_8_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_8_on_searchable_type_and_searchable_id", unique: true + end + + create_table "search_records_9", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_9_on_account_id" + t.index ["content", "title"], name: "index_search_records_9_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_9_on_searchable_type_and_searchable_id", unique: true + end + create_table "search_results", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false diff --git a/lib/tasks/search.rake b/lib/tasks/search.rake index 94d4051ab..0bb3591ee 100644 --- a/lib/tasks/search.rake +++ b/lib/tasks/search.rake @@ -1,9 +1,9 @@ namespace :search do desc "Reindex all cards and comments in the search index" task reindex: :environment do - puts "Clearing search index shards..." - 16.times do |i| - ActiveRecord::Base.connection.execute("DELETE FROM search_index_#{i}") + puts "Clearing search records..." + Search::Record::SHARD_COUNT.times do |shard_id| + ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") end puts "Reindexing cards..." diff --git a/test/controllers/searches_controller_test.rb b/test/controllers/searches_controller_test.rb index 5e1e1ccb4..0cc0bceee 100644 --- a/test/controllers/searches_controller_test.rb +++ b/test/controllers/searches_controller_test.rb @@ -1,31 +1,32 @@ require "test_helper" class SearchesControllerTest < ActionDispatch::IntegrationTest + include SearchTestHelper + setup do - Card.all.each(&:reindex) - Comment.all.each(&:reindex) + @board.update!(all_access: true) + @card = @board.cards.create!(title: "Layout is broken", creator: @user) + @comment_card = @board.cards.create!(title: "Some card", creator: @user) + @comment_card.comments.create!(body: "overflowing text issue", creator: @user) - sign_in_as :kevin + untenanted { sign_in_as @user } end - test "show" do - skip("TODO:PLANB: search") - get search_path(q: "broken") - + test "search" do + # Searching by card title + get search_path(q: "broken", script_name: "/#{@account.external_account_id}") assert_select "li", text: /Layout is broken/ - end - test "show with card id" do - skip("TODO:PLANB: search") - get search_path(q: cards(:logo).id) + # Searching by comment + get search_path(q: "overflowing", script_name: "/#{@account.external_account_id}") + assert_select "li", text: /Some card/ + # Searching by card id + get search_path(q: @card.id, script_name: "/#{@account.external_account_id}") assert_select "form[data-controller='auto-submit']" - end - - test "show with non-existent card id" do - skip("TODO:PLANB: search") - get search_path(q: "999999") + # Searching with non-existent card id + get search_path(q: "999999", script_name: "/#{@account.external_account_id}") assert_select "form[data-controller='auto-submit']", count: 0 assert_select ".search__empty", text: "No matches" end diff --git a/test/models/card/searchable_test.rb b/test/models/card/searchable_test.rb index 1c65606c0..18845ab7a 100644 --- a/test/models/card/searchable_test.rb +++ b/test/models/card/searchable_test.rb @@ -1,22 +1,7 @@ require "test_helper" class Card::SearchableTest < ActiveSupport::TestCase - self.use_transactional_tests = false - - setup do - 16.times { |i| ActiveRecord::Base.connection.execute "DELETE FROM search_index_#{i}" } - Account.find_by(name: "Search Test")&.destroy - - @account = Account.create!(name: "Search Test") - @user = User.create!(name: "Test User", account: @account) - @board = Board.create!(name: "Test Board", account: @account, creator: @user) - Current.account = @account - end - - teardown do - 16.times { |i| ActiveRecord::Base.connection.execute "DELETE FROM search_index_#{i}" } - Account.find_by(name: "Search Test")&.destroy - end + include SearchTestHelper test "card search" do # Searching by title diff --git a/test/models/comment/searchable_test.rb b/test/models/comment/searchable_test.rb index 36528f1e7..a8e52bfc2 100644 --- a/test/models/comment/searchable_test.rb +++ b/test/models/comment/searchable_test.rb @@ -1,58 +1,28 @@ require "test_helper" class Comment::SearchableTest < ActiveSupport::TestCase - self.use_transactional_tests = false + include SearchTestHelper setup do - 16.times { |i| ActiveRecord::Base.connection.execute "DELETE FROM search_index_#{i}" } - Account.find_by(name: "Search Test")&.destroy - - @account = Account.create!(name: "Search Test") - @user = User.create!(name: "Test User", account: @account) - @board = Board.create!(name: "Test Board", account: @account, creator: @user) @card = @board.cards.create!(title: "Test Card", creator: @user) - Current.account = @account - end - - teardown do - 16.times { |i| ActiveRecord::Base.connection.execute "DELETE FROM search_index_#{i}" } - Account.find_by(name: "Search Test")&.destroy end test "comment search" do - table_name = Searchable.search_index_table_name(@account.id) - uuid_type = ActiveRecord::Type.lookup(:uuid, adapter: :trilogy) - # Comment is indexed on create comment = @card.comments.create!(body: "searchable comment text", creator: @user) - result = ActiveRecord::Base.connection.execute( - ActiveRecord::Base.sanitize_sql([ - "SELECT COUNT(*) FROM #{table_name} WHERE searchable_type = 'Comment' AND searchable_id = ?", - uuid_type.serialize(comment.id) - ]) - ).first[0] - assert_equal 1, result + record = Search::Record.for_account(@account.id).find_by(searchable_type: "Comment", searchable_id: comment.id) + assert_not_nil record # Comment is updated in index comment.update!(body: "updated text") - content = ActiveRecord::Base.connection.execute( - ActiveRecord::Base.sanitize_sql([ - "SELECT content FROM #{table_name} WHERE searchable_type = 'Comment' AND searchable_id = ?", - uuid_type.serialize(comment.id) - ]) - ).first[0] - assert_match /updat/, content + record = Search::Record.for_account(@account.id).find_by(searchable_type: "Comment", searchable_id: comment.id) + assert_match /updat/, record.content # Comment is removed from index on destroy comment_id = comment.id comment.destroy - result = ActiveRecord::Base.connection.execute( - ActiveRecord::Base.sanitize_sql([ - "SELECT COUNT(*) FROM #{table_name} WHERE searchable_type = 'Comment' AND searchable_id = ?", - uuid_type.serialize(comment_id) - ]) - ).first[0] - assert_equal 0, result + record = Search::Record.for_account(@account.id).find_by(searchable_type: "Comment", searchable_id: comment_id) + assert_nil record # Finding cards via comment search card_with_comment = @board.cards.create!(title: "Card One", creator: @user) @@ -64,14 +34,8 @@ class Comment::SearchableTest < ActiveSupport::TestCase # Comment stores parent card_id and board_id new_comment = @card.comments.create!(body: "test comment", creator: @user) - row = ActiveRecord::Base.connection.execute( - ActiveRecord::Base.sanitize_sql([ - "SELECT card_id, board_id FROM #{table_name} WHERE searchable_type = 'Comment' AND searchable_id = ?", - uuid_type.serialize(new_comment.id) - ]) - ).first - # Deserialize binary UUIDs from result - assert_equal @card.id, uuid_type.deserialize(row[0]) - assert_equal @board.id, uuid_type.deserialize(row[1]) + record = Search::Record.for_account(@account.id).find_by(searchable_type: "Comment", searchable_id: new_comment.id) + assert_equal @card.id, record.card_id + assert_equal @board.id, record.board_id end end diff --git a/test/models/search_test.rb b/test/models/search_test.rb index d17c721e7..e27bcbdfe 100644 --- a/test/models/search_test.rb +++ b/test/models/search_test.rb @@ -1,22 +1,7 @@ require "test_helper" class SearchTest < ActiveSupport::TestCase - self.use_transactional_tests = false - - setup do - 16.times { |i| ActiveRecord::Base.connection.execute "DELETE FROM search_index_#{i}" } - Account.find_by(name: "Search Test")&.destroy - - @account = Account.create!(name: "Search Test") - @user = User.create!(name: "Test User", account: @account) - @board = Board.create!(name: "Test Board", account: @account, creator: @user) - Current.account = @account - end - - teardown do - 16.times { |i| ActiveRecord::Base.connection.execute "DELETE FROM search_index_#{i}" } - Account.find_by(name: "Search Test")&.destroy - end + include SearchTestHelper test "search" do # Search cards and comments @@ -24,11 +9,11 @@ class SearchTest < ActiveSupport::TestCase comment_card = @board.cards.create!(title: "Some card", creator: @user) comment_card.comments.create!(body: "overflowing text", creator: @user) - results = Search.new(@user, "layout").results + results = Search.results(query: "layout", user: @user) assert results.find { |it| it.card_id == card.id } - results = Search.new(@user, "overflowing").results - assert results.find { |it| it.card_id == comment_card.id && it.comment_id.present? } + results = Search.results(query: "overflowing", user: @user) + assert results.find { |it| it.card_id == comment_card.id && it.searchable_type == "Comment" } # Don't include inaccessible boards other_user = User.create!(name: "Other User", account: @account) @@ -36,13 +21,13 @@ class SearchTest < ActiveSupport::TestCase accessible_card = @board.cards.create!(title: "searchable content", creator: @user) inaccessible_card = inaccessible_board.cards.create!(title: "searchable content", creator: other_user) - results = Search.new(@user, "searchable").results + results = Search.results(query: "searchable", user: @user) assert results.find { |it| it.card_id == accessible_card.id } assert_not results.find { |it| it.card_id == inaccessible_card.id } # Empty board_ids returns no results user_without_access = User.create!(name: "No Access User", account: @account) - results = Search.new(user_without_access, "anything").results + results = Search.results(query: "anything", user: user_without_access) assert_empty results end end diff --git a/test/test_helpers/search_test_helper.rb b/test/test_helpers/search_test_helper.rb new file mode 100644 index 000000000..97d050157 --- /dev/null +++ b/test/test_helpers/search_test_helper.rb @@ -0,0 +1,36 @@ +module SearchTestHelper + extend ActiveSupport::Concern + + included do + self.use_transactional_tests = false + + setup :setup_search_test + teardown :teardown_search_test + end + + def setup_search_test + clear_search_records + Account.find_by(name: "Search Test")&.destroy + Identity.find_by(email_address: "test@example.com")&.destroy + + @account = Account.create!(name: "Search Test", external_account_id: ActiveRecord::FixtureSet.identify("search_test")) + @identity = Identity.create!(email_address: "test@example.com") + @membership = Membership.create!(identity: @identity, tenant: @account.external_account_id) + @user = User.create!(name: "Test User", account: @account, membership: @membership) + @board = Board.create!(name: "Test Board", account: @account, creator: @user) + Current.account = @account + end + + def teardown_search_test + clear_search_records + Account.find_by(name: "Search Test")&.destroy + Identity.find_by(email_address: "test@example.com")&.destroy + end + + private + def clear_search_records + Search::Record::SHARD_COUNT.times do |shard_id| + ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") + end + end +end