From e8abc66ebac27742457cc267ae58a4d7d970d960 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 12 Nov 2025 10:09:34 +0000 Subject: [PATCH] MySQL search Add a single search_index table for full-text search of cards and comments. For the search there is a full-text index on the title and content columns. The board_ids is also included in the table and accessible board ids are pre-loaded and included in the search query. This allows us to filter out inaccessible records before joining with other tables. Right now the search is just using boolean search. This would give us a bunch of syntax options (see https://dev.mysql.com/doc/refman/8.4/en/fulltext-boolean.html) except the search query filters those out. I've removed the searchable_by method for now - everything is built on the assumption that there's a single search index table and all data must fit into it. Queries are written in SQL, we don't have a SearchIndex ActiveRecord model. That's because we'll likely want to shard the table and it will be simpler to just keep with the raw SQL for that. There's no stemming, highlighting or snippet extraction yet - we are dumping the full description in the search results. Data can be reindexed with the search:reindex rake task. --- app/models/card/searchable.rb | 19 +++- app/models/comment.rb | 1 - app/models/comment/searchable.rb | 24 +++++ app/models/concerns/searchable.rb | 96 ++++++++----------- app/models/search.rb | 61 ++++++------ .../20251112093037_create_search_index.rb | 20 ++++ db/schema.rb | 14 ++- lib/tasks/search.rake | 19 ++++ 8 files changed, 165 insertions(+), 89 deletions(-) create mode 100644 app/models/comment/searchable.rb create mode 100644 db/migrate/20251112093037_create_search_index.rb create mode 100644 lib/tasks/search.rake diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb index e0c3ced20..b115db1ea 100644 --- a/app/models/card/searchable.rb +++ b/app/models/card/searchable.rb @@ -4,8 +4,6 @@ module Card::Searchable included do include ::Searchable - searchable_by :title, :description, using: :cards_search_index - scope :mentioning, ->(query) do cards = Card.search(query).select(:id).to_sql comments = Comment.search(query).select(:id).to_sql @@ -15,8 +13,19 @@ module Card::Searchable end private - # TODO: Temporary until we stabilize the search API - def title_and_description - [ title, description.to_plain_text ].join(" ") + def search_title + title + end + + def search_content + description.to_plain_text + end + + def search_card_id + id + end + + def search_board_id + board_id end end diff --git a/app/models/comment.rb b/app/models/comment.rb index d67ef430b..b5e84d692 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -7,7 +7,6 @@ class Comment < ApplicationRecord has_many :reactions, dependent: :delete_all has_rich_text :body - searchable_by :body, using: :comments_search_index scope :chronologically, -> { order created_at: :asc, id: :desc } scope :by_system, -> { joins(:creator).where(creator: { role: "system" }) } diff --git a/app/models/comment/searchable.rb b/app/models/comment/searchable.rb new file mode 100644 index 000000000..92fdde7f5 --- /dev/null +++ b/app/models/comment/searchable.rb @@ -0,0 +1,24 @@ +module Comment::Searchable + extend ActiveSupport::Concern + + included do + include ::Searchable + end + + private + def search_title + nil + end + + def search_content + body.to_plain_text + end + + def search_card_id + card_id + end + + def search_board_id + card.board_id + end +end diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 10a88223d..c94be9193 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -1,37 +1,20 @@ module Searchable extend ActiveSupport::Concern - class_methods do - def searchable_by(*fields, using:) - define_method :search_fields do - fields - end + included do + after_create_commit :create_in_search_index + after_update_commit :update_in_search_index + after_destroy_commit :remove_from_search_index - define_method :search_values do - fields.map do - value = send(it) - value.respond_to?(:to_plain_text) ? value.to_plain_text : value - end - end + scope :search, ->(query) do + query = Search::Query.wrap(query) - define_method :search_table do - using - end + base = joins("INNER JOIN search_index ON search_index.searchable_type = '#{name}' AND search_index.searchable_id = #{table_name}.id") - after_create_commit :create_in_search_index - after_update_commit :update_in_search_index - after_destroy_commit :remove_from_search_index - - scope :search, ->(query) do - query = Search::Query.wrap(query) - - base = joins("join #{using} idx on #{table_name}.id = idx.rowid") - - if query.valid? - base.where("#{using} match ?", query.to_s) - else - base.none - end + if query.valid? + base.where("MATCH(search_index.content, search_index.title) AGAINST(? IN BOOLEAN MODE)", query.to_s) + else + base.none end end end @@ -42,39 +25,44 @@ module Searchable private def create_in_search_index - # # TODO:PLANB: need to replace SQLite FTS - # fields_sql = [ "rowid", *search_fields ].join(", ") - # placeholders = ([ "?" ] * (search_fields.size + 1)).join(", ") - # values = [ id, *search_values ] - - # execute_sql_with_binds( - # "insert into #{search_table}(#{fields_sql}) values (#{placeholders})", - # *values - # ) + self.class.connection.execute self.class.sanitize_sql([ + "INSERT INTO search_index (searchable_type, searchable_id, card_id, board_id, title, content, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + self.class.name, + id, + search_card_id, + search_board_id, + search_title, + search_content, + created_at + ]) end def update_in_search_index - # # TODO:PLANB: need to replace SQLite FTS - # transaction do - # set_clause = search_fields.map { |field| "#{field} = ?" }.join(", ") - # binds = search_values + [ id ] + result = self.class.connection.execute(self.class.sanitize_sql([ + "UPDATE search_index SET card_id = ?, board_id = ?, title = ?, content = ?, created_at = ? WHERE searchable_type = ? AND searchable_id = ?", + search_card_id, + search_board_id, + search_title, + search_content, + created_at, + self.class.name, + id + ])) - # updated = execute_sql_with_binds( - # "update #{search_table} set #{set_clause} where rowid = ?", - # *binds - # ) - - # create_in_search_index unless updated - # end + create_in_search_index if result.affected_rows == 0 end def remove_from_search_index - # # TODO:PLANB: need to replace SQLite FTS - # execute_sql_with_binds "delete from #{search_table} where rowid = ?", id + self.class.connection.execute self.class.sanitize_sql([ + "DELETE FROM search_index WHERE searchable_type = ? AND searchable_id = ?", + self.class.name, + id + ]) end - def execute_sql_with_binds(*statement) - self.class.connection.execute self.class.sanitize_sql(statement) - self.class.connection.raw_connection.changes.nonzero? - end + # Models must implement these methods: + # - search_title: returns title string or nil + # - search_content: returns content string + # - search_card_id: returns the card id (self.id for cards, card_id for comments) + # - search_board_id: returns the board id end diff --git a/app/models/search.rb b/app/models/search.rb index 518462094..fd7ca072e 100644 --- a/app/models/search.rb +++ b/app/models/search.rb @@ -14,33 +14,38 @@ class Search end def results - cards = user.accessible_cards.search(query) - .select([ - "cards.id as card_id", - "null as comment_id", - "highlight(cards_search_index, 0, '#{HIGHLIGHT_OPENING_MARK}', '#{HIGHLIGHT_CLOSING_MARK}') AS card_title", - "snippet(cards_search_index, 1, '#{HIGHLIGHT_OPENING_MARK}', '#{HIGHLIGHT_CLOSING_MARK}', '...', 20) AS card_description", - "null as comment_body", - "boards.name as board_name", - "cards.creator_id", - "cards.created_at", - "bm25(cards_search_index, 10.0, 2.0) AS score" - ].join(",")) - - comments = user.accessible_comments.search(query) - .select([ - "comments.card_id as card_id", - "comments.id as comment_id", - "cards.title AS card_title", - "null AS card_description", - "snippet(comments_search_index, 0, '#{HIGHLIGHT_OPENING_MARK}', '#{HIGHLIGHT_CLOSING_MARK}', '...', 20) AS comment_body", - "boards.name as board_name", - "comments.creator_id", - "comments.created_at", - "bm25(comments_search_index, 1.0) AS score" - ].join(",")) - - union_sql = "(#{cards.to_sql} UNION #{comments.to_sql}) as search_results" - Search::Result.from(union_sql).order(created_at: :desc) + if query.valid? && board_ids.any? + perform_search + else + Search::Result.none + end end + + private + def board_ids + @board_ids ||= user.board_ids + end + + def perform_search + query_string = query.to_s + sanitized_query = Search::Result.connection.quote(query_string) + + Search::Result.from("search_index") + .joins("INNER JOIN cards ON search_index.card_id = cards.id") + .joins("INNER JOIN boards ON cards.board_id = boards.id") + .where("search_index.board_id IN (?)", board_ids) + .where("MATCH(search_index.content, search_index.title) AGAINST(? IN BOOLEAN MODE)", query_string) + .select([ + "search_index.card_id as card_id", + "CASE WHEN search_index.searchable_type = 'Comment' THEN search_index.searchable_id ELSE NULL END as comment_id", + "COALESCE(search_index.title, cards.title) AS card_title_in_database", + "CASE WHEN search_index.searchable_type = 'Card' THEN search_index.content ELSE NULL END AS card_description_in_database", + "CASE WHEN search_index.searchable_type = 'Comment' THEN search_index.content ELSE NULL END AS comment_body_in_database", + "boards.name as board_name", + "cards.creator_id", + "search_index.created_at as created_at", + "MATCH(search_index.content, search_index.title) AGAINST(#{sanitized_query} IN BOOLEAN MODE) AS score" + ].join(",")) + .order("search_index.created_at DESC") + end end diff --git a/db/migrate/20251112093037_create_search_index.rb b/db/migrate/20251112093037_create_search_index.rb new file mode 100644 index 000000000..4b524b76e --- /dev/null +++ b/db/migrate/20251112093037_create_search_index.rb @@ -0,0 +1,20 @@ +class CreateSearchIndex < ActiveRecord::Migration[8.2] + def up + create_table :search_index do |t| + t.string :searchable_type, null: false + t.bigint :searchable_id, null: false + t.bigint :card_id, null: false + t.bigint :board_id, null: false + t.string :title + t.text :content + t.datetime :created_at, null: false + + t.index [:searchable_type, :searchable_id], unique: true + t.index [:content, :title], type: :fulltext, name: "index_search_index_on_content_and_title" + end + end + + def down + drop_table :search_index + end +end diff --git a/db/schema.rb b/db/schema.rb index 205a0c5ad..3d25b0b65 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2025_11_11_153019) do +ActiveRecord::Schema[8.2].define(version: 2025_11_12_093037) do create_table "accesses", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.datetime "accessed_at" t.bigint "board_id", null: false @@ -375,6 +375,18 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_11_153019) do t.index ["reacter_id"], name: "index_reactions_on_reacter_id" end + create_table "search_index", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.bigint "board_id", null: false + t.bigint "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.bigint "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["content", "title"], name: "index_search_index_on_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_index_on_searchable_type_and_searchable_id", unique: true + end + create_table "search_queries", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.datetime "created_at", null: false t.string "terms", limit: 2000, null: false diff --git a/lib/tasks/search.rake b/lib/tasks/search.rake new file mode 100644 index 000000000..c414f6dc2 --- /dev/null +++ b/lib/tasks/search.rake @@ -0,0 +1,19 @@ +namespace :search do + desc "Reindex all cards and comments in the search index" + task reindex: :environment do + puts "Clearing search index..." + ActiveRecord::Base.connection.execute("DELETE FROM search_index") + + puts "Reindexing cards..." + Card.find_each do |card| + card.reindex + end + + puts "Reindexing comments..." + Comment.find_each do |comment| + comment.reindex + end + + puts "Done! Reindexed #{Card.count} cards and #{Comment.count} comments." + end +end