MySQL search

Add a single search_index table for full-text search of cards and
comments.

For the search there is a full-text index on the title and content
columns. The board_ids is also included in the table and accessible
board ids are pre-loaded and included in the search query. This allows
us to filter out inaccessible records before joining with other tables.

Right now the search is just using boolean search. This would give us
a bunch of syntax options
(see https://dev.mysql.com/doc/refman/8.4/en/fulltext-boolean.html)
except the search query filters those out.

I've removed the searchable_by method for now - everything is built
on the assumption that there's a single search index table and all data
must fit into it.

Queries are written in SQL, we don't have a SearchIndex ActiveRecord
model. That's because we'll likely want to shard the table and it will
be simpler to just keep with the raw SQL for that.

There's no stemming, highlighting or snippet extraction yet - we are
dumping the full description in the search results.

Data can be reindexed with the search:reindex rake task.

app/models/card/searchable.rb

@@ -4,8 +4,6 @@ module Card::Searchable
   included do
     include ::Searchable
 
-    searchable_by :title, :description, using: :cards_search_index
-
     scope :mentioning, ->(query) do
       cards = Card.search(query).select(:id).to_sql
       comments = Comment.search(query).select(:id).to_sql
@@ -15,8 +13,19 @@ module Card::Searchable
   end
 
   private
-    # TODO: Temporary until we stabilize the search API
-    def title_and_description
-      [ title, description.to_plain_text ].join(" ")
+    def search_title
+      title
+    end
+
+    def search_content
+      description.to_plain_text
+    end
+
+    def search_card_id
+      id
+    end
+
+    def search_board_id
+      board_id
     end
 end

app/models/comment.rb

@@ -7,7 +7,6 @@ class Comment < ApplicationRecord
   has_many :reactions, dependent: :delete_all
 
   has_rich_text :body
-  searchable_by :body, using: :comments_search_index
 
   scope :chronologically, -> { order created_at: :asc, id: :desc }
   scope :by_system, -> { joins(:creator).where(creator: { role: "system" }) }

app/models/comment/searchable.rb

@@ -0,0 +1,24 @@
+module Comment::Searchable
+  extend ActiveSupport::Concern
+
+  included do
+    include ::Searchable
+  end
+
+  private
+    def search_title
+      nil
+    end
+
+    def search_content
+      body.to_plain_text
+    end
+
+    def search_card_id
+      card_id
+    end
+
+    def search_board_id
+      card.board_id
+    end
+end

app/models/concerns/searchable.rb

@@ -1,37 +1,20 @@
 module Searchable
   extend ActiveSupport::Concern
 
-  class_methods do
-    def searchable_by(*fields, using:)
-      define_method :search_fields do
-        fields
-      end
+  included do
+    after_create_commit :create_in_search_index
+    after_update_commit :update_in_search_index
+    after_destroy_commit :remove_from_search_index
 
-      define_method :search_values do
-        fields.map do
-          value = send(it)
-          value.respond_to?(:to_plain_text) ? value.to_plain_text : value
-        end
-      end
+    scope :search, ->(query) do
+      query = Search::Query.wrap(query)
 
-      define_method :search_table do
-        using
-      end
+      base = joins("INNER JOIN search_index ON search_index.searchable_type = '#{name}' AND search_index.searchable_id = #{table_name}.id")
 
-      after_create_commit :create_in_search_index
-      after_update_commit :update_in_search_index
-      after_destroy_commit :remove_from_search_index
-
-      scope :search, ->(query) do
-        query = Search::Query.wrap(query)
-
-        base = joins("join #{using} idx on #{table_name}.id = idx.rowid")
-
-        if query.valid?
-          base.where("#{using} match ?", query.to_s)
-        else
-          base.none
-        end
+      if query.valid?
+        base.where("MATCH(search_index.content, search_index.title) AGAINST(? IN BOOLEAN MODE)", query.to_s)
+      else
+        base.none
       end
     end
   end
@@ -42,39 +25,44 @@ module Searchable
 
   private
     def create_in_search_index
-      # # TODO:PLANB: need to replace SQLite FTS
-      # fields_sql = [ "rowid", *search_fields ].join(", ")
-      # placeholders = ([ "?" ] * (search_fields.size + 1)).join(", ")
-      # values = [ id, *search_values ]
-
-      # execute_sql_with_binds(
-      #   "insert into #{search_table}(#{fields_sql}) values (#{placeholders})",
-      #   *values
-      # )
+      self.class.connection.execute self.class.sanitize_sql([
+        "INSERT INTO search_index (searchable_type, searchable_id, card_id, board_id, title, content, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
+        self.class.name,
+        id,
+        search_card_id,
+        search_board_id,
+        search_title,
+        search_content,
+        created_at
+      ])
     end
 
     def update_in_search_index
-      # # TODO:PLANB: need to replace SQLite FTS
-      # transaction do
-      #   set_clause = search_fields.map { |field| "#{field} = ?" }.join(", ")
-      #   binds = search_values + [ id ]
+      result = self.class.connection.execute(self.class.sanitize_sql([
+        "UPDATE search_index SET card_id = ?, board_id = ?, title = ?, content = ?, created_at = ? WHERE searchable_type = ? AND searchable_id = ?",
+        search_card_id,
+        search_board_id,
+        search_title,
+        search_content,
+        created_at,
+        self.class.name,
+        id
+      ]))
 
-      #   updated = execute_sql_with_binds(
-      #     "update #{search_table} set #{set_clause} where rowid = ?",
-      #     *binds
-      #   )
-
-      #   create_in_search_index unless updated
-      # end
+      create_in_search_index if result.affected_rows == 0
     end
 
     def remove_from_search_index
-      # # TODO:PLANB: need to replace SQLite FTS
-      # execute_sql_with_binds "delete from #{search_table} where rowid = ?", id
+      self.class.connection.execute self.class.sanitize_sql([
+        "DELETE FROM search_index WHERE searchable_type = ? AND searchable_id = ?",
+        self.class.name,
+        id
+      ])
     end
 
-    def execute_sql_with_binds(*statement)
-      self.class.connection.execute self.class.sanitize_sql(statement)
-      self.class.connection.raw_connection.changes.nonzero?
-    end
+    # Models must implement these methods:
+    # - search_title: returns title string or nil
+    # - search_content: returns content string
+    # - search_card_id: returns the card id (self.id for cards, card_id for comments)
+    # - search_board_id: returns the board id
 end

app/models/search.rb

@@ -14,33 +14,38 @@ class Search
   end
 
   def results
-    cards = user.accessible_cards.search(query)
-      .select([
-        "cards.id as card_id",
-        "null as comment_id",
-        "highlight(cards_search_index, 0, '#{HIGHLIGHT_OPENING_MARK}', '#{HIGHLIGHT_CLOSING_MARK}') AS card_title",
-        "snippet(cards_search_index, 1, '#{HIGHLIGHT_OPENING_MARK}', '#{HIGHLIGHT_CLOSING_MARK}', '...', 20) AS card_description",
-        "null as comment_body",
-        "boards.name as board_name",
-        "cards.creator_id",
-        "cards.created_at",
-        "bm25(cards_search_index, 10.0, 2.0) AS score"
-      ].join(","))
-
-    comments = user.accessible_comments.search(query)
-      .select([
-        "comments.card_id as card_id",
-        "comments.id as comment_id",
-        "cards.title AS card_title",
-        "null AS card_description",
-        "snippet(comments_search_index, 0, '#{HIGHLIGHT_OPENING_MARK}', '#{HIGHLIGHT_CLOSING_MARK}', '...', 20) AS comment_body",
-        "boards.name as board_name",
-        "comments.creator_id",
-        "comments.created_at",
-        "bm25(comments_search_index, 1.0) AS score"
-      ].join(","))
-
-    union_sql = "(#{cards.to_sql} UNION #{comments.to_sql}) as search_results"
-    Search::Result.from(union_sql).order(created_at: :desc)
+    if query.valid? && board_ids.any?
+      perform_search
+    else
+      Search::Result.none
+    end
   end
+
+  private
+    def board_ids
+      @board_ids ||= user.board_ids
+    end
+
+    def perform_search
+      query_string = query.to_s
+      sanitized_query = Search::Result.connection.quote(query_string)
+
+      Search::Result.from("search_index")
+        .joins("INNER JOIN cards ON search_index.card_id = cards.id")
+        .joins("INNER JOIN boards ON cards.board_id = boards.id")
+        .where("search_index.board_id IN (?)", board_ids)
+        .where("MATCH(search_index.content, search_index.title) AGAINST(? IN BOOLEAN MODE)", query_string)
+        .select([
+          "search_index.card_id as card_id",
+          "CASE WHEN search_index.searchable_type = 'Comment' THEN search_index.searchable_id ELSE NULL END as comment_id",
+          "COALESCE(search_index.title, cards.title) AS card_title_in_database",
+          "CASE WHEN search_index.searchable_type = 'Card' THEN search_index.content ELSE NULL END AS card_description_in_database",
+          "CASE WHEN search_index.searchable_type = 'Comment' THEN search_index.content ELSE NULL END AS comment_body_in_database",
+          "boards.name as board_name",
+          "cards.creator_id",
+          "search_index.created_at as created_at",
+          "MATCH(search_index.content, search_index.title) AGAINST(#{sanitized_query} IN BOOLEAN MODE) AS score"
+        ].join(","))
+        .order("search_index.created_at DESC")
+    end
 end

db/migrate/20251112093037_create_search_index.rb

@@ -0,0 +1,20 @@
+class CreateSearchIndex < ActiveRecord::Migration[8.2]
+  def up
+    create_table :search_index do |t|
+      t.string :searchable_type, null: false
+      t.bigint :searchable_id, null: false
+      t.bigint :card_id, null: false
+      t.bigint :board_id, null: false
+      t.string :title
+      t.text :content
+      t.datetime :created_at, null: false
+
+      t.index [:searchable_type, :searchable_id], unique: true
+      t.index [:content, :title], type: :fulltext, name: "index_search_index_on_content_and_title"
+    end
+  end
+
+  def down
+    drop_table :search_index
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.2].define(version: 2025_11_11_153019) do
+ActiveRecord::Schema[8.2].define(version: 2025_11_12_093037) do
   create_table "accesses", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
     t.datetime "accessed_at"
     t.bigint "board_id", null: false
@@ -375,6 +375,18 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_11_153019) do
     t.index ["reacter_id"], name: "index_reactions_on_reacter_id"
   end
 
+  create_table "search_index", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
+    t.bigint "board_id", null: false
+    t.bigint "card_id", null: false
+    t.text "content"
+    t.datetime "created_at", null: false
+    t.bigint "searchable_id", null: false
+    t.string "searchable_type", null: false
+    t.string "title"
+    t.index ["content", "title"], name: "index_search_index_on_content_and_title", type: :fulltext
+    t.index ["searchable_type", "searchable_id"], name: "index_search_index_on_searchable_type_and_searchable_id", unique: true
+  end
+
   create_table "search_queries", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
     t.datetime "created_at", null: false
     t.string "terms", limit: 2000, null: false

lib/tasks/search.rake

@@ -0,0 +1,19 @@
+namespace :search do
+  desc "Reindex all cards and comments in the search index"
+  task reindex: :environment do
+    puts "Clearing search index..."
+    ActiveRecord::Base.connection.execute("DELETE FROM search_index")
+
+    puts "Reindexing cards..."
+    Card.find_each do |card|
+      card.reindex
+    end
+
+    puts "Reindexing comments..."
+    Comment.find_each do |comment|
+      comment.reindex
+    end
+
+    puts "Done! Reindexed #{Card.count} cards and #{Comment.count} comments."
+  end
+end