diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 12586b2a3..81f59f96c 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -1,18 +1,5 @@ class Search::Record < ApplicationRecord - self.abstract_class = true - - SHARD_COUNT = 16 - - # MySQL sharded table classes - SHARD_CLASSES = SHARD_COUNT.times.map do |shard_id| - Class.new(self) do - self.table_name = "search_records_#{shard_id}" - - def self.name - "Search::Record" - end - end - end.freeze + include const_get(connection.adapter_name) belongs_to :searchable, polymorphic: true, optional: true belongs_to :card, optional: true @@ -22,94 +9,10 @@ class Search::Record < ApplicationRecord validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true - before_save :stem_content, unless: :sqlite? - after_save :upsert_to_fts5_table, if: :sqlite? - after_destroy :delete_from_fts5_table, if: :sqlite? - class << self - def sqlite? - connection.adapter_name == "SQLite" - end - - def upsert_to_fts5(record_id, title:, content:) - # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically - # Note: FTS5 virtual tables don't work properly with bound parameters in SQLite, - # so we need to use string interpolation with proper quoting - conn = connection - sql = "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{record_id}, #{conn.quote(title)}, #{conn.quote(content)})" - conn.execute(sql) - end - - def delete_from_fts5(record_id) - # Note: Use string interpolation for consistency (rowid is always an integer, so safe) - connection.execute( - "DELETE FROM search_records_fts WHERE rowid = #{record_id}" - ) - end - - def for_account(account_id) - if sqlite? - # SQLite uses a single table, no sharding - use a non-abstract subclass - @sqlite_class ||= Class.new(self) do - self.abstract_class = false - self.table_name = "search_records" - - # Override the UUID id attribute from ApplicationRecord - # SQLite uses integer auto-increment primary key (no default) - attribute :id, :integer, default: nil - - def self.name - "Search::Record" - end - end - else - # MySQL uses sharded tables - SHARD_CLASSES[shard_id_for_account(account_id)] - end - end - - def shard_id_for_account(account_id) - Zlib.crc32(account_id.to_s) % SHARD_COUNT - end - def card_join "INNER JOIN #{table_name} ON #{table_name}.card_id = cards.id" end - - # Convert MySQL BOOLEAN MODE syntax to SQLite FTS5 syntax - def convert_to_fts5_query(query) - query = query.to_s.dup - - # Handle quoted phrases - these work the same in both - # Extract them to protect during processing - phrases = [] - query.gsub!(/"([^"]+)"/) do - phrases << $1 - "__PHRASE_#{phrases.length - 1}__" - end - - # Split into tokens - tokens = query.split(/\s+/) - - processed_tokens = tokens.map do |token| - if token.start_with?("__PHRASE_") - # Restore phrase - idx = token[/\d+/].to_i - "\"#{phrases[idx]}\"" - elsif token.start_with?("+") - # Remove + prefix (FTS5 ANDs by default) - token[1..] - elsif token.start_with?("-") - # Convert -term to NOT term - "NOT #{token[1..]}" - else - token - end - end - - # Join with AND (FTS5 uses AND/OR/NOT operators) - processed_tokens.reject(&:blank?).join(" ") - end end scope :for_query, ->(query:, user:) do @@ -121,18 +24,7 @@ class Search::Record < ApplicationRecord end scope :matching, ->(query) do - if sqlite? - # SQLite FTS5: join on rowid for fast lookup with native highlighting - # Porter tokenizer handles stemming automatically - fts_query = convert_to_fts5_query(query) - - joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") - .where("search_records_fts MATCH ?", fts_query) - else - # MySQL FULLTEXT: manually stem query terms - stemmed_query = Search::Stemmer.stem(query) - where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", stemmed_query) - end + matching_scope(query) end scope :for_user, ->(user) do @@ -144,36 +36,7 @@ class Search::Record < ApplicationRecord .includes(:searchable, card: [ :board, :creator ]) .order(created_at: :desc) - if sqlite? - # SQLite: matching scope already selected all columns + FTS5 highlight columns - # Re-select to add query terms (ActiveRecord replaces the select list) - opening_mark = Search::Highlighter::OPENING_MARK - closing_mark = Search::Highlighter::CLOSING_MARK - ellipsis = Search::Highlighter::ELIPSIS - - relation.select( - "#{table_name}.id", - "#{table_name}.account_id", - "#{table_name}.searchable_type", - "#{table_name}.searchable_id", - "#{table_name}.card_id", - "#{table_name}.board_id", - "#{table_name}.title", - "#{table_name}.content", - "#{table_name}.created_at", - "highlight(search_records_fts, 0, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_title", - "highlight(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_content", - "snippet(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}, #{connection.quote(ellipsis)}, 20) AS content_snippet", - "#{connection.quote(query.terms)} AS query" - ) - else - # MySQL: select specific columns needed - relation.select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query") - end - end - - def sqlite? - self.class.sqlite? + search_scope(relation, query) end def source @@ -183,65 +46,4 @@ class Search::Record < ApplicationRecord def comment searchable if searchable_type == "Comment" end - - def card_title - if card_id - if self.class.sqlite? && attribute?(:highlighted_title) - # Use FTS5 native highlighting (already HTML-safe from FTS5) - highlighted_title.html_safe - else - # MySQL: use Ruby highlighter - highlight(card.title, show: :full) - end - end - end - - def card_description - if card_id - if self.class.sqlite? && attribute?(:content_snippet) - # Use FTS5 native snippet for content (already HTML-safe from FTS5) - content_snippet.html_safe if content_snippet.present? - else - # MySQL: use Ruby highlighter - highlight(card.description.to_plain_text, show: :snippet) - end - end - end - - def comment_body - if comment - if self.class.sqlite? && attribute?(:content_snippet) - # Use FTS5 native snippet for content (already HTML-safe from FTS5) - content_snippet.html_safe if content_snippet.present? - else - # MySQL: use Ruby highlighter - highlight(comment.body.to_plain_text, show: :snippet) - end - end - end - - private - def stem_content - # MySQL: stem content for FULLTEXT search - self.title = Search::Stemmer.stem(title) if title_changed? - self.content = Search::Stemmer.stem(content) if content_changed? - end - - def upsert_to_fts5_table - # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically - self.class.upsert_to_fts5(id, title: title, content: content) - end - - def delete_from_fts5_table - self.class.delete_from_fts5(id) - end - - def highlight(text, show:) - if text.present? && attribute?(:query) - highlighter = Search::Highlighter.new(query) - show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) - else - text - end - end end diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb new file mode 100644 index 000000000..eb2c9639b --- /dev/null +++ b/app/models/search/record/sqlite.rb @@ -0,0 +1,93 @@ +module Search::Record::SQLite + extend ActiveSupport::Concern + + included do + self.table_name = "search_records" + + # Override the UUID id attribute from ApplicationRecord + # SQLite uses integer auto-increment primary key (no default) + attribute :id, :integer, default: nil + + after_save :upsert_to_fts5_table + after_destroy :delete_from_fts5_table + end + + class_methods do + def for_account(account_id) + # SQLite uses a single table, no sharding + self + end + + def matching_scope(query) + # SQLite FTS5: join on rowid for fast lookup with native highlighting + # Porter tokenizer handles stemming automatically + joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") + .where("search_records_fts MATCH ?", query) + end + + def search_scope(relation, query) + # SQLite: matching scope already selected all columns + FTS5 highlight columns + # Re-select to add query terms (ActiveRecord replaces the select list) + opening_mark = Search::Highlighter::OPENING_MARK + closing_mark = Search::Highlighter::CLOSING_MARK + ellipsis = Search::Highlighter::ELIPSIS + + relation.select( + "#{table_name}.id", + "#{table_name}.account_id", + "#{table_name}.searchable_type", + "#{table_name}.searchable_id", + "#{table_name}.card_id", + "#{table_name}.board_id", + "#{table_name}.title", + "#{table_name}.content", + "#{table_name}.created_at", + "highlight(search_records_fts, 0, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_title", + "highlight(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_content", + "snippet(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}, #{connection.quote(ellipsis)}, 20) AS content_snippet", + "#{connection.quote(query.terms)} AS query" + ) + end + end + + def card_title + if card_id + # Use FTS5 native highlighting (already HTML-safe from FTS5) + # Fall back to card.title if highlight is nil (e.g., for comment matches where title is NULL) + if has_attribute?(:highlighted_title) && highlighted_title.present? + highlighted_title.html_safe + else + card.title + end + end + end + + def card_description + if card_id && has_attribute?(:content_snippet) && content_snippet.present? + # Use FTS5 native snippet for content (already HTML-safe from FTS5) + content_snippet.html_safe + end + end + + def comment_body + if comment && has_attribute?(:content_snippet) && content_snippet.present? + # Use FTS5 native snippet for content (already HTML-safe from FTS5) + content_snippet.html_safe + end + end + + private + def upsert_to_fts5_table + # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically + # Note: FTS5 virtual tables don't work properly with bound parameters in SQLite, + # so we need to use string interpolation with proper quoting + self.class.connection.execute( + "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{id}, #{self.class.connection.quote(title)}, #{self.class.connection.quote(content)})" + ) + end + + def delete_from_fts5_table + # Note: Use string interpolation for consistency (rowid is always an integer, so safe) + self.class.connection.execute("DELETE FROM search_records_fts WHERE rowid = #{id}") + end +end diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb new file mode 100644 index 000000000..b55deeb0b --- /dev/null +++ b/app/models/search/record/trilogy.rb @@ -0,0 +1,73 @@ +module Search::Record::Trilogy + extend ActiveSupport::Concern + + SHARD_COUNT = 16 + + included do + self.abstract_class = true + + before_save :stem_content + end + + class_methods do + def shard_classes + @shard_classes ||= SHARD_COUNT.times.map do |shard_id| + Class.new(Search::Record) do + self.table_name = "search_records_#{shard_id}" + + def self.name + "Search::Record" + end + end + end.freeze + end + + def for_account(account_id) + # MySQL uses sharded tables + shard_classes[shard_id_for_account(account_id)] + end + + def shard_id_for_account(account_id) + Zlib.crc32(account_id.to_s) % SHARD_COUNT + end + + def matching_scope(query) + # MySQL FULLTEXT: manually stem query terms + stemmed_query = Search::Stemmer.stem(query) + where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", stemmed_query) + end + + def search_scope(relation, query) + # MySQL: select specific columns needed + relation.select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query") + end + end + + def card_title + highlight(card.title, show: :full) if card_id + end + + def card_description + highlight(card.description.to_plain_text, show: :snippet) if card_id + end + + def comment_body + highlight(comment.body.to_plain_text, show: :snippet) if comment + end + + private + def stem_content + # MySQL: stem content for FULLTEXT search + self.title = Search::Stemmer.stem(title) if title_changed? + self.content = Search::Stemmer.stem(content) if content_changed? + end + + def highlight(text, show:) + if text.present? && attribute?(:query) + highlighter = Search::Highlighter.new(query) + show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) + else + text + end + end +end diff --git a/lib/tasks/search.rake b/lib/tasks/search.rake index 0bb3591ee..c1ec0e28a 100644 --- a/lib/tasks/search.rake +++ b/lib/tasks/search.rake @@ -2,7 +2,7 @@ namespace :search do desc "Reindex all cards and comments in the search index" task reindex: :environment do puts "Clearing search records..." - Search::Record::SHARD_COUNT.times do |shard_id| + Search::Record::Trilogy::SHARD_COUNT.times do |shard_id| ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") end diff --git a/test/test_helpers/search_test_helper.rb b/test/test_helpers/search_test_helper.rb index 7132c2ed2..78b113781 100644 --- a/test/test_helpers/search_test_helper.rb +++ b/test/test_helpers/search_test_helper.rb @@ -28,11 +28,11 @@ module SearchTestHelper private def clear_search_records - if Search::Record.sqlite? + if ActiveRecord::Base.connection.adapter_name == "SQLite" ActiveRecord::Base.connection.execute("DELETE FROM search_records") ActiveRecord::Base.connection.execute("DELETE FROM search_records_fts") else - Search::Record::SHARD_COUNT.times do |shard_id| + Search::Record::Trilogy::SHARD_COUNT.times do |shard_id| ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") end end