bterkuile/fizzy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split sqlite/mysql record stuff

Browse Source
This commit is contained in:
Donal McBreen
2025-11-20 17:37:44 +00:00
parent a4850fbe73
commit 19b0e25eac
5 changed files with 172 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/models/search/record.rb
+3 -201
View File
@@ -1,18 +1,5 @@
class Search::Record < ApplicationRecord
self.abstract_class = true
SHARD_COUNT = 16
# MySQL sharded table classes
SHARD_CLASSES = SHARD_COUNT.times.map do |shard_id|
Class.new(self) do
self.table_name = "search_records_#{shard_id}"
def self.name
"Search::Record"
end
end
end.freeze
include const_get(connection.adapter_name)
belongs_to :searchable, polymorphic: true, optional: true
belongs_to :card, optional: true
@@ -22,94 +9,10 @@ class Search::Record < ApplicationRecord
validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true
before_save :stem_content, unless: :sqlite?
after_save :upsert_to_fts5_table, if: :sqlite?
after_destroy :delete_from_fts5_table, if: :sqlite?
class << self
def sqlite?
connection.adapter_name == "SQLite"
end
def upsert_to_fts5(record_id, title:, content:)
# Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically
# Note: FTS5 virtual tables don't work properly with bound parameters in SQLite,
# so we need to use string interpolation with proper quoting
conn = connection
sql = "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{record_id}, #{conn.quote(title)}, #{conn.quote(content)})"
conn.execute(sql)
end
def delete_from_fts5(record_id)
# Note: Use string interpolation for consistency (rowid is always an integer, so safe)
connection.execute(
"DELETE FROM search_records_fts WHERE rowid = #{record_id}"
)
end
def for_account(account_id)
if sqlite?
# SQLite uses a single table, no sharding - use a non-abstract subclass
@sqlite_class ||= Class.new(self) do
self.abstract_class = false
self.table_name = "search_records"
# Override the UUID id attribute from ApplicationRecord
# SQLite uses integer auto-increment primary key (no default)
attribute :id, :integer, default: nil
def self.name
"Search::Record"
end
end
else
# MySQL uses sharded tables
SHARD_CLASSES[shard_id_for_account(account_id)]
end
end
def shard_id_for_account(account_id)
Zlib.crc32(account_id.to_s) % SHARD_COUNT
end
def card_join
"INNER JOIN #{table_name} ON #{table_name}.card_id = cards.id"
end
# Convert MySQL BOOLEAN MODE syntax to SQLite FTS5 syntax
def convert_to_fts5_query(query)
query = query.to_s.dup
# Handle quoted phrases - these work the same in both
# Extract them to protect during processing
phrases = []
query.gsub!(/"([^"]+)"/) do
phrases << $1
"__PHRASE_#{phrases.length - 1}__"
end
# Split into tokens
tokens = query.split(/\s+/)
processed_tokens = tokens.map do |token|
if token.start_with?("__PHRASE_")
# Restore phrase
idx = token[/\d+/].to_i
"\"#{phrases[idx]}\""
elsif token.start_with?("+")
# Remove + prefix (FTS5 ANDs by default)
token[1..]
elsif token.start_with?("-")
# Convert -term to NOT term
"NOT #{token[1..]}"
else
token
end
end
# Join with AND (FTS5 uses AND/OR/NOT operators)
processed_tokens.reject(&:blank?).join(" ")
end
end
scope :for_query, ->(query:, user:) do
@@ -121,18 +24,7 @@ class Search::Record < ApplicationRecord
end
scope :matching, ->(query) do
if sqlite?
# SQLite FTS5: join on rowid for fast lookup with native highlighting
# Porter tokenizer handles stemming automatically
fts_query = convert_to_fts5_query(query)
joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id")
.where("search_records_fts MATCH ?", fts_query)
else
# MySQL FULLTEXT: manually stem query terms
stemmed_query = Search::Stemmer.stem(query)
where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", stemmed_query)
end
matching_scope(query)
end
scope :for_user, ->(user) do
@@ -144,36 +36,7 @@ class Search::Record < ApplicationRecord
.includes(:searchable, card: [ :board, :creator ])
.order(created_at: :desc)
if sqlite?
# SQLite: matching scope already selected all columns + FTS5 highlight columns
# Re-select to add query terms (ActiveRecord replaces the select list)
opening_mark = Search::Highlighter::OPENING_MARK
closing_mark = Search::Highlighter::CLOSING_MARK
ellipsis = Search::Highlighter::ELIPSIS
relation.select(
"#{table_name}.id",
"#{table_name}.account_id",
"#{table_name}.searchable_type",
"#{table_name}.searchable_id",
"#{table_name}.card_id",
"#{table_name}.board_id",
"#{table_name}.title",
"#{table_name}.content",
"#{table_name}.created_at",
"highlight(search_records_fts, 0, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_title",
"highlight(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_content",
"snippet(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}, #{connection.quote(ellipsis)}, 20) AS content_snippet",
"#{connection.quote(query.terms)} AS query"
)
else
# MySQL: select specific columns needed
relation.select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query")
end
end
def sqlite?
self.class.sqlite?
search_scope(relation, query)
end
def source
@@ -183,65 +46,4 @@ class Search::Record < ApplicationRecord
def comment
searchable if searchable_type == "Comment"
end
def card_title
if card_id
if self.class.sqlite? && attribute?(:highlighted_title)
# Use FTS5 native highlighting (already HTML-safe from FTS5)
highlighted_title.html_safe
else
# MySQL: use Ruby highlighter
highlight(card.title, show: :full)
end
end
end
def card_description
if card_id
if self.class.sqlite? && attribute?(:content_snippet)
# Use FTS5 native snippet for content (already HTML-safe from FTS5)
content_snippet.html_safe if content_snippet.present?
else
# MySQL: use Ruby highlighter
highlight(card.description.to_plain_text, show: :snippet)
end
end
end
def comment_body
if comment
if self.class.sqlite? && attribute?(:content_snippet)
# Use FTS5 native snippet for content (already HTML-safe from FTS5)
content_snippet.html_safe if content_snippet.present?
else
# MySQL: use Ruby highlighter
highlight(comment.body.to_plain_text, show: :snippet)
end
end
end
private
def stem_content
# MySQL: stem content for FULLTEXT search
self.title = Search::Stemmer.stem(title) if title_changed?
self.content = Search::Stemmer.stem(content) if content_changed?
end
def upsert_to_fts5_table
# Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically
self.class.upsert_to_fts5(id, title: title, content: content)
end
def delete_from_fts5_table
self.class.delete_from_fts5(id)
end
def highlight(text, show:)
if text.present? && attribute?(:query)
highlighter = Search::Highlighter.new(query)
show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text)
else
text
end
end
end
app/models/search/record/sqlite.rb
+93
View File
@@ -0,0 +1,93 @@
module Search::Record::SQLite
extend ActiveSupport::Concern
included do
self.table_name = "search_records"
# Override the UUID id attribute from ApplicationRecord
# SQLite uses integer auto-increment primary key (no default)
attribute :id, :integer, default: nil
after_save :upsert_to_fts5_table
after_destroy :delete_from_fts5_table
end
class_methods do
def for_account(account_id)
# SQLite uses a single table, no sharding
self
end
def matching_scope(query)
# SQLite FTS5: join on rowid for fast lookup with native highlighting
# Porter tokenizer handles stemming automatically
joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id")
.where("search_records_fts MATCH ?", query)
end
def search_scope(relation, query)
# SQLite: matching scope already selected all columns + FTS5 highlight columns
# Re-select to add query terms (ActiveRecord replaces the select list)
opening_mark = Search::Highlighter::OPENING_MARK
closing_mark = Search::Highlighter::CLOSING_MARK
ellipsis = Search::Highlighter::ELIPSIS
relation.select(
"#{table_name}.id",
"#{table_name}.account_id",
"#{table_name}.searchable_type",
"#{table_name}.searchable_id",
"#{table_name}.card_id",
"#{table_name}.board_id",
"#{table_name}.title",
"#{table_name}.content",
"#{table_name}.created_at",
"highlight(search_records_fts, 0, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_title",
"highlight(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_content",
"snippet(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}, #{connection.quote(ellipsis)}, 20) AS content_snippet",
"#{connection.quote(query.terms)} AS query"
)
end
end
def card_title
if card_id
# Use FTS5 native highlighting (already HTML-safe from FTS5)
# Fall back to card.title if highlight is nil (e.g., for comment matches where title is NULL)
if has_attribute?(:highlighted_title) && highlighted_title.present?
highlighted_title.html_safe
else
card.title
end
end
end
def card_description
if card_id && has_attribute?(:content_snippet) && content_snippet.present?
# Use FTS5 native snippet for content (already HTML-safe from FTS5)
content_snippet.html_safe
end
end
def comment_body
if comment && has_attribute?(:content_snippet) && content_snippet.present?
# Use FTS5 native snippet for content (already HTML-safe from FTS5)
content_snippet.html_safe
end
end
private
def upsert_to_fts5_table
# Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically
# Note: FTS5 virtual tables don't work properly with bound parameters in SQLite,
# so we need to use string interpolation with proper quoting
self.class.connection.execute(
"INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{id}, #{self.class.connection.quote(title)}, #{self.class.connection.quote(content)})"
)
end
def delete_from_fts5_table
# Note: Use string interpolation for consistency (rowid is always an integer, so safe)
self.class.connection.execute("DELETE FROM search_records_fts WHERE rowid = #{id}")
end
end
app/models/search/record/trilogy.rb
+73
View File
@@ -0,0 +1,73 @@
module Search::Record::Trilogy
extend ActiveSupport::Concern
SHARD_COUNT = 16
included do
self.abstract_class = true
before_save :stem_content
end
class_methods do
def shard_classes
@shard_classes ||= SHARD_COUNT.times.map do |shard_id|
Class.new(Search::Record) do
self.table_name = "search_records_#{shard_id}"
def self.name
"Search::Record"
end
end
end.freeze
end
def for_account(account_id)
# MySQL uses sharded tables
shard_classes[shard_id_for_account(account_id)]
end
def shard_id_for_account(account_id)
Zlib.crc32(account_id.to_s) % SHARD_COUNT
end
def matching_scope(query)
# MySQL FULLTEXT: manually stem query terms
stemmed_query = Search::Stemmer.stem(query)
where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", stemmed_query)
end
def search_scope(relation, query)
# MySQL: select specific columns needed
relation.select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query")
end
end
def card_title
highlight(card.title, show: :full) if card_id
end
def card_description
highlight(card.description.to_plain_text, show: :snippet) if card_id
end
def comment_body
highlight(comment.body.to_plain_text, show: :snippet) if comment
end
private
def stem_content
# MySQL: stem content for FULLTEXT search
self.title = Search::Stemmer.stem(title) if title_changed?
self.content = Search::Stemmer.stem(content) if content_changed?
end
def highlight(text, show:)
if text.present? && attribute?(:query)
highlighter = Search::Highlighter.new(query)
show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text)
else
text
end
end
end
lib/tasks/search.rake
+1 -1
View File
@@ -2,7 +2,7 @@ namespace :search do
desc "Reindex all cards and comments in the search index"
task reindex: :environment do
puts "Clearing search records..."
Search::Record::SHARD_COUNT.times do |shard_id|
Search::Record::Trilogy::SHARD_COUNT.times do |shard_id|
ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}")
end
test/test_helpers/search_test_helper.rb
+2 -2
View File
@@ -28,11 +28,11 @@ module SearchTestHelper
private
def clear_search_records
if Search::Record.sqlite?
if ActiveRecord::Base.connection.adapter_name == "SQLite"
ActiveRecord::Base.connection.execute("DELETE FROM search_records")
ActiveRecord::Base.connection.execute("DELETE FROM search_records_fts")
else
Search::Record::SHARD_COUNT.times do |shard_id|
Search::Record::Trilogy::SHARD_COUNT.times do |shard_id|
ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}")
end
end