From a2333d9a37aaf2f2b386e5b66fc222aaf80a3204 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 19 Nov 2025 12:02:26 +0000 Subject: [PATCH 01/55] Add SQLite support - UUID support - Schema compatibility layer, ignore MySQL specific options - Search not working yet --- .gitignore | 3 ++ app/models/application_record.rb | 7 ++- config/database.yml | 29 +++++++++-- config/initializers/active_storage.rb | 7 ++- config/initializers/sqlite_compatibility.rb | 54 +++++++++++++++++++++ config/initializers/uuid_primary_keys.rb | 41 +++++++++++++++- lib/rails_ext/active_record_uuid_type.rb | 3 +- test/fixtures/sessions.yml | 4 +- 8 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 config/initializers/sqlite_compatibility.rb diff --git a/.gitignore b/.gitignore index e890337a2..38bd80d0d 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ !/tmp/storage/.keep /data +*.sqlite3 +*.sqlite3_* + /public/assets # Ignore master key for decrypting credentials and more. diff --git a/app/models/application_record.rb b/app/models/application_record.rb index a043d384f..502f427b4 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -1,7 +1,12 @@ class ApplicationRecord < ActiveRecord::Base primary_abstract_class - connects_to database: { writing: :primary, reading: :replica } + # SQLite doesn't use separate replica databases + if ENV.fetch("DATABASE_ADAPTER", "mysql") == "sqlite" + connects_to database: { writing: :primary, reading: :primary } + else + connects_to database: { writing: :primary, reading: :replica } + end attribute :id, :uuid, default: -> { ActiveRecord::Type::Uuid.generate } end diff --git a/config/database.yml b/config/database.yml index 2b5085410..9cd4820a5 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,4 +1,7 @@ <% + database_adapter = ENV.fetch("DATABASE_ADAPTER", "mysql") + use_sqlite = database_adapter == "sqlite" + if ENV["MIGRATE"].present? mysql_app_user_key = "MYSQL_ALTER_USER" mysql_app_password_key = "MYSQL_ALTER_PASSWORD" @@ -12,13 +15,24 @@ %> default: &default + <% if use_sqlite %> + adapter: sqlite3 + pool: 50 + timeout: 5000 + <% else %> adapter: trilogy host: <%= ENV.fetch "FIZZY_DB_HOST", "127.0.0.1" %> port: <%= ENV.fetch "FIZZY_DB_PORT", 3306 %> pool: 50 timeout: 5000 + <% end %> development: + <% if use_sqlite %> + primary: + <<: *default + database: db/development.sqlite3 + <% else %> primary: <<: *default database: fizzy_development @@ -26,28 +40,34 @@ development: replica: <<: *default database: fizzy_development - replica: true port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %> + replica: true cable: <<: *default database: development_cable - migrations_paths: db/cable_migrate port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %> + migrations_paths: db/cable_migrate cache: <<: *default database: development_cache - migrations_paths: db/cache_migrate port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %> + migrations_paths: db/cache_migrate queue: <<: *default database: development_queue - migrations_paths: db/queue_migrate port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %> + migrations_paths: db/queue_migrate + <% end %> # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". # Do not set this db to the same as development or production. test: + <% if use_sqlite %> + primary: + <<: *default + database: db/test.sqlite3 + <% else %> primary: <<: *default database: fizzy_test @@ -57,6 +77,7 @@ test: database: fizzy_test port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %> replica: true + <% end %> production: &production primary: diff --git a/config/initializers/active_storage.rb b/config/initializers/active_storage.rb index de51ae1f1..361d86075 100644 --- a/config/initializers/active_storage.rb +++ b/config/initializers/active_storage.rb @@ -6,7 +6,12 @@ end # Use DB read/write splitting for Active Storage models ActiveSupport.on_load(:active_storage_record) do - connects_to database: { writing: :primary, reading: :replica } + # SQLite doesn't use separate replica databases + if ENV.fetch("DATABASE_ADAPTER", "mysql") == "sqlite" + connects_to database: { writing: :primary, reading: :primary } + else + connects_to database: { writing: :primary, reading: :replica } + end end module ActiveStorageControllerExtensions diff --git a/config/initializers/sqlite_compatibility.rb b/config/initializers/sqlite_compatibility.rb new file mode 100644 index 000000000..3cff5522f --- /dev/null +++ b/config/initializers/sqlite_compatibility.rb @@ -0,0 +1,54 @@ +# SQLite compatibility layer - filters out MySQL-specific options when using SQLite + +# Define modules outside on_load so they're available immediately +module SQLiteCompatibility + module SQLiteTableDefinitionCompatibility + # Override column method to filter out MySQL-specific options + def column(name, type, **options) + # Check if we're using SQLite by checking the connection adapter + if @conn.is_a?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) + # Remove MySQL-specific options that SQLite doesn't support + options = options.except(:size, :charset, :collation, :unsigned) + end + super(name, type, **options) + end + + # Override index method to filter out MySQL-specific index options + def index(column_name, **options) + if @conn.is_a?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) + # Skip fulltext indexes entirely for SQLite + return if options[:type] == :fulltext + # SQLite doesn't support length-limited indexes + options = options.except(:length) + end + super(column_name, **options) + end + end + + module SQLiteSchemaStatementCompatibility + # Override create_table to filter out MySQL-specific table options + def create_table(table_name, **options) + if is_a?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) + # Remove MySQL-specific table options + options = options.except(:charset, :collation) + end + super(table_name, **options) + end + end +end + +# Apply the prepends - both in on_load callback and immediately +def apply_sqlite_compatibility + if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) + ActiveRecord::ConnectionAdapters::TableDefinition.prepend(SQLiteCompatibility::SQLiteTableDefinitionCompatibility) + ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(SQLiteCompatibility::SQLiteSchemaStatementCompatibility) + end +end + +# Run immediately if ActiveRecord is already loaded +apply_sqlite_compatibility + +# Also run when ActiveRecord loads (for cases where it loads later) +ActiveSupport.on_load(:active_record) do + apply_sqlite_compatibility +end diff --git a/config/initializers/uuid_primary_keys.rb b/config/initializers/uuid_primary_keys.rb index 560f5b384..a1a9dd8ec 100644 --- a/config/initializers/uuid_primary_keys.rb +++ b/config/initializers/uuid_primary_keys.rb @@ -16,7 +16,42 @@ ActiveSupport.on_load(:active_record) do end end - ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlUuidAdapter) + if defined?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter) + ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlUuidAdapter) + end + + module SqliteUuidAdapter + # Add UUID to SQLite's native database types + def native_database_types + @native_database_types_with_uuid ||= super.merge(uuid: { name: "blob", limit: 16 }) + end + + # Override lookup_cast_type to recognize BLOB as UUID type + def lookup_cast_type(sql_type) + if sql_type == "blob(16)" + ActiveRecord::Type.lookup(:uuid, adapter: :sqlite3) + else + super + end + end + + # Override fetch_type_metadata to preserve UUID type and limit + def fetch_type_metadata(sql_type) + if sql_type == "blob(16)" + ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new( + sql_type: sql_type, + type: :uuid, + limit: 16 + ) + else + super + end + end + end + + if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) + ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(SqliteUuidAdapter) + end module SchemaDumperUuidType # Map binary(16) columns to :uuid type in schema.rb @@ -29,7 +64,9 @@ ActiveSupport.on_load(:active_record) do end end - ActiveRecord::ConnectionAdapters::MySQL::SchemaDumper.prepend(SchemaDumperUuidType) + if defined?(ActiveRecord::ConnectionAdapters::MySQL::SchemaDumper) + ActiveRecord::ConnectionAdapters::MySQL::SchemaDumper.prepend(SchemaDumperUuidType) + end module TableDefinitionUuidSupport def uuid(name, **options) diff --git a/lib/rails_ext/active_record_uuid_type.rb b/lib/rails_ext/active_record_uuid_type.rb index 3aca09762..02bdf3ed3 100644 --- a/lib/rails_ext/active_record_uuid_type.rb +++ b/lib/rails_ext/active_record_uuid_type.rb @@ -39,5 +39,6 @@ module ActiveRecord end end -# Register the UUID type for Trilogy adapter +# Register the UUID type for Trilogy (MySQL) and SQLite3 adapters ActiveRecord::Type.register(:uuid, ActiveRecord::Type::Uuid, adapter: :trilogy) +ActiveRecord::Type.register(:uuid, ActiveRecord::Type::Uuid, adapter: :sqlite3) diff --git a/test/fixtures/sessions.yml b/test/fixtures/sessions.yml index a062f53fd..cf6abd120 100644 --- a/test/fixtures/sessions.yml +++ b/test/fixtures/sessions.yml @@ -7,5 +7,5 @@ kevin: jz: identity: jz -jason: - identity: jason +mike: + identity: mike From bb36b4846f272fa1feb0fb3b3ae519a25caf063e Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 20 Nov 2025 13:40:14 +0000 Subject: [PATCH 02/55] Support SQLite searching --- app/models/concerns/searchable.rb | 82 ++- app/models/search/query.rb | 5 +- app/models/search/record.rb | 136 ++++- config/database.yml | 4 + config/initializers/sqlite_compatibility.rb | 2 - config/initializers/sqlite_schema_dumper.rb | 33 + config/initializers/uuid_primary_keys.rb | 48 +- .../20251112093037_create_search_indices.rb | 6 + ...51113190256_create_search_record_shards.rb | 3 + .../20251120110206_add_search_records.rb | 37 ++ db/schema.rb | 19 +- db/schema_sqlite.rb | 578 ++++++++++++++++++ test/test_helpers/search_test_helper.rb | 9 +- 13 files changed, 925 insertions(+), 37 deletions(-) create mode 100644 config/initializers/sqlite_schema_dumper.rb create mode 100644 db/migrate/20251120110206_add_search_records.rb create mode 100644 db/schema_sqlite.rb diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 17e965298..017c1d500 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -13,21 +13,79 @@ module Searchable private def create_in_search_index - Search::Record.for_account(account_id).create!(search_record_attributes) + search_class = Search::Record.for_account(account_id) + + if Search::Record.sqlite? + # SQLite: create with unstemmed content, FTS5 handles stemming + record = search_class.create!(search_record_attributes) + upsert_to_fts5(record.id) + else + # MySQL: create with stemmed content for FULLTEXT search + attrs = search_record_attributes.merge( + title: Search::Stemmer.stem(search_record_attributes[:title]), + content: Search::Stemmer.stem(search_record_attributes[:content]) + ) + search_class.create!(attrs) + end end def update_in_search_index - Search::Record.for_account(account_id).upsert_all( - [ search_record_attributes.merge(id: ActiveRecord::Type::Uuid.generate) ], - update_only: [ :card_id, :board_id, :title, :content, :created_at ] - ) + search_class = Search::Record.for_account(account_id) + + if Search::Record.sqlite? + # SQLite: find or create record, then upsert to FTS5 + record = search_class.find_or_initialize_by( + searchable_type: self.class.name, + searchable_id: id + ) + record.assign_attributes(search_record_attributes) + record.save! + upsert_to_fts5(record.id) + else + # MySQL: use upsert_all with stemmed content + attrs = search_record_attributes.merge( + id: ActiveRecord::Type::Uuid.generate, + title: Search::Stemmer.stem(search_record_attributes[:title]), + content: Search::Stemmer.stem(search_record_attributes[:content]) + ) + search_class.upsert_all( + [ attrs ], + update_only: [ :card_id, :board_id, :title, :content, :created_at ] + ) + end end def remove_from_search_index - Search::Record.for_account(account_id).where( - searchable_type: self.class.name, - searchable_id: id - ).delete_all + search_class = Search::Record.for_account(account_id) + record = search_class.find_by(searchable_type: self.class.name, searchable_id: id) + + if record + # For SQLite, delete from FTS5 first + if Search::Record.sqlite? + delete_from_fts5(record.id) + end + + record.delete + end + end + + def upsert_to_fts5(record_id) + # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically + title = search_title + content = search_content + + # Note: FTS5 virtual tables don't work properly with bound parameters in SQLite, + # so we need to use string interpolation with proper quoting + conn = ActiveRecord::Base.connection + sql = "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{record_id}, #{conn.quote(title)}, #{conn.quote(content)})" + conn.execute(sql) + end + + def delete_from_fts5(record_id) + # Note: Use string interpolation for consistency (rowid is always an integer, so safe) + ActiveRecord::Base.connection.execute( + "DELETE FROM search_records_fts WHERE rowid = #{record_id}" + ) end def search_record_attributes @@ -37,9 +95,9 @@ module Searchable searchable_id: id, card_id: search_card_id, board_id: search_board_id, - title: Search::Stemmer.stem(search_title), - content: Search::Stemmer.stem(search_content), - created_at: created_at + title: search_title, + content: search_content, + created_at: created_at || Time.current } end diff --git a/app/models/search/query.rb b/app/models/search/query.rb index d54142f9e..7d96d5170 100644 --- a/app/models/search/query.rb +++ b/app/models/search/query.rb @@ -16,7 +16,10 @@ class Search::Query < ApplicationRecord end def to_s - Search::Stemmer.stem(terms.to_s) + # Return unstemmed terms - each adapter handles stemming appropriately: + # - SQLite: FTS5 Porter tokenizer stems automatically + # - MySQL: Search::Record.matching applies stemming + terms.to_s end private diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 72cf39865..3f08ddfea 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -3,6 +3,7 @@ class Search::Record < ApplicationRecord SHARD_COUNT = 16 + # MySQL sharded table classes SHARD_CLASSES = SHARD_COUNT.times.map do |shard_id| Class.new(self) do self.table_name = "search_records_#{shard_id}" @@ -13,8 +14,8 @@ class Search::Record < ApplicationRecord end end.freeze - belongs_to :searchable, polymorphic: true - belongs_to :card + belongs_to :searchable, polymorphic: true, optional: true + belongs_to :card, optional: true # Virtual attributes from search query attribute :query, :string @@ -22,8 +23,29 @@ class Search::Record < ApplicationRecord validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true class << self + def sqlite? + connection.adapter_name == "SQLite" + end + def for_account(account_id) - SHARD_CLASSES[shard_id_for_account(account_id)] + if sqlite? + # SQLite uses a single table, no sharding - use a non-abstract subclass + @sqlite_class ||= Class.new(self) do + self.abstract_class = false + self.table_name = "search_records" + + # Override the UUID id attribute from ApplicationRecord + # SQLite uses integer auto-increment primary key (no default) + attribute :id, :integer, default: nil + + def self.name + "Search::Record" + end + end + else + # MySQL uses sharded tables + SHARD_CLASSES[shard_id_for_account(account_id)] + end end def shard_id_for_account(account_id) @@ -33,6 +55,41 @@ class Search::Record < ApplicationRecord def card_join "INNER JOIN #{table_name} ON #{table_name}.card_id = cards.id" end + + # Convert MySQL BOOLEAN MODE syntax to SQLite FTS5 syntax + def convert_to_fts5_query(query) + query = query.to_s.dup + + # Handle quoted phrases - these work the same in both + # Extract them to protect during processing + phrases = [] + query.gsub!(/"([^"]+)"/) do + phrases << $1 + "__PHRASE_#{phrases.length - 1}__" + end + + # Split into tokens + tokens = query.split(/\s+/) + + processed_tokens = tokens.map do |token| + if token.start_with?("__PHRASE_") + # Restore phrase + idx = token[/\d+/].to_i + "\"#{phrases[idx]}\"" + elsif token.start_with?("+") + # Remove + prefix (FTS5 ANDs by default) + token[1..] + elsif token.start_with?("-") + # Convert -term to NOT term + "NOT #{token[1..]}" + else + token + end + end + + # Join with AND (FTS5 uses AND/OR/NOT operators) + processed_tokens.reject(&:blank?).join(" ") + end end scope :for_query, ->(query:, user:) do @@ -44,7 +101,18 @@ class Search::Record < ApplicationRecord end scope :matching, ->(query) do - where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", query) + if sqlite? + # SQLite FTS5: join on rowid for fast lookup with native highlighting + # Porter tokenizer handles stemming automatically + fts_query = convert_to_fts5_query(query) + + joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") + .where("search_records_fts MATCH ?", fts_query) + else + # MySQL FULLTEXT: manually stem query terms + stemmed_query = Search::Stemmer.stem(query) + where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", stemmed_query) + end end scope :for_user, ->(user) do @@ -52,10 +120,36 @@ class Search::Record < ApplicationRecord end scope :search, ->(query:, user:) do - for_query(query: query, user: user) + relation = for_query(query: query, user: user) .includes(:searchable, card: [ :board, :creator ]) - .select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query") .order(created_at: :desc) + + if sqlite? + # SQLite: matching scope already selected all columns + FTS5 highlight columns + # Re-select to add query terms (ActiveRecord replaces the select list) + opening_mark = Search::Highlighter::OPENING_MARK + closing_mark = Search::Highlighter::CLOSING_MARK + ellipsis = Search::Highlighter::ELIPSIS + + relation.select( + "#{table_name}.id", + "#{table_name}.account_id", + "#{table_name}.searchable_type", + "#{table_name}.searchable_id", + "#{table_name}.card_id", + "#{table_name}.board_id", + "#{table_name}.title", + "#{table_name}.content", + "#{table_name}.created_at", + "highlight(search_records_fts, 0, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_title", + "highlight(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_content", + "snippet(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}, #{connection.quote(ellipsis)}, 20) AS content_snippet", + "#{connection.quote(query.terms)} AS query" + ) + else + # MySQL: select specific columns needed + relation.select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query") + end end def source @@ -67,15 +161,39 @@ class Search::Record < ApplicationRecord end def card_title - highlight(card.title, show: :full) if card_id + if card_id + if self.class.sqlite? && attribute?(:highlighted_title) + # Use FTS5 native highlighting (already HTML-safe from FTS5) + highlighted_title.html_safe + else + # MySQL: use Ruby highlighter + highlight(card.title, show: :full) + end + end end def card_description - highlight(card.description.to_plain_text, show: :snippet) if card_id + if card_id + if self.class.sqlite? && attribute?(:content_snippet) + # Use FTS5 native snippet for content (already HTML-safe from FTS5) + content_snippet.html_safe if content_snippet.present? + else + # MySQL: use Ruby highlighter + highlight(card.description.to_plain_text, show: :snippet) + end + end end def comment_body - highlight(comment.body.to_plain_text, show: :snippet) if comment + if comment + if self.class.sqlite? && attribute?(:content_snippet) + # Use FTS5 native snippet for content (already HTML-safe from FTS5) + content_snippet.html_safe if content_snippet.present? + else + # MySQL: use Ruby highlighter + highlight(comment.body.to_plain_text, show: :snippet) + end + end end private diff --git a/config/database.yml b/config/database.yml index 9cd4820a5..39396ffb3 100644 --- a/config/database.yml +++ b/config/database.yml @@ -32,11 +32,13 @@ development: primary: <<: *default database: db/development.sqlite3 + schema_dump: schema_sqlite.rb <% else %> primary: <<: *default database: fizzy_development port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %> + schema_dump: schema.rb replica: <<: *default database: fizzy_development @@ -67,11 +69,13 @@ test: primary: <<: *default database: db/test.sqlite3 + schema_dump: schema_sqlite.rb <% else %> primary: <<: *default database: fizzy_test port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %> + schema_dump: schema.rb replica: <<: *default database: fizzy_test diff --git a/config/initializers/sqlite_compatibility.rb b/config/initializers/sqlite_compatibility.rb index 3cff5522f..b0d192912 100644 --- a/config/initializers/sqlite_compatibility.rb +++ b/config/initializers/sqlite_compatibility.rb @@ -16,8 +16,6 @@ module SQLiteCompatibility # Override index method to filter out MySQL-specific index options def index(column_name, **options) if @conn.is_a?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) - # Skip fulltext indexes entirely for SQLite - return if options[:type] == :fulltext # SQLite doesn't support length-limited indexes options = options.except(:length) end diff --git a/config/initializers/sqlite_schema_dumper.rb b/config/initializers/sqlite_schema_dumper.rb new file mode 100644 index 000000000..c1a43417c --- /dev/null +++ b/config/initializers/sqlite_schema_dumper.rb @@ -0,0 +1,33 @@ +# Fix for SQLite FTS5 virtual table schema dumping +# Rails has a bug where it doesn't handle FTS5 content= and content_rowid= options + +module SQLiteFTS5SchemaDumperFix + # Override the virtual_tables method to handle FTS5 syntax properly + def virtual_tables(stream) + # Query sqlite_master for all virtual tables + virtual_table_sqls = @connection.select_rows( + "SELECT name, sql FROM sqlite_master WHERE type='table' AND sql LIKE 'CREATE VIRTUAL TABLE%'" + ) + + virtual_table_sqls.each do |table_name, sql| + # Just output the raw SQL since create_virtual_table doesn't handle our syntax + stream.puts " execute #{sql.inspect}" + stream.puts + end + end +end + +if ENV.fetch("DATABASE_ADAPTER", "mysql") == "sqlite" + ActiveSupport.on_load(:active_record) do + # Ensure schema dumper is loaded + begin + require 'active_record/connection_adapters/sqlite3/schema_dumper' + rescue LoadError + # SQLite3 adapter not available + end + + if defined?(ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper) + ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper.prepend(SQLiteFTS5SchemaDumperFix) + end + end +end diff --git a/config/initializers/uuid_primary_keys.rb b/config/initializers/uuid_primary_keys.rb index a1a9dd8ec..1b41315ec 100644 --- a/config/initializers/uuid_primary_keys.rb +++ b/config/initializers/uuid_primary_keys.rb @@ -1,4 +1,15 @@ # Automatically use UUID type for all binary(16) columns + +# Define schema dumper module outside on_load so it can be prepended +module SchemaDumperUuidType + # Map binary(16) and blob(16) columns to :uuid type in schema.rb + def schema_type(column) + return :uuid if column.sql_type == "binary(16)" + return :uuid if column.sql_type == "blob(16)" + super + end +end + ActiveSupport.on_load(:active_record) do module MysqlUuidAdapter # Add UUID to MySQL's native database types @@ -21,7 +32,7 @@ ActiveSupport.on_load(:active_record) do end module SqliteUuidAdapter - # Add UUID to SQLite's native database types + # Add UUID to SQLite's native database types (instance method) def native_database_types @native_database_types_with_uuid ||= super.merge(uuid: { name: "blob", limit: 16 }) end @@ -51,23 +62,42 @@ ActiveSupport.on_load(:active_record) do if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(SqliteUuidAdapter) - end - module SchemaDumperUuidType - # Map binary(16) columns to :uuid type in schema.rb - def schema_type(column) - if column.sql_type == "binary(16)" - :uuid - else - super + # Also add UUID to class-level native_database_types + ActiveRecord::ConnectionAdapters::SQLite3Adapter.class_eval do + @native_database_types = nil # Clear cache + + class << self + alias_method :original_native_database_types, :native_database_types + + def native_database_types + @native_database_types_with_uuid ||= original_native_database_types.merge(uuid: { name: "blob", limit: 16 }) + end end end end + # Ensure schema dumper classes are loaded before prepending + begin + require 'active_record/connection_adapters/mysql/schema_dumper' + rescue LoadError + # MySQL adapter not available + end + + begin + require 'active_record/connection_adapters/sqlite3/schema_dumper' + rescue LoadError + # SQLite3 adapter not available + end + if defined?(ActiveRecord::ConnectionAdapters::MySQL::SchemaDumper) ActiveRecord::ConnectionAdapters::MySQL::SchemaDumper.prepend(SchemaDumperUuidType) end + if defined?(ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper) + ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper.prepend(SchemaDumperUuidType) + end + module TableDefinitionUuidSupport def uuid(name, **options) column(name, :uuid, **options) diff --git a/db/migrate/20251112093037_create_search_indices.rb b/db/migrate/20251112093037_create_search_indices.rb index 441284590..26ca71ee5 100644 --- a/db/migrate/20251112093037_create_search_indices.rb +++ b/db/migrate/20251112093037_create_search_indices.rb @@ -1,5 +1,8 @@ class CreateSearchIndices < ActiveRecord::Migration[8.2] def up + # Skip for SQLite - it doesn't use these tables + return if connection.adapter_name == "SQLite" + 16.times do |i| create_table "search_index_#{i}".to_sym, id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.string :searchable_type, null: false @@ -17,6 +20,9 @@ class CreateSearchIndices < ActiveRecord::Migration[8.2] end def down + # Skip for SQLite - it doesn't use these tables + return if connection.adapter_name == "SQLite" + 16.times do |i| drop_table "search_index_#{i}".to_sym end diff --git a/db/migrate/20251113190256_create_search_record_shards.rb b/db/migrate/20251113190256_create_search_record_shards.rb index 5afb2ab8e..83f8f626e 100644 --- a/db/migrate/20251113190256_create_search_record_shards.rb +++ b/db/migrate/20251113190256_create_search_record_shards.rb @@ -2,6 +2,9 @@ class CreateSearchRecordShards < ActiveRecord::Migration[8.2] SHARD_COUNT = 16 def change + # Skip for SQLite - it uses a single search_records table instead + return if connection.adapter_name == "SQLite" + # Create 16 sharded search_records tables SHARD_COUNT.times do |shard_id| create_table "search_records_#{shard_id}", id: :uuid do |t| diff --git a/db/migrate/20251120110206_add_search_records.rb b/db/migrate/20251120110206_add_search_records.rb new file mode 100644 index 000000000..866a39b48 --- /dev/null +++ b/db/migrate/20251120110206_add_search_records.rb @@ -0,0 +1,37 @@ +class AddSearchRecords < ActiveRecord::Migration[8.2] + def up + return unless connection.adapter_name == "SQLite" + + # Create regular table with integer primary key for FTS5 rowid compatibility + create_table :search_records do |t| + t.uuid :account_id, null: false + t.string :searchable_type, null: false + t.uuid :searchable_id, null: false + t.uuid :card_id, null: false + t.uuid :board_id, null: false + t.string :title + t.text :content + t.datetime :created_at, null: false + + t.index [:searchable_type, :searchable_id], unique: true + t.index :account_id + end + + # Create FTS5 virtual table using Porter stemmer + # No triggers needed - Searchable concern handles sync via callbacks + execute <<-SQL + CREATE VIRTUAL TABLE search_records_fts USING fts5( + title, + content, + tokenize='porter' + ) + SQL + end + + def down + return unless connection.adapter_name == "SQLite" + + execute "DROP TABLE IF EXISTS search_records_fts" + drop_table :search_records, if_exists: true + end +end diff --git a/db/schema.rb b/db/schema.rb index e6a403fee..6164fd37f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2025_11_20_203100) do +ActiveRecord::Schema[8.2].define(version: 2025_11_20_110206) do create_table "accesses", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false @@ -46,7 +46,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_20_203100) do create_table "action_text_rich_texts", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.uuid "account_id", null: false - t.text "body", size: :long + t.text "body" t.datetime "created_at", null: false t.string "name", null: false t.uuid "record_id", null: false @@ -408,6 +408,21 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_20_203100) do t.index ["user_id"], name: "index_search_queries_on_user_id" end + create_table "search_records", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.string "account_key", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_on_account_id" + t.index ["account_key", "content", "title"], name: "index_search_records_on_account_key_and_content_and_title", type: :fulltext + t.index ["searchable_type", "searchable_id"], name: "index_search_records_on_searchable_type_and_searchable_id", unique: true + end + create_table "search_records_0", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.uuid "account_id", null: false t.uuid "board_id", null: false diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb new file mode 100644 index 000000000..387e0aa06 --- /dev/null +++ b/db/schema_sqlite.rb @@ -0,0 +1,578 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[8.2].define(version: 2025_11_20_110206) do + create_table "accesses", id: :uuid, force: :cascade do |t| + t.datetime "accessed_at" + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.datetime "created_at", null: false + t.string "involvement", default: "access_only", null: false + t.datetime "updated_at", null: false + t.uuid "user_id", null: false + t.index ["account_id", "accessed_at"], name: "index_accesses_on_account_id_and_accessed_at" + t.index ["board_id", "user_id"], name: "index_accesses_on_board_id_and_user_id", unique: true + t.index ["board_id"], name: "index_accesses_on_board_id" + t.index ["user_id"], name: "index_accesses_on_user_id" + end + + create_table "account_join_codes", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.string "code", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.bigint "usage_count", default: 0, null: false + t.bigint "usage_limit", default: 10, null: false + t.index ["account_id", "code"], name: "index_account_join_codes_on_account_id_and_code", unique: true + end + + create_table "accounts", id: :uuid, force: :cascade do |t| + t.bigint "cards_count", default: 0, null: false + t.datetime "created_at", null: false + t.bigint "external_account_id" + t.string "name", null: false + t.datetime "updated_at", null: false + t.index ["external_account_id"], name: "index_accounts_on_external_account_id", unique: true + end + + create_table "action_text_rich_texts", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.text "body" + t.datetime "created_at", null: false + t.string "name", null: false + t.uuid "record_id", null: false + t.string "record_type", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_action_text_rich_texts_on_account_id" + t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true + end + + create_table "active_storage_attachments", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "blob_id", null: false + t.datetime "created_at", null: false + t.string "name", null: false + t.uuid "record_id", null: false + t.string "record_type", null: false + t.index ["account_id"], name: "index_active_storage_attachments_on_account_id" + t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" + t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true + end + + create_table "active_storage_blobs", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.bigint "byte_size", null: false + t.string "checksum" + t.string "content_type" + t.datetime "created_at", null: false + t.string "filename", null: false + t.string "key", null: false + t.text "metadata" + t.string "service_name", null: false + t.index ["account_id"], name: "index_active_storage_blobs_on_account_id" + t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true + end + + create_table "active_storage_variant_records", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "blob_id", null: false + t.string "variation_digest", null: false + t.index ["account_id"], name: "index_active_storage_variant_records_on_account_id" + t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true + end + + create_table "assignees_filters", id: false, force: :cascade do |t| + t.uuid "assignee_id", null: false + t.uuid "filter_id", null: false + t.index ["assignee_id"], name: "index_assignees_filters_on_assignee_id" + t.index ["filter_id"], name: "index_assignees_filters_on_filter_id" + end + + create_table "assigners_filters", id: false, force: :cascade do |t| + t.uuid "assigner_id", null: false + t.uuid "filter_id", null: false + t.index ["assigner_id"], name: "index_assigners_filters_on_assigner_id" + t.index ["filter_id"], name: "index_assigners_filters_on_filter_id" + end + + create_table "assignments", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "assignee_id", null: false + t.uuid "assigner_id", null: false + t.uuid "card_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_assignments_on_account_id" + t.index ["assignee_id", "card_id"], name: "index_assignments_on_assignee_id_and_card_id", unique: true + t.index ["card_id"], name: "index_assignments_on_card_id" + end + + create_table "board_publications", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.datetime "created_at", null: false + t.string "key" + t.datetime "updated_at", null: false + t.index ["account_id", "key"], name: "index_board_publications_on_account_id_and_key" + t.index ["board_id"], name: "index_board_publications_on_board_id" + end + + create_table "boards", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.boolean "all_access", default: false, null: false + t.datetime "created_at", null: false + t.uuid "creator_id", null: false + t.string "name", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_boards_on_account_id" + t.index ["creator_id"], name: "index_boards_on_creator_id" + end + + create_table "boards_filters", id: false, force: :cascade do |t| + t.uuid "board_id", null: false + t.uuid "filter_id", null: false + t.index ["board_id"], name: "index_boards_filters_on_board_id" + t.index ["filter_id"], name: "index_boards_filters_on_filter_id" + end + + create_table "card_activity_spikes", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_card_activity_spikes_on_account_id" + t.index ["card_id"], name: "index_card_activity_spikes_on_card_id" + end + + create_table "card_engagements", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id" + t.datetime "created_at", null: false + t.string "status", default: "doing", null: false + t.datetime "updated_at", null: false + t.index ["account_id", "status"], name: "index_card_engagements_on_account_id_and_status" + t.index ["card_id"], name: "index_card_engagements_on_card_id" + end + + create_table "card_goldnesses", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_card_goldnesses_on_account_id" + t.index ["card_id"], name: "index_card_goldnesses_on_card_id", unique: true + end + + create_table "card_not_nows", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "user_id" + t.index ["account_id"], name: "index_card_not_nows_on_account_id" + t.index ["card_id"], name: "index_card_not_nows_on_card_id", unique: true + t.index ["user_id"], name: "index_card_not_nows_on_user_id" + end + + create_table "cards", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "column_id" + t.datetime "created_at", null: false + t.uuid "creator_id", null: false + t.date "due_on" + t.datetime "last_active_at", null: false + t.bigint "number", null: false + t.string "status", default: "drafted", null: false + t.string "title" + t.datetime "updated_at", null: false + t.index ["account_id", "last_active_at", "status"], name: "index_cards_on_account_id_and_last_active_at_and_status" + t.index ["account_id", "number"], name: "index_cards_on_account_id_and_number", unique: true + t.index ["board_id"], name: "index_cards_on_board_id" + t.index ["column_id"], name: "index_cards_on_column_id" + end + + create_table "closers_filters", id: false, force: :cascade do |t| + t.uuid "closer_id", null: false + t.uuid "filter_id", null: false + t.index ["closer_id"], name: "index_closers_filters_on_closer_id" + t.index ["filter_id"], name: "index_closers_filters_on_filter_id" + end + + create_table "closures", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "user_id" + t.index ["account_id"], name: "index_closures_on_account_id" + t.index ["card_id", "created_at"], name: "index_closures_on_card_id_and_created_at" + t.index ["card_id"], name: "index_closures_on_card_id", unique: true + t.index ["user_id"], name: "index_closures_on_user_id" + end + + create_table "columns", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.string "color", null: false + t.datetime "created_at", null: false + t.string "name", null: false + t.integer "position", default: 0, null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_columns_on_account_id" + t.index ["board_id", "position"], name: "index_columns_on_board_id_and_position" + t.index ["board_id"], name: "index_columns_on_board_id" + end + + create_table "comments", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id", null: false + t.datetime "created_at", null: false + t.uuid "creator_id", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_comments_on_account_id" + t.index ["card_id"], name: "index_comments_on_card_id" + end + + create_table "creators_filters", id: false, force: :cascade do |t| + t.uuid "creator_id", null: false + t.uuid "filter_id", null: false + t.index ["creator_id"], name: "index_creators_filters_on_creator_id" + t.index ["filter_id"], name: "index_creators_filters_on_filter_id" + end + + create_table "entropies", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.bigint "auto_postpone_period", default: 2592000, null: false + t.uuid "container_id", null: false + t.string "container_type", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_entropies_on_account_id" + t.index ["container_type", "container_id", "auto_postpone_period"], name: "idx_on_container_type_container_id_auto_postpone_pe_3d79b50517" + t.index ["container_type", "container_id"], name: "index_entropy_configurations_on_container", unique: true + end + + create_table "events", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.string "action", null: false + t.uuid "board_id", null: false + t.datetime "created_at", null: false + t.uuid "creator_id", null: false + t.uuid "eventable_id", null: false + t.string "eventable_type", null: false + t.json "particulars", default: -> { "json_object()" } + t.datetime "updated_at", null: false + t.index ["account_id", "action"], name: "index_events_on_account_id_and_action" + t.index ["board_id", "action", "created_at"], name: "index_events_on_board_id_and_action_and_created_at" + t.index ["board_id"], name: "index_events_on_board_id" + t.index ["creator_id"], name: "index_events_on_creator_id" + t.index ["eventable_type", "eventable_id"], name: "index_events_on_eventable" + end + + create_table "filters", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.datetime "created_at", null: false + t.uuid "creator_id", null: false + t.json "fields", default: -> { "json_object()" }, null: false + t.string "params_digest", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_filters_on_account_id" + t.index ["creator_id", "params_digest"], name: "index_filters_on_creator_id_and_params_digest", unique: true + end + + create_table "filters_tags", id: false, force: :cascade do |t| + t.uuid "filter_id", null: false + t.uuid "tag_id", null: false + t.index ["filter_id"], name: "index_filters_tags_on_filter_id" + t.index ["tag_id"], name: "index_filters_tags_on_tag_id" + end + + create_table "identities", id: :uuid, force: :cascade do |t| + t.datetime "created_at", null: false + t.string "email_address", null: false + t.datetime "updated_at", null: false + t.index ["email_address"], name: "index_identities_on_email_address", unique: true + end + + create_table "magic_links", id: :uuid, force: :cascade do |t| + t.string "code", null: false + t.datetime "created_at", null: false + t.datetime "expires_at", null: false + t.uuid "identity_id" + t.datetime "updated_at", null: false + t.index ["code"], name: "index_magic_links_on_code", unique: true + t.index ["expires_at"], name: "index_magic_links_on_expires_at" + t.index ["identity_id"], name: "index_magic_links_on_identity_id" + end + + create_table "mentions", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.datetime "created_at", null: false + t.uuid "mentionee_id", null: false + t.uuid "mentioner_id", null: false + t.uuid "source_id", null: false + t.string "source_type", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_mentions_on_account_id" + t.index ["mentionee_id"], name: "index_mentions_on_mentionee_id" + t.index ["mentioner_id"], name: "index_mentions_on_mentioner_id" + t.index ["source_type", "source_id"], name: "index_mentions_on_source" + end + + create_table "notification_bundles", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.datetime "created_at", null: false + t.datetime "ends_at", null: false + t.datetime "starts_at", null: false + t.integer "status", default: 0, null: false + t.datetime "updated_at", null: false + t.uuid "user_id", null: false + t.index ["account_id"], name: "index_notification_bundles_on_account_id" + t.index ["ends_at", "status"], name: "index_notification_bundles_on_ends_at_and_status" + t.index ["user_id", "starts_at", "ends_at"], name: "idx_on_user_id_starts_at_ends_at_7eae5d3ac5" + t.index ["user_id", "status"], name: "index_notification_bundles_on_user_id_and_status" + end + + create_table "notifications", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.datetime "created_at", null: false + t.uuid "creator_id" + t.datetime "read_at" + t.uuid "source_id", null: false + t.string "source_type", null: false + t.datetime "updated_at", null: false + t.uuid "user_id", null: false + t.index ["account_id"], name: "index_notifications_on_account_id" + t.index ["creator_id"], name: "index_notifications_on_creator_id" + t.index ["source_type", "source_id"], name: "index_notifications_on_source" + t.index ["user_id", "read_at", "created_at"], name: "index_notifications_on_user_id_and_read_at_and_created_at", order: { read_at: :desc, created_at: :desc } + t.index ["user_id"], name: "index_notifications_on_user_id" + end + + create_table "pins", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "user_id", null: false + t.index ["account_id"], name: "index_pins_on_account_id" + t.index ["card_id", "user_id"], name: "index_pins_on_card_id_and_user_id", unique: true + t.index ["card_id"], name: "index_pins_on_card_id" + t.index ["user_id"], name: "index_pins_on_user_id" + end + + create_table "push_subscriptions", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.string "auth_key" + t.datetime "created_at", null: false + t.text "endpoint" + t.string "p256dh_key" + t.datetime "updated_at", null: false + t.string "user_agent" + t.uuid "user_id", null: false + t.index ["account_id"], name: "index_push_subscriptions_on_account_id" + t.index ["user_id", "endpoint"], name: "index_push_subscriptions_on_user_id_and_endpoint", unique: true + end + + create_table "reactions", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "comment_id", null: false + t.string "content", limit: 16, null: false + t.datetime "created_at", null: false + t.uuid "reacter_id", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_reactions_on_account_id" + t.index ["comment_id"], name: "index_reactions_on_comment_id" + t.index ["reacter_id"], name: "index_reactions_on_reacter_id" + end + + create_table "search_queries", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.datetime "created_at", null: false + t.string "terms", limit: 2000, null: false + t.datetime "updated_at", null: false + t.uuid "user_id", null: false + t.index ["account_id"], name: "index_search_queries_on_account_id" + t.index ["user_id", "terms"], name: "index_search_queries_on_user_id_and_terms" + t.index ["user_id", "updated_at"], name: "index_search_queries_on_user_id_and_updated_at", unique: true + t.index ["user_id"], name: "index_search_queries_on_user_id" + end + + create_table "search_records", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "board_id", null: false + t.uuid "card_id", null: false + t.text "content" + t.datetime "created_at", null: false + t.uuid "searchable_id", null: false + t.string "searchable_type", null: false + t.string "title" + t.index ["account_id"], name: "index_search_records_on_account_id" + t.index ["searchable_type", "searchable_id"], name: "index_search_records_on_searchable_type_and_searchable_id", unique: true + end + + create_table "sessions", id: :uuid, force: :cascade do |t| + t.datetime "created_at", null: false + t.uuid "identity_id", null: false + t.string "ip_address" + t.datetime "updated_at", null: false + t.string "user_agent" + t.index ["identity_id"], name: "index_sessions_on_identity_id" + end + + create_table "steps", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id", null: false + t.boolean "completed", default: false, null: false + t.text "content", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_steps_on_account_id" + t.index ["card_id", "completed"], name: "index_steps_on_card_id_and_completed" + t.index ["card_id"], name: "index_steps_on_card_id" + end + + create_table "taggings", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id", null: false + t.datetime "created_at", null: false + t.uuid "tag_id", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_taggings_on_account_id" + t.index ["card_id", "tag_id"], name: "index_taggings_on_card_id_and_tag_id", unique: true + t.index ["tag_id"], name: "index_taggings_on_tag_id" + end + + create_table "tags", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.datetime "created_at", null: false + t.string "title" + t.datetime "updated_at", null: false + t.index ["account_id", "title"], name: "index_tags_on_account_id_and_title", unique: true + end + + create_table "user_settings", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.integer "bundle_email_frequency", default: 0, null: false + t.datetime "created_at", null: false + t.string "timezone_name" + t.datetime "updated_at", null: false + t.uuid "user_id", null: false + t.index ["account_id"], name: "index_user_settings_on_account_id" + t.index ["user_id", "bundle_email_frequency"], name: "index_user_settings_on_user_id_and_bundle_email_frequency" + t.index ["user_id"], name: "index_user_settings_on_user_id" + end + + create_table "users", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.boolean "active", default: true, null: false + t.datetime "created_at", null: false + t.uuid "identity_id" + t.string "name", null: false + t.string "role", default: "member", null: false + t.datetime "updated_at", null: false + t.index ["account_id", "identity_id"], name: "index_users_on_account_id_and_identity_id", unique: true + t.index ["account_id", "role"], name: "index_users_on_account_id_and_role" + t.index ["identity_id"], name: "index_users_on_identity_id" + end + + create_table "watches", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "card_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "user_id", null: false + t.boolean "watching", default: true, null: false + t.index ["account_id"], name: "index_watches_on_account_id" + t.index ["card_id"], name: "index_watches_on_card_id" + t.index ["user_id", "card_id"], name: "index_watches_on_user_id_and_card_id" + t.index ["user_id"], name: "index_watches_on_user_id" + end + + create_table "webhook_delinquency_trackers", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.integer "consecutive_failures_count", default: 0 + t.datetime "created_at", null: false + t.datetime "first_failure_at" + t.datetime "updated_at", null: false + t.uuid "webhook_id", null: false + t.index ["account_id"], name: "index_webhook_delinquency_trackers_on_account_id" + t.index ["webhook_id"], name: "index_webhook_delinquency_trackers_on_webhook_id" + end + + create_table "webhook_deliveries", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.datetime "created_at", null: false + t.uuid "event_id", null: false + t.text "request" + t.text "response" + t.string "state", null: false + t.datetime "updated_at", null: false + t.uuid "webhook_id", null: false + t.index ["account_id"], name: "index_webhook_deliveries_on_account_id" + t.index ["event_id"], name: "index_webhook_deliveries_on_event_id" + t.index ["webhook_id"], name: "index_webhook_deliveries_on_webhook_id" + end + + create_table "webhooks", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.boolean "active", default: true, null: false + t.uuid "board_id", null: false + t.datetime "created_at", null: false + t.string "name" + t.string "signing_secret", null: false + t.text "subscribed_actions" + t.datetime "updated_at", null: false + t.text "url", null: false + t.index ["account_id"], name: "index_webhooks_on_account_id" + t.index ["board_id", "subscribed_actions"], name: "index_webhooks_on_board_id_and_subscribed_actions" + end + + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" + add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "board_publications", "boards" + add_foreign_key "card_activity_spikes", "cards" + add_foreign_key "card_goldnesses", "cards" + add_foreign_key "card_not_nows", "cards" + add_foreign_key "card_not_nows", "users" + add_foreign_key "cards", "columns" + add_foreign_key "closures", "cards" + add_foreign_key "closures", "users" + add_foreign_key "columns", "boards" + add_foreign_key "comments", "cards" + add_foreign_key "events", "boards" + add_foreign_key "magic_links", "identities" + add_foreign_key "mentions", "users", column: "mentionee_id" + add_foreign_key "mentions", "users", column: "mentioner_id" + add_foreign_key "notification_bundles", "users" + add_foreign_key "notifications", "users" + add_foreign_key "notifications", "users", column: "creator_id" + add_foreign_key "pins", "cards" + add_foreign_key "pins", "users" + add_foreign_key "push_subscriptions", "users" + add_foreign_key "search_queries", "users" + add_foreign_key "sessions", "identities" + add_foreign_key "steps", "cards" + add_foreign_key "taggings", "cards" + add_foreign_key "taggings", "tags" + add_foreign_key "user_settings", "users" + add_foreign_key "users", "identities" + add_foreign_key "watches", "cards" + add_foreign_key "watches", "users" + add_foreign_key "webhook_delinquency_trackers", "webhooks" + add_foreign_key "webhook_deliveries", "events" + add_foreign_key "webhook_deliveries", "webhooks" + add_foreign_key "webhooks", "boards" + execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n title,\n content,\n tokenize='porter'\n )" + +end diff --git a/test/test_helpers/search_test_helper.rb b/test/test_helpers/search_test_helper.rb index f35f1c2a8..7132c2ed2 100644 --- a/test/test_helpers/search_test_helper.rb +++ b/test/test_helpers/search_test_helper.rb @@ -28,8 +28,13 @@ module SearchTestHelper private def clear_search_records - Search::Record::SHARD_COUNT.times do |shard_id| - ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") + if Search::Record.sqlite? + ActiveRecord::Base.connection.execute("DELETE FROM search_records") + ActiveRecord::Base.connection.execute("DELETE FROM search_records_fts") + else + Search::Record::SHARD_COUNT.times do |shard_id| + ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") + end end end end From cc7e091508ddca4dd4ed4c0f2c3af60605931a08 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 20 Nov 2025 14:57:44 +0000 Subject: [PATCH 03/55] Disable multi-db for SQLite --- app/models/application_record.rb | 7 +--- config/initializers/active_storage.rb | 8 +--- config/initializers/database_role_logging.rb | 6 ++- config/initializers/multi_db.rb | 11 +++-- config/initializers/tenanting/account_slug.rb | 4 +- .../active_record_replica_support.rb | 40 +++++++++++++++++++ 6 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 lib/rails_ext/active_record_replica_support.rb diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 502f427b4..45e9c2f21 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -1,12 +1,7 @@ class ApplicationRecord < ActiveRecord::Base primary_abstract_class - # SQLite doesn't use separate replica databases - if ENV.fetch("DATABASE_ADAPTER", "mysql") == "sqlite" - connects_to database: { writing: :primary, reading: :primary } - else - connects_to database: { writing: :primary, reading: :replica } - end + configure_replica_connections attribute :id, :uuid, default: -> { ActiveRecord::Type::Uuid.generate } end diff --git a/config/initializers/active_storage.rb b/config/initializers/active_storage.rb index 361d86075..d00bd74bf 100644 --- a/config/initializers/active_storage.rb +++ b/config/initializers/active_storage.rb @@ -4,14 +4,8 @@ ActiveSupport.on_load(:active_storage_blob) do end end -# Use DB read/write splitting for Active Storage models ActiveSupport.on_load(:active_storage_record) do - # SQLite doesn't use separate replica databases - if ENV.fetch("DATABASE_ADAPTER", "mysql") == "sqlite" - connects_to database: { writing: :primary, reading: :primary } - else - connects_to database: { writing: :primary, reading: :replica } - end + configure_replica_connections end module ActiveStorageControllerExtensions diff --git a/config/initializers/database_role_logging.rb b/config/initializers/database_role_logging.rb index 698d52080..e2b8779b8 100644 --- a/config/initializers/database_role_logging.rb +++ b/config/initializers/database_role_logging.rb @@ -1,3 +1,5 @@ +require_relative "extensions" + class DatabaseRoleLogger def initialize(app) @app = app @@ -10,4 +12,6 @@ class DatabaseRoleLogger end end -Rails.application.config.middleware.insert_after ActiveRecord::Middleware::DatabaseSelector, DatabaseRoleLogger +if ActiveRecord::Base.replica_configured? + Rails.application.config.middleware.insert_after ActiveRecord::Middleware::DatabaseSelector, DatabaseRoleLogger +end diff --git a/config/initializers/multi_db.rb b/config/initializers/multi_db.rb index b4d91085d..f7b1be6f5 100644 --- a/config/initializers/multi_db.rb +++ b/config/initializers/multi_db.rb @@ -1,7 +1,10 @@ require "deployment" +require_relative "extensions" -Rails.application.configure do - config.active_record.database_selector = { delay: 1.second } - config.active_record.database_resolver = Deployment::DatabaseResolver - config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session +if ActiveRecord::Base.replica_configured? + Rails.application.configure do + config.active_record.database_selector = { delay: 1.second } + config.active_record.database_resolver = Deployment::DatabaseResolver + config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session + end end diff --git a/config/initializers/tenanting/account_slug.rb b/config/initializers/tenanting/account_slug.rb index dffbb6bf8..9d6d83ca7 100644 --- a/config/initializers/tenanting/account_slug.rb +++ b/config/initializers/tenanting/account_slug.rb @@ -43,6 +43,4 @@ module AccountSlug def self.encode(id) FORMAT % id end end -Rails.application.config.middleware.tap do |stack| - stack.insert_before ActiveRecord::Middleware::DatabaseSelector, AccountSlug::Extractor -end +Rails.application.config.middleware.insert_after Rack::TempfileReaper, AccountSlug::Extractor diff --git a/lib/rails_ext/active_record_replica_support.rb b/lib/rails_ext/active_record_replica_support.rb new file mode 100644 index 000000000..2625a1ac9 --- /dev/null +++ b/lib/rails_ext/active_record_replica_support.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Adds a helper method to check if replica database connections are configured +# and automatically configures read/write splitting when replicas are available. +# +# Usage: +# class ApplicationRecord < ActiveRecord::Base +# configure_replica_connections +# end +module ActiveRecordReplicaSupport + extend ActiveSupport::Concern + + class_methods do + # Automatically configures connects_to for read/write splitting if a replica + # database is configured for the current environment. This is a no-op if no + # replica configuration exists. + # + # Example: + # class ApplicationRecord < ActiveRecord::Base + # configure_replica_connections + # end + def configure_replica_connections + if replica_configured? + connects_to database: { writing: :primary, reading: :replica } + end + end + + # Returns true if a replica database configuration exists for the current + # environment. This allows different database adapters to opt in or out of + # read/write splitting based on their database.yml configuration. + # + # Example: + # ApplicationRecord.replica_configured? # => true for MySQL, false for SQLite + def replica_configured? + configurations.find_db_config("replica").present? + end + end +end + +ActiveRecord::Base.include ActiveRecordReplicaSupport From a682a807ed313156dbd72eb2bb96ddc99f73ff97 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 20 Nov 2025 15:00:03 +0000 Subject: [PATCH 04/55] Handle SQLite date subtraction --- app/models/card/entropic.rb | 6 +-- .../active_record_date_arithmetic.rb | 53 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 lib/rails_ext/active_record_date_arithmetic.rb diff --git a/app/models/card/entropic.rb b/app/models/card/entropic.rb index 23e33c18a..b238156f9 100644 --- a/app/models/card/entropic.rb +++ b/app/models/card/entropic.rb @@ -7,7 +7,7 @@ module Card::Entropic .joins(board: :account) .left_outer_joins(board: :entropy) .joins("LEFT OUTER JOIN entropies AS account_entropies ON account_entropies.account_id = accounts.id AND account_entropies.container_type = 'Account' AND account_entropies.container_id = accounts.id") - .where("last_active_at <= DATE_SUB(?, INTERVAL COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period) SECOND)", Time.now) + .where("last_active_at <= #{connection.date_subtract('?', 'COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period)')}", Time.now) end scope :postponing_soon, -> do @@ -16,8 +16,8 @@ module Card::Entropic .joins(board: :account) .left_outer_joins(board: :entropy) .joins("LEFT OUTER JOIN entropies AS account_entropies ON account_entropies.account_id = accounts.id AND account_entropies.container_type = 'Account' AND account_entropies.container_id = accounts.id") - .where("last_active_at > DATE_SUB(?, INTERVAL COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period) SECOND)", now) - .where("last_active_at <= DATE_SUB(?, INTERVAL CAST(COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period) * 0.75 AS SIGNED) SECOND)", now) + .where("last_active_at > #{connection.date_subtract('?', 'COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period)')}", now) + .where("last_active_at <= #{connection.date_subtract('?', 'COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period) * 0.75')}", now) end delegate :auto_postpone_period, to: :board diff --git a/lib/rails_ext/active_record_date_arithmetic.rb b/lib/rails_ext/active_record_date_arithmetic.rb new file mode 100644 index 000000000..5121c3f12 --- /dev/null +++ b/lib/rails_ext/active_record_date_arithmetic.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# Adds database-agnostic date arithmetic methods to ActiveRecord adapters. +# This allows code to perform date calculations without checking which database adapter is in use. +# +# Usage: +# connection.date_subtract("created_at", "3600") +# # MySQL/Trilogy: "DATE_SUB(created_at, INTERVAL 3600 SECOND)" +# # SQLite: "datetime(created_at, '-' || (3600) || ' seconds')" + +# Module for MySQL-based adapters (Trilogy, Mysql2, etc.) +module MysqlDateArithmetic + # Generates SQL for subtracting seconds from a date/time column in MySQL. + # + # @param date_column [String] The date/time column or expression + # @param seconds_expression [String] SQL expression that evaluates to number of seconds + # @return [String] MySQL DATE_SUB expression + # + # Example: + # date_subtract("last_active_at", "COALESCE(auto_postpone_period, 3600)") + # # => "DATE_SUB(last_active_at, INTERVAL COALESCE(auto_postpone_period, 3600) SECOND)" + def date_subtract(date_column, seconds_expression) + "DATE_SUB(#{date_column}, INTERVAL #{seconds_expression} SECOND)" + end +end + +# Module for SQLite adapter +module SqliteDateArithmetic + # Generates SQL for subtracting seconds from a date/time column in SQLite. + # + # @param date_column [String] The date/time column or expression + # @param seconds_expression [String] SQL expression that evaluates to number of seconds + # @return [String] SQLite datetime expression + # + # Example: + # date_subtract("last_active_at", "COALESCE(auto_postpone_period, 3600)") + # # => "datetime(last_active_at, '-' || (COALESCE(auto_postpone_period, 3600)) || ' seconds')" + def date_subtract(date_column, seconds_expression) + "datetime(#{date_column}, '-' || (#{seconds_expression}) || ' seconds')" + end +end + +ActiveSupport.on_load(:active_record) do + # Prepend MySQL date arithmetic to AbstractMysqlAdapter (covers Trilogy, Mysql2, etc.) + if defined?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter) + ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlDateArithmetic) + end + + # Prepend SQLite date arithmetic to SQLite3Adapter + if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) + ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(SqliteDateArithmetic) + end +end From a4850fbe73ff53a036c3425ed4e5de38590b2d91 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 20 Nov 2025 16:52:34 +0000 Subject: [PATCH 05/55] Remove adapter specific logic from searchable --- app/models/concerns/searchable.rb | 70 +++---------------------------- app/models/search/record.rb | 39 +++++++++++++++++ 2 files changed, 45 insertions(+), 64 deletions(-) diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 017c1d500..15b01ad09 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -13,79 +13,21 @@ module Searchable private def create_in_search_index - search_class = Search::Record.for_account(account_id) - - if Search::Record.sqlite? - # SQLite: create with unstemmed content, FTS5 handles stemming - record = search_class.create!(search_record_attributes) - upsert_to_fts5(record.id) - else - # MySQL: create with stemmed content for FULLTEXT search - attrs = search_record_attributes.merge( - title: Search::Stemmer.stem(search_record_attributes[:title]), - content: Search::Stemmer.stem(search_record_attributes[:content]) - ) - search_class.create!(attrs) - end + search_record_class.create!(search_record_attributes) end def update_in_search_index - search_class = Search::Record.for_account(account_id) - - if Search::Record.sqlite? - # SQLite: find or create record, then upsert to FTS5 - record = search_class.find_or_initialize_by( - searchable_type: self.class.name, - searchable_id: id - ) - record.assign_attributes(search_record_attributes) - record.save! - upsert_to_fts5(record.id) - else - # MySQL: use upsert_all with stemmed content - attrs = search_record_attributes.merge( - id: ActiveRecord::Type::Uuid.generate, - title: Search::Stemmer.stem(search_record_attributes[:title]), - content: Search::Stemmer.stem(search_record_attributes[:content]) - ) - search_class.upsert_all( - [ attrs ], - update_only: [ :card_id, :board_id, :title, :content, :created_at ] - ) + search_record_class.find_or_initialize_by(searchable_type: self.class.name, searchable_id: id).tap do |record| + record.update!(search_record_attributes) end end def remove_from_search_index - search_class = Search::Record.for_account(account_id) - record = search_class.find_by(searchable_type: self.class.name, searchable_id: id) - - if record - # For SQLite, delete from FTS5 first - if Search::Record.sqlite? - delete_from_fts5(record.id) - end - - record.delete - end + search_record_class.find_by(searchable_type: self.class.name, searchable_id: id)&.destroy end - def upsert_to_fts5(record_id) - # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically - title = search_title - content = search_content - - # Note: FTS5 virtual tables don't work properly with bound parameters in SQLite, - # so we need to use string interpolation with proper quoting - conn = ActiveRecord::Base.connection - sql = "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{record_id}, #{conn.quote(title)}, #{conn.quote(content)})" - conn.execute(sql) - end - - def delete_from_fts5(record_id) - # Note: Use string interpolation for consistency (rowid is always an integer, so safe) - ActiveRecord::Base.connection.execute( - "DELETE FROM search_records_fts WHERE rowid = #{record_id}" - ) + def search_record_class + Search::Record.for_account(account_id) end def search_record_attributes diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 3f08ddfea..12586b2a3 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -22,11 +22,31 @@ class Search::Record < ApplicationRecord validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true + before_save :stem_content, unless: :sqlite? + after_save :upsert_to_fts5_table, if: :sqlite? + after_destroy :delete_from_fts5_table, if: :sqlite? + class << self def sqlite? connection.adapter_name == "SQLite" end + def upsert_to_fts5(record_id, title:, content:) + # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically + # Note: FTS5 virtual tables don't work properly with bound parameters in SQLite, + # so we need to use string interpolation with proper quoting + conn = connection + sql = "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{record_id}, #{conn.quote(title)}, #{conn.quote(content)})" + conn.execute(sql) + end + + def delete_from_fts5(record_id) + # Note: Use string interpolation for consistency (rowid is always an integer, so safe) + connection.execute( + "DELETE FROM search_records_fts WHERE rowid = #{record_id}" + ) + end + def for_account(account_id) if sqlite? # SQLite uses a single table, no sharding - use a non-abstract subclass @@ -152,6 +172,10 @@ class Search::Record < ApplicationRecord end end + def sqlite? + self.class.sqlite? + end + def source searchable_type == "Comment" ? searchable : card end @@ -197,6 +221,21 @@ class Search::Record < ApplicationRecord end private + def stem_content + # MySQL: stem content for FULLTEXT search + self.title = Search::Stemmer.stem(title) if title_changed? + self.content = Search::Stemmer.stem(content) if content_changed? + end + + def upsert_to_fts5_table + # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically + self.class.upsert_to_fts5(id, title: title, content: content) + end + + def delete_from_fts5_table + self.class.delete_from_fts5(id) + end + def highlight(text, show:) if text.present? && attribute?(:query) highlighter = Search::Highlighter.new(query) From 19b0e25eacd50f8bde277ac001e22d8d91c17866 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 20 Nov 2025 17:37:44 +0000 Subject: [PATCH 06/55] Split sqlite/mysql record stuff --- app/models/search/record.rb | 204 +----------------------- app/models/search/record/sqlite.rb | 93 +++++++++++ app/models/search/record/trilogy.rb | 73 +++++++++ lib/tasks/search.rake | 2 +- test/test_helpers/search_test_helper.rb | 4 +- 5 files changed, 172 insertions(+), 204 deletions(-) create mode 100644 app/models/search/record/sqlite.rb create mode 100644 app/models/search/record/trilogy.rb diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 12586b2a3..81f59f96c 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -1,18 +1,5 @@ class Search::Record < ApplicationRecord - self.abstract_class = true - - SHARD_COUNT = 16 - - # MySQL sharded table classes - SHARD_CLASSES = SHARD_COUNT.times.map do |shard_id| - Class.new(self) do - self.table_name = "search_records_#{shard_id}" - - def self.name - "Search::Record" - end - end - end.freeze + include const_get(connection.adapter_name) belongs_to :searchable, polymorphic: true, optional: true belongs_to :card, optional: true @@ -22,94 +9,10 @@ class Search::Record < ApplicationRecord validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true - before_save :stem_content, unless: :sqlite? - after_save :upsert_to_fts5_table, if: :sqlite? - after_destroy :delete_from_fts5_table, if: :sqlite? - class << self - def sqlite? - connection.adapter_name == "SQLite" - end - - def upsert_to_fts5(record_id, title:, content:) - # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically - # Note: FTS5 virtual tables don't work properly with bound parameters in SQLite, - # so we need to use string interpolation with proper quoting - conn = connection - sql = "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{record_id}, #{conn.quote(title)}, #{conn.quote(content)})" - conn.execute(sql) - end - - def delete_from_fts5(record_id) - # Note: Use string interpolation for consistency (rowid is always an integer, so safe) - connection.execute( - "DELETE FROM search_records_fts WHERE rowid = #{record_id}" - ) - end - - def for_account(account_id) - if sqlite? - # SQLite uses a single table, no sharding - use a non-abstract subclass - @sqlite_class ||= Class.new(self) do - self.abstract_class = false - self.table_name = "search_records" - - # Override the UUID id attribute from ApplicationRecord - # SQLite uses integer auto-increment primary key (no default) - attribute :id, :integer, default: nil - - def self.name - "Search::Record" - end - end - else - # MySQL uses sharded tables - SHARD_CLASSES[shard_id_for_account(account_id)] - end - end - - def shard_id_for_account(account_id) - Zlib.crc32(account_id.to_s) % SHARD_COUNT - end - def card_join "INNER JOIN #{table_name} ON #{table_name}.card_id = cards.id" end - - # Convert MySQL BOOLEAN MODE syntax to SQLite FTS5 syntax - def convert_to_fts5_query(query) - query = query.to_s.dup - - # Handle quoted phrases - these work the same in both - # Extract them to protect during processing - phrases = [] - query.gsub!(/"([^"]+)"/) do - phrases << $1 - "__PHRASE_#{phrases.length - 1}__" - end - - # Split into tokens - tokens = query.split(/\s+/) - - processed_tokens = tokens.map do |token| - if token.start_with?("__PHRASE_") - # Restore phrase - idx = token[/\d+/].to_i - "\"#{phrases[idx]}\"" - elsif token.start_with?("+") - # Remove + prefix (FTS5 ANDs by default) - token[1..] - elsif token.start_with?("-") - # Convert -term to NOT term - "NOT #{token[1..]}" - else - token - end - end - - # Join with AND (FTS5 uses AND/OR/NOT operators) - processed_tokens.reject(&:blank?).join(" ") - end end scope :for_query, ->(query:, user:) do @@ -121,18 +24,7 @@ class Search::Record < ApplicationRecord end scope :matching, ->(query) do - if sqlite? - # SQLite FTS5: join on rowid for fast lookup with native highlighting - # Porter tokenizer handles stemming automatically - fts_query = convert_to_fts5_query(query) - - joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") - .where("search_records_fts MATCH ?", fts_query) - else - # MySQL FULLTEXT: manually stem query terms - stemmed_query = Search::Stemmer.stem(query) - where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", stemmed_query) - end + matching_scope(query) end scope :for_user, ->(user) do @@ -144,36 +36,7 @@ class Search::Record < ApplicationRecord .includes(:searchable, card: [ :board, :creator ]) .order(created_at: :desc) - if sqlite? - # SQLite: matching scope already selected all columns + FTS5 highlight columns - # Re-select to add query terms (ActiveRecord replaces the select list) - opening_mark = Search::Highlighter::OPENING_MARK - closing_mark = Search::Highlighter::CLOSING_MARK - ellipsis = Search::Highlighter::ELIPSIS - - relation.select( - "#{table_name}.id", - "#{table_name}.account_id", - "#{table_name}.searchable_type", - "#{table_name}.searchable_id", - "#{table_name}.card_id", - "#{table_name}.board_id", - "#{table_name}.title", - "#{table_name}.content", - "#{table_name}.created_at", - "highlight(search_records_fts, 0, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_title", - "highlight(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_content", - "snippet(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}, #{connection.quote(ellipsis)}, 20) AS content_snippet", - "#{connection.quote(query.terms)} AS query" - ) - else - # MySQL: select specific columns needed - relation.select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query") - end - end - - def sqlite? - self.class.sqlite? + search_scope(relation, query) end def source @@ -183,65 +46,4 @@ class Search::Record < ApplicationRecord def comment searchable if searchable_type == "Comment" end - - def card_title - if card_id - if self.class.sqlite? && attribute?(:highlighted_title) - # Use FTS5 native highlighting (already HTML-safe from FTS5) - highlighted_title.html_safe - else - # MySQL: use Ruby highlighter - highlight(card.title, show: :full) - end - end - end - - def card_description - if card_id - if self.class.sqlite? && attribute?(:content_snippet) - # Use FTS5 native snippet for content (already HTML-safe from FTS5) - content_snippet.html_safe if content_snippet.present? - else - # MySQL: use Ruby highlighter - highlight(card.description.to_plain_text, show: :snippet) - end - end - end - - def comment_body - if comment - if self.class.sqlite? && attribute?(:content_snippet) - # Use FTS5 native snippet for content (already HTML-safe from FTS5) - content_snippet.html_safe if content_snippet.present? - else - # MySQL: use Ruby highlighter - highlight(comment.body.to_plain_text, show: :snippet) - end - end - end - - private - def stem_content - # MySQL: stem content for FULLTEXT search - self.title = Search::Stemmer.stem(title) if title_changed? - self.content = Search::Stemmer.stem(content) if content_changed? - end - - def upsert_to_fts5_table - # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically - self.class.upsert_to_fts5(id, title: title, content: content) - end - - def delete_from_fts5_table - self.class.delete_from_fts5(id) - end - - def highlight(text, show:) - if text.present? && attribute?(:query) - highlighter = Search::Highlighter.new(query) - show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) - else - text - end - end end diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb new file mode 100644 index 000000000..eb2c9639b --- /dev/null +++ b/app/models/search/record/sqlite.rb @@ -0,0 +1,93 @@ +module Search::Record::SQLite + extend ActiveSupport::Concern + + included do + self.table_name = "search_records" + + # Override the UUID id attribute from ApplicationRecord + # SQLite uses integer auto-increment primary key (no default) + attribute :id, :integer, default: nil + + after_save :upsert_to_fts5_table + after_destroy :delete_from_fts5_table + end + + class_methods do + def for_account(account_id) + # SQLite uses a single table, no sharding + self + end + + def matching_scope(query) + # SQLite FTS5: join on rowid for fast lookup with native highlighting + # Porter tokenizer handles stemming automatically + joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") + .where("search_records_fts MATCH ?", query) + end + + def search_scope(relation, query) + # SQLite: matching scope already selected all columns + FTS5 highlight columns + # Re-select to add query terms (ActiveRecord replaces the select list) + opening_mark = Search::Highlighter::OPENING_MARK + closing_mark = Search::Highlighter::CLOSING_MARK + ellipsis = Search::Highlighter::ELIPSIS + + relation.select( + "#{table_name}.id", + "#{table_name}.account_id", + "#{table_name}.searchable_type", + "#{table_name}.searchable_id", + "#{table_name}.card_id", + "#{table_name}.board_id", + "#{table_name}.title", + "#{table_name}.content", + "#{table_name}.created_at", + "highlight(search_records_fts, 0, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_title", + "highlight(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_content", + "snippet(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}, #{connection.quote(ellipsis)}, 20) AS content_snippet", + "#{connection.quote(query.terms)} AS query" + ) + end + end + + def card_title + if card_id + # Use FTS5 native highlighting (already HTML-safe from FTS5) + # Fall back to card.title if highlight is nil (e.g., for comment matches where title is NULL) + if has_attribute?(:highlighted_title) && highlighted_title.present? + highlighted_title.html_safe + else + card.title + end + end + end + + def card_description + if card_id && has_attribute?(:content_snippet) && content_snippet.present? + # Use FTS5 native snippet for content (already HTML-safe from FTS5) + content_snippet.html_safe + end + end + + def comment_body + if comment && has_attribute?(:content_snippet) && content_snippet.present? + # Use FTS5 native snippet for content (already HTML-safe from FTS5) + content_snippet.html_safe + end + end + + private + def upsert_to_fts5_table + # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically + # Note: FTS5 virtual tables don't work properly with bound parameters in SQLite, + # so we need to use string interpolation with proper quoting + self.class.connection.execute( + "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{id}, #{self.class.connection.quote(title)}, #{self.class.connection.quote(content)})" + ) + end + + def delete_from_fts5_table + # Note: Use string interpolation for consistency (rowid is always an integer, so safe) + self.class.connection.execute("DELETE FROM search_records_fts WHERE rowid = #{id}") + end +end diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb new file mode 100644 index 000000000..b55deeb0b --- /dev/null +++ b/app/models/search/record/trilogy.rb @@ -0,0 +1,73 @@ +module Search::Record::Trilogy + extend ActiveSupport::Concern + + SHARD_COUNT = 16 + + included do + self.abstract_class = true + + before_save :stem_content + end + + class_methods do + def shard_classes + @shard_classes ||= SHARD_COUNT.times.map do |shard_id| + Class.new(Search::Record) do + self.table_name = "search_records_#{shard_id}" + + def self.name + "Search::Record" + end + end + end.freeze + end + + def for_account(account_id) + # MySQL uses sharded tables + shard_classes[shard_id_for_account(account_id)] + end + + def shard_id_for_account(account_id) + Zlib.crc32(account_id.to_s) % SHARD_COUNT + end + + def matching_scope(query) + # MySQL FULLTEXT: manually stem query terms + stemmed_query = Search::Stemmer.stem(query) + where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", stemmed_query) + end + + def search_scope(relation, query) + # MySQL: select specific columns needed + relation.select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query") + end + end + + def card_title + highlight(card.title, show: :full) if card_id + end + + def card_description + highlight(card.description.to_plain_text, show: :snippet) if card_id + end + + def comment_body + highlight(comment.body.to_plain_text, show: :snippet) if comment + end + + private + def stem_content + # MySQL: stem content for FULLTEXT search + self.title = Search::Stemmer.stem(title) if title_changed? + self.content = Search::Stemmer.stem(content) if content_changed? + end + + def highlight(text, show:) + if text.present? && attribute?(:query) + highlighter = Search::Highlighter.new(query) + show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) + else + text + end + end +end diff --git a/lib/tasks/search.rake b/lib/tasks/search.rake index 0bb3591ee..c1ec0e28a 100644 --- a/lib/tasks/search.rake +++ b/lib/tasks/search.rake @@ -2,7 +2,7 @@ namespace :search do desc "Reindex all cards and comments in the search index" task reindex: :environment do puts "Clearing search records..." - Search::Record::SHARD_COUNT.times do |shard_id| + Search::Record::Trilogy::SHARD_COUNT.times do |shard_id| ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") end diff --git a/test/test_helpers/search_test_helper.rb b/test/test_helpers/search_test_helper.rb index 7132c2ed2..78b113781 100644 --- a/test/test_helpers/search_test_helper.rb +++ b/test/test_helpers/search_test_helper.rb @@ -28,11 +28,11 @@ module SearchTestHelper private def clear_search_records - if Search::Record.sqlite? + if ActiveRecord::Base.connection.adapter_name == "SQLite" ActiveRecord::Base.connection.execute("DELETE FROM search_records") ActiveRecord::Base.connection.execute("DELETE FROM search_records_fts") else - Search::Record::SHARD_COUNT.times do |shard_id| + Search::Record::Trilogy::SHARD_COUNT.times do |shard_id| ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") end end From 5edb0a2075390cf37cb429c1ea29b67dbcad0be9 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 08:49:33 +0000 Subject: [PATCH 07/55] Update schema --- ...ndex_to_card_activity_spikes_on_card_id.rb | 18 +++++---- db/schema.rb | 2 +- db/schema_sqlite.rb | 40 +------------------ 3 files changed, 13 insertions(+), 47 deletions(-) diff --git a/db/migrate/20251120203100_add_unique_index_to_card_activity_spikes_on_card_id.rb b/db/migrate/20251120203100_add_unique_index_to_card_activity_spikes_on_card_id.rb index 850c8e604..b9f6ab3d4 100644 --- a/db/migrate/20251120203100_add_unique_index_to_card_activity_spikes_on_card_id.rb +++ b/db/migrate/20251120203100_add_unique_index_to_card_activity_spikes_on_card_id.rb @@ -1,13 +1,15 @@ class AddUniqueIndexToCardActivitySpikesOnCardId < ActiveRecord::Migration[8.2] def change - reversible do |dir| - dir.up do - execute <<-SQL - DELETE s1 FROM card_activity_spikes s1 - INNER JOIN card_activity_spikes s2 - WHERE s1.card_id = s2.card_id - AND s1.updated_at < s2.updated_at - SQL + if ActiveRecord::Base.connection.adapter_name != "SQLite" + reversible do |dir| + dir.up do + execute <<-SQL + DELETE s1 FROM card_activity_spikes s1 + INNER JOIN card_activity_spikes s2 + WHERE s1.card_id = s2.card_id + AND s1.updated_at < s2.updated_at + SQL + end end end diff --git a/db/schema.rb b/db/schema.rb index 6164fd37f..35dd2e00d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2025_11_20_110206) do +ActiveRecord::Schema[8.2].define(version: 2025_11_20_203100) do create_table "accesses", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb index 387e0aa06..76c143481 100644 --- a/db/schema_sqlite.rb +++ b/db/schema_sqlite.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2025_11_20_110206) do +ActiveRecord::Schema[8.2].define(version: 2025_11_20_203100) do create_table "accesses", id: :uuid, force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false @@ -150,7 +150,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_20_110206) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_card_activity_spikes_on_account_id" - t.index ["card_id"], name: "index_card_activity_spikes_on_card_id" + t.index ["card_id"], name: "index_card_activity_spikes_on_card_id", unique: true end create_table "card_engagements", id: :uuid, force: :cascade do |t| @@ -537,42 +537,6 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_20_110206) do t.index ["account_id"], name: "index_webhooks_on_account_id" t.index ["board_id", "subscribed_actions"], name: "index_webhooks_on_board_id_and_subscribed_actions" end - - add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" - add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" - add_foreign_key "board_publications", "boards" - add_foreign_key "card_activity_spikes", "cards" - add_foreign_key "card_goldnesses", "cards" - add_foreign_key "card_not_nows", "cards" - add_foreign_key "card_not_nows", "users" - add_foreign_key "cards", "columns" - add_foreign_key "closures", "cards" - add_foreign_key "closures", "users" - add_foreign_key "columns", "boards" - add_foreign_key "comments", "cards" - add_foreign_key "events", "boards" - add_foreign_key "magic_links", "identities" - add_foreign_key "mentions", "users", column: "mentionee_id" - add_foreign_key "mentions", "users", column: "mentioner_id" - add_foreign_key "notification_bundles", "users" - add_foreign_key "notifications", "users" - add_foreign_key "notifications", "users", column: "creator_id" - add_foreign_key "pins", "cards" - add_foreign_key "pins", "users" - add_foreign_key "push_subscriptions", "users" - add_foreign_key "search_queries", "users" - add_foreign_key "sessions", "identities" - add_foreign_key "steps", "cards" - add_foreign_key "taggings", "cards" - add_foreign_key "taggings", "tags" - add_foreign_key "user_settings", "users" - add_foreign_key "users", "identities" - add_foreign_key "watches", "cards" - add_foreign_key "watches", "users" - add_foreign_key "webhook_delinquency_trackers", "webhooks" - add_foreign_key "webhook_deliveries", "events" - add_foreign_key "webhook_deliveries", "webhooks" - add_foreign_key "webhooks", "boards" execute "CREATE VIRTUAL TABLE search_records_fts USING fts5(\n title,\n content,\n tokenize='porter'\n )" end From ec06b7085b373c7f586592dd791835060ce35c99 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 08:51:53 +0000 Subject: [PATCH 08/55] No need to default created_at --- app/models/concerns/searchable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 15b01ad09..39e5d95f8 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -39,7 +39,7 @@ module Searchable board_id: search_board_id, title: search_title, content: search_content, - created_at: created_at || Time.current + created_at: created_at } end From 6dfb961b83d4d3c6aed1c8aeeac424d404323b21 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 08:52:55 +0000 Subject: [PATCH 09/55] Delegate to_s to terms --- app/models/search/query.rb | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/app/models/search/query.rb b/app/models/search/query.rb index 7d96d5170..5fe0829e4 100644 --- a/app/models/search/query.rb +++ b/app/models/search/query.rb @@ -5,6 +5,8 @@ class Search::Query < ApplicationRecord validates :terms, presence: true before_validation :sanitize_terms + delegate :to_s, to: :terms + class << self def wrap(query) if query.is_a?(self) @@ -15,13 +17,6 @@ class Search::Query < ApplicationRecord end end - def to_s - # Return unstemmed terms - each adapter handles stemming appropriately: - # - SQLite: FTS5 Porter tokenizer stems automatically - # - MySQL: Search::Record.matching applies stemming - terms.to_s - end - private def sanitize_terms self.terms = sanitize(terms) From ca17193620b00b7d7d68cac0e179fd921a033388 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 08:53:52 +0000 Subject: [PATCH 10/55] Belongs_to not optional --- app/models/search/record.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 81f59f96c..8b54a7c33 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -1,8 +1,8 @@ class Search::Record < ApplicationRecord include const_get(connection.adapter_name) - belongs_to :searchable, polymorphic: true, optional: true - belongs_to :card, optional: true + belongs_to :searchable, polymorphic: true + belongs_to :card # Virtual attributes from search query attribute :query, :string From 62d102578f668276a13b0a9766df7d35c6727268 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 08:55:23 +0000 Subject: [PATCH 11/55] Table name already correct --- app/models/search/record/sqlite.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index eb2c9639b..108004a7d 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -2,10 +2,8 @@ module Search::Record::SQLite extend ActiveSupport::Concern included do - self.table_name = "search_records" - # Override the UUID id attribute from ApplicationRecord - # SQLite uses integer auto-increment primary key (no default) + # FTS tables require integer rowids attribute :id, :integer, default: nil after_save :upsert_to_fts5_table From 0e11a139a501f2a34e3bf89f3c7c22e170610645 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 08:56:33 +0000 Subject: [PATCH 12/55] Move quoting --- app/models/search/record/sqlite.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index 108004a7d..9dffdc7ff 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -26,9 +26,9 @@ module Search::Record::SQLite def search_scope(relation, query) # SQLite: matching scope already selected all columns + FTS5 highlight columns # Re-select to add query terms (ActiveRecord replaces the select list) - opening_mark = Search::Highlighter::OPENING_MARK - closing_mark = Search::Highlighter::CLOSING_MARK - ellipsis = Search::Highlighter::ELIPSIS + opening_mark = connection.quote(Search::Highlighter::OPENING_MARK) + closing_mark = connection.quote(Search::Highlighter::CLOSING_MARK) + ellipsis = connection.quote(Search::Highlighter::ELIPSIS) relation.select( "#{table_name}.id", @@ -40,9 +40,9 @@ module Search::Record::SQLite "#{table_name}.title", "#{table_name}.content", "#{table_name}.created_at", - "highlight(search_records_fts, 0, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_title", - "highlight(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}) AS highlighted_content", - "snippet(search_records_fts, 1, #{connection.quote(opening_mark)}, #{connection.quote(closing_mark)}, #{connection.quote(ellipsis)}, 20) AS content_snippet", + "highlight(search_records_fts, 0, #{opening_mark}, #{closing_mark}) AS highlighted_title", + "highlight(search_records_fts, 1, #{opening_mark}, #{closing_mark}) AS highlighted_content", + "snippet(search_records_fts, 1, #{opening_mark}, #{closing_mark}, #{ellipsis}, 20) AS content_snippet", "#{connection.quote(query.terms)} AS query" ) end From 4f6ec19491af93f7c217cde9da84a885275df921 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 08:59:28 +0000 Subject: [PATCH 13/55] Drop comments --- app/models/search/record/sqlite.rb | 13 ------------- app/models/search/record/trilogy.rb | 4 ---- 2 files changed, 17 deletions(-) diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index 9dffdc7ff..5566963e3 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -12,20 +12,15 @@ module Search::Record::SQLite class_methods do def for_account(account_id) - # SQLite uses a single table, no sharding self end def matching_scope(query) - # SQLite FTS5: join on rowid for fast lookup with native highlighting - # Porter tokenizer handles stemming automatically joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") .where("search_records_fts MATCH ?", query) end def search_scope(relation, query) - # SQLite: matching scope already selected all columns + FTS5 highlight columns - # Re-select to add query terms (ActiveRecord replaces the select list) opening_mark = connection.quote(Search::Highlighter::OPENING_MARK) closing_mark = connection.quote(Search::Highlighter::CLOSING_MARK) ellipsis = connection.quote(Search::Highlighter::ELIPSIS) @@ -50,8 +45,6 @@ module Search::Record::SQLite def card_title if card_id - # Use FTS5 native highlighting (already HTML-safe from FTS5) - # Fall back to card.title if highlight is nil (e.g., for comment matches where title is NULL) if has_attribute?(:highlighted_title) && highlighted_title.present? highlighted_title.html_safe else @@ -62,30 +55,24 @@ module Search::Record::SQLite def card_description if card_id && has_attribute?(:content_snippet) && content_snippet.present? - # Use FTS5 native snippet for content (already HTML-safe from FTS5) content_snippet.html_safe end end def comment_body if comment && has_attribute?(:content_snippet) && content_snippet.present? - # Use FTS5 native snippet for content (already HTML-safe from FTS5) content_snippet.html_safe end end private def upsert_to_fts5_table - # Use raw unstemmed text - FTS5 Porter tokenizer handles stemming automatically - # Note: FTS5 virtual tables don't work properly with bound parameters in SQLite, - # so we need to use string interpolation with proper quoting self.class.connection.execute( "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{id}, #{self.class.connection.quote(title)}, #{self.class.connection.quote(content)})" ) end def delete_from_fts5_table - # Note: Use string interpolation for consistency (rowid is always an integer, so safe) self.class.connection.execute("DELETE FROM search_records_fts WHERE rowid = #{id}") end end diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index b55deeb0b..e55c895cd 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -23,7 +23,6 @@ module Search::Record::Trilogy end def for_account(account_id) - # MySQL uses sharded tables shard_classes[shard_id_for_account(account_id)] end @@ -32,13 +31,11 @@ module Search::Record::Trilogy end def matching_scope(query) - # MySQL FULLTEXT: manually stem query terms stemmed_query = Search::Stemmer.stem(query) where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", stemmed_query) end def search_scope(relation, query) - # MySQL: select specific columns needed relation.select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query") end end @@ -57,7 +54,6 @@ module Search::Record::Trilogy private def stem_content - # MySQL: stem content for FULLTEXT search self.title = Search::Stemmer.stem(title) if title_changed? self.content = Search::Stemmer.stem(content) if content_changed? end From f665b5a1fa62f28f490747aa022cf1df3fb5aefb Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 09:02:05 +0000 Subject: [PATCH 14/55] No need for default schema --- config/database.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/database.yml b/config/database.yml index 39396ffb3..b7f1d3297 100644 --- a/config/database.yml +++ b/config/database.yml @@ -38,7 +38,6 @@ development: <<: *default database: fizzy_development port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %> - schema_dump: schema.rb replica: <<: *default database: fizzy_development @@ -75,7 +74,6 @@ test: <<: *default database: fizzy_test port: <%= ENV.fetch "FIZZY_DB_PORT", 33380 %> - schema_dump: schema.rb replica: <<: *default database: fizzy_test From 9c8d5e40add70d3129bce5d94dcd2c69a9a5c1b4 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 09:03:52 +0000 Subject: [PATCH 15/55] Always apply patches on load --- config/initializers/sqlite_compatibility.rb | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/config/initializers/sqlite_compatibility.rb b/config/initializers/sqlite_compatibility.rb index b0d192912..50d44c28b 100644 --- a/config/initializers/sqlite_compatibility.rb +++ b/config/initializers/sqlite_compatibility.rb @@ -35,18 +35,10 @@ module SQLiteCompatibility end end -# Apply the prepends - both in on_load callback and immediately -def apply_sqlite_compatibility +# Also run when ActiveRecord loads (for cases where it loads later) +ActiveSupport.on_load(:active_record) do if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) ActiveRecord::ConnectionAdapters::TableDefinition.prepend(SQLiteCompatibility::SQLiteTableDefinitionCompatibility) ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(SQLiteCompatibility::SQLiteSchemaStatementCompatibility) end end - -# Run immediately if ActiveRecord is already loaded -apply_sqlite_compatibility - -# Also run when ActiveRecord loads (for cases where it loads later) -ActiveSupport.on_load(:active_record) do - apply_sqlite_compatibility -end From 33ef482189d2a583a13ca7a4b729066fb60f1deb Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 09:05:24 +0000 Subject: [PATCH 16/55] Patch dumper on load --- config/initializers/sqlite_schema_dumper.rb | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/config/initializers/sqlite_schema_dumper.rb b/config/initializers/sqlite_schema_dumper.rb index c1a43417c..b3d76b186 100644 --- a/config/initializers/sqlite_schema_dumper.rb +++ b/config/initializers/sqlite_schema_dumper.rb @@ -17,17 +17,8 @@ module SQLiteFTS5SchemaDumperFix end end -if ENV.fetch("DATABASE_ADAPTER", "mysql") == "sqlite" - ActiveSupport.on_load(:active_record) do - # Ensure schema dumper is loaded - begin - require 'active_record/connection_adapters/sqlite3/schema_dumper' - rescue LoadError - # SQLite3 adapter not available - end - - if defined?(ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper) - ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper.prepend(SQLiteFTS5SchemaDumperFix) - end +ActiveSupport.on_load(:active_record) do + if defined?(ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper) + ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper.prepend(SQLiteFTS5SchemaDumperFix) end end From dd18b6641dfd98c176cc86c6c2684a74af1e0643 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 09:08:52 +0000 Subject: [PATCH 17/55] Simplify patching --- config/initializers/uuid_primary_keys.rb | 31 +++++++----------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/config/initializers/uuid_primary_keys.rb b/config/initializers/uuid_primary_keys.rb index 1b41315ec..698da184f 100644 --- a/config/initializers/uuid_primary_keys.rb +++ b/config/initializers/uuid_primary_keys.rb @@ -1,15 +1,5 @@ # Automatically use UUID type for all binary(16) columns -# Define schema dumper module outside on_load so it can be prepended -module SchemaDumperUuidType - # Map binary(16) and blob(16) columns to :uuid type in schema.rb - def schema_type(column) - return :uuid if column.sql_type == "binary(16)" - return :uuid if column.sql_type == "blob(16)" - super - end -end - ActiveSupport.on_load(:active_record) do module MysqlUuidAdapter # Add UUID to MySQL's native database types @@ -32,7 +22,6 @@ ActiveSupport.on_load(:active_record) do end module SqliteUuidAdapter - # Add UUID to SQLite's native database types (instance method) def native_database_types @native_database_types_with_uuid ||= super.merge(uuid: { name: "blob", limit: 16 }) end @@ -77,17 +66,15 @@ ActiveSupport.on_load(:active_record) do end end - # Ensure schema dumper classes are loaded before prepending - begin - require 'active_record/connection_adapters/mysql/schema_dumper' - rescue LoadError - # MySQL adapter not available - end - - begin - require 'active_record/connection_adapters/sqlite3/schema_dumper' - rescue LoadError - # SQLite3 adapter not available + module SchemaDumperUuidType + # Map binary(16) and blob(16) columns to :uuid type in schema.rb + def schema_type(column) + if column.sql_type == "binary(16)" || column.sql_type == "blob(16)" + :uuid + else + super + end + end end if defined?(ActiveRecord::ConnectionAdapters::MySQL::SchemaDumper) From b4e3d08689639957179de61fd41b93b51731128a Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 09:09:15 +0000 Subject: [PATCH 18/55] Add long back to dumped schema, need to check why it wants to change it though --- db/schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 35dd2e00d..7130719ae 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -46,7 +46,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_20_203100) do create_table "action_text_rich_texts", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.uuid "account_id", null: false - t.text "body" + t.text "body", size: :long t.datetime "created_at", null: false t.string "name", null: false t.uuid "record_id", null: false From 3df9961b611b553b82e4c3f25806f92b9f707f93 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 09:10:38 +0000 Subject: [PATCH 19/55] Ignore sqlite schema for rubocop --- .rubocop.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.rubocop.yml b/.rubocop.yml index 88329d008..16773cd4c 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -10,3 +10,4 @@ inherit_gem: { rubocop-rails-omakase: rubocop.yml } AllCops: Exclude: - 'db/migrate/**/*' + - 'db/schema*.rb' From f24f959d190bd4b1dd46659ace665da9e4ed3d7a Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 09:14:53 +0000 Subject: [PATCH 20/55] Add brakeman ignores --- config/brakeman.ignore | 48 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/config/brakeman.ignore b/config/brakeman.ignore index 6c2eb7f67..511c6a3a9 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -1,5 +1,28 @@ { "ignored_warnings": [ + { + "warning_type": "SQL Injection", + "warning_code": 0, + "fingerprint": "4ea2e6d704b817af1d896dcdf148a89da8b9e428b3327497631ef8d9ed587307", + "check_name": "SQL", + "message": "Possible SQL injection", + "file": "app/models/card/entropic.rb", + "line": 19, + "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", + "code": "active.joins(:board => :account).left_outer_joins(:board => :entropy).joins(\"LEFT OUTER JOIN entropies AS account_entropies ON account_entropies.account_id = accounts.id AND account_entropies.container_type = 'Account' AND account_entropies.container_id = accounts.id\").where(\"last_active_at > #{connection.date_subtract(\"?\", \"COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period)\")}\", Time.now)", + "render_path": null, + "location": { + "type": "method", + "class": "Card::Entropic", + "method": null + }, + "user_input": "connection.date_subtract(\"?\", \"COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period)\")", + "confidence": "Weak", + "cwe_id": [ + 89 + ], + "note": "No user input allowed" + }, { "warning_type": "Mass Assignment", "warning_code": 70, @@ -45,7 +68,30 @@ 470 ], "note": "" + }, + { + "warning_type": "SQL Injection", + "warning_code": 0, + "fingerprint": "bbd8fe3cbbc6b393df770d3142d6fa46e2b9441b21f48a52175211acaae4efad", + "check_name": "SQL", + "message": "Possible SQL injection", + "file": "app/models/card/entropic.rb", + "line": 10, + "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", + "code": "active.joins(:board => :account).left_outer_joins(:board => :entropy).joins(\"LEFT OUTER JOIN entropies AS account_entropies ON account_entropies.account_id = accounts.id AND account_entropies.container_type = 'Account' AND account_entropies.container_id = accounts.id\").where(\"last_active_at <= #{connection.date_subtract(\"?\", \"COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period)\")}\", Time.now)", + "render_path": null, + "location": { + "type": "method", + "class": "Card::Entropic", + "method": null + }, + "user_input": "connection.date_subtract(\"?\", \"COALESCE(entropies.auto_postpone_period, account_entropies.auto_postpone_period)\")", + "confidence": "Weak", + "cwe_id": [ + 89 + ], + "note": "No user input allowed" } ], - "brakeman_version": "7.1.1" + "brakeman_version": "7.1.0" } From 2e4d0b38dbc2edecd681b175384e38ccd1a0c07f Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 11:42:07 +0000 Subject: [PATCH 21/55] Fix matching scope after merge --- app/models/search/record.rb | 4 ++-- app/models/search/record/sqlite.rb | 2 +- app/models/search/record/trilogy.rb | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/models/search/record.rb b/app/models/search/record.rb index dc53d3095..5f1f19bc2 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -23,8 +23,8 @@ class Search::Record < ApplicationRecord end end - scope :matching, ->(query) do - matching_scope(query) + scope :matching, ->(query, account_id) do + matching_scope(query, account_id) end scope :for_user, ->(user) do diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index 5566963e3..9517f9219 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -15,7 +15,7 @@ module Search::Record::SQLite self end - def matching_scope(query) + def matching_scope(query, account_id) joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") .where("search_records_fts MATCH ?", query) end diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index e55c895cd..c4520b7f2 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -30,9 +30,11 @@ module Search::Record::Trilogy Zlib.crc32(account_id.to_s) % SHARD_COUNT end - def matching_scope(query) + def matching_scope(query, account_id) stemmed_query = Search::Stemmer.stem(query) - where("MATCH(#{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", stemmed_query) + account_key = "account#{account_id}" + full_query = "+#{account_key} +(#{stemmed_query})" + where("MATCH(#{table_name}.account_key, #{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", full_query) end def search_scope(relation, query) From e76f159f05f1a6b058a268104ee7941f50c180af Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 11:44:37 +0000 Subject: [PATCH 22/55] Gate search_record shard migrations on adapter type --- .../20251121092508_add_account_key_to_search_records.rb | 4 ++++ ...2416_remove_old_fulltext_indexes_from_search_records.rb | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/db/migrate/20251121092508_add_account_key_to_search_records.rb b/db/migrate/20251121092508_add_account_key_to_search_records.rb index 8960f56bc..83aaf1988 100644 --- a/db/migrate/20251121092508_add_account_key_to_search_records.rb +++ b/db/migrate/20251121092508_add_account_key_to_search_records.rb @@ -1,5 +1,7 @@ class AddAccountKeyToSearchRecords < ActiveRecord::Migration[8.2] def up + return if ActiveRecord::Base.connection.adapter_name == "SQLite" + 16.times do |shard_id| table_name = "search_records_#{shard_id}" @@ -9,6 +11,8 @@ class AddAccountKeyToSearchRecords < ActiveRecord::Migration[8.2] end def down + return if ActiveRecord::Base.connection.adapter_name == "SQLite" + 16.times do |shard_id| table_name = "search_records_#{shard_id}" diff --git a/db/migrate/20251121112416_remove_old_fulltext_indexes_from_search_records.rb b/db/migrate/20251121112416_remove_old_fulltext_indexes_from_search_records.rb index f9d38dadf..1fbd67a28 100644 --- a/db/migrate/20251121112416_remove_old_fulltext_indexes_from_search_records.rb +++ b/db/migrate/20251121112416_remove_old_fulltext_indexes_from_search_records.rb @@ -1,14 +1,15 @@ class RemoveOldFulltextIndexesFromSearchRecords < ActiveRecord::Migration[8.2] def up - # Remove the old fulltext indexes (content, title) from all 16 search_records shards - # We're keeping the new indexes (account_key, content, title) for tenant-aware searching + return if ActiveRecord::Base.connection.adapter_name == "SQLite" + (0..15).each do |shard| remove_index "search_records_#{shard}", name: "index_search_records_#{shard}_on_content_and_title" end end def down - # Re-create the old fulltext indexes in case we need to rollback + return if ActiveRecord::Base.connection.adapter_name == "SQLite" + (0..15).each do |shard| add_index "search_records_#{shard}", [ :content, :title ], type: :fulltext, name: "index_search_records_#{shard}_on_content_and_title" end From b92aba383e38f4c02e872752c75cde2c518efe94 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 11:47:28 +0000 Subject: [PATCH 23/55] account_key is MySQL specific --- app/models/concerns/searchable.rb | 1 - app/models/search/record/trilogy.rb | 6 +++++- db/schema.rb | 15 --------------- db/schema_sqlite.rb | 2 +- 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index ebb1d7bff..39e5d95f8 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -33,7 +33,6 @@ module Searchable def search_record_attributes { account_id: account_id, - account_key: "account#{account_id}", searchable_type: self.class.name, searchable_id: id, card_id: search_card_id, diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index c4520b7f2..57a94ab2c 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -6,7 +6,7 @@ module Search::Record::Trilogy included do self.abstract_class = true - before_save :stem_content + before_save :set_account_key, :stem_content end class_methods do @@ -60,6 +60,10 @@ module Search::Record::Trilogy self.content = Search::Stemmer.stem(content) if content_changed? end + def set_account_key + self.account_key = "account#{account_id}" + end + def highlight(text, show:) if text.present? && attribute?(:query) highlighter = Search::Highlighter.new(query) diff --git a/db/schema.rb b/db/schema.rb index 54bedccb4..f951af36b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -408,21 +408,6 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.index ["user_id"], name: "index_search_queries_on_user_id" end - create_table "search_records", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.uuid "account_id", null: false - t.string "account_key", null: false - t.uuid "board_id", null: false - t.uuid "card_id", null: false - t.text "content" - t.datetime "created_at", null: false - t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" - t.index ["account_id"], name: "index_search_records_on_account_id" - t.index ["account_key", "content", "title"], name: "index_search_records_on_account_key_and_content_and_title", type: :fulltext - t.index ["searchable_type", "searchable_id"], name: "index_search_records_on_searchable_type_and_searchable_id", unique: true - end - create_table "search_records_0", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.uuid "account_id", null: false t.string "account_key", default: "", null: false diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb index 76c143481..e3444f3fe 100644 --- a/db/schema_sqlite.rb +++ b/db/schema_sqlite.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2025_11_20_203100) do +ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "accesses", id: :uuid, force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false From 4008819390a535983576d45dba46122d41cfda39 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 11:54:21 +0000 Subject: [PATCH 24/55] Use table_name with Current.account to drop for_account --- app/models/search/record/trilogy.rb | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index 57a94ab2c..3620e530d 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -4,32 +4,26 @@ module Search::Record::Trilogy SHARD_COUNT = 16 included do - self.abstract_class = true - before_save :set_account_key, :stem_content end class_methods do - def shard_classes - @shard_classes ||= SHARD_COUNT.times.map do |shard_id| - Class.new(Search::Record) do - self.table_name = "search_records_#{shard_id}" - - def self.name - "Search::Record" - end - end - end.freeze - end - - def for_account(account_id) - shard_classes[shard_id_for_account(account_id)] + def table_name + if Current.account + "search_records_#{shard_id_for_account(Current.account.id)}" + else + super + end end def shard_id_for_account(account_id) Zlib.crc32(account_id.to_s) % SHARD_COUNT end + def for_account(account_id) + self + end + def matching_scope(query, account_id) stemmed_query = Search::Stemmer.stem(query) account_key = "account#{account_id}" From 01d16f96d47f47a7dab8922d441f70a77cf95922 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 12:06:32 +0000 Subject: [PATCH 25/55] Remove Search::Record.for_account Instead we'll compute the table name dynamically based on Current.account where needed. Also we'll prevent searchable records from being saved if Current.account is not set, otherwise the after commit callbacks will fail. --- app/models/card/searchable.rb | 6 ++---- app/models/concerns/searchable.rb | 17 ++++++++++------- app/models/search.rb | 2 +- app/models/search/record/sqlite.rb | 4 ---- app/models/search/record/trilogy.rb | 8 ++------ lib/tasks/search.rake | 11 +++++++++-- test/models/card/searchable_test.rb | 10 ++++++++++ test/models/comment/searchable_test.rb | 18 ++++++++++++++---- 8 files changed, 48 insertions(+), 28 deletions(-) diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb index 81e704e43..e2c0c11dc 100644 --- a/app/models/card/searchable.rb +++ b/app/models/card/searchable.rb @@ -5,10 +5,8 @@ module Card::Searchable include ::Searchable scope :mentioning, ->(query, user:) do - search_records = Search::Record.for_account(user.account_id) - - joins(search_records.card_join) - .merge(search_records.for_query(query: Search::Query.wrap(query), user: user)) + joins(Search::Record.card_join) + .merge(Search::Record.for_query(query: Search::Query.wrap(query), user: user)) end end diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 39e5d95f8..97534bbd7 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -2,6 +2,7 @@ module Searchable extend ActiveSupport::Concern included do + before_save :ensure_current_account after_create_commit :create_in_search_index after_update_commit :update_in_search_index after_destroy_commit :remove_from_search_index @@ -12,22 +13,24 @@ module Searchable end private + def ensure_current_account + unless Current.account + raise "Current.account must be set to save #{self.class.name}" + end + end + def create_in_search_index - search_record_class.create!(search_record_attributes) + Search::Record.create!(search_record_attributes) end def update_in_search_index - search_record_class.find_or_initialize_by(searchable_type: self.class.name, searchable_id: id).tap do |record| + Search::Record.find_or_initialize_by(searchable_type: self.class.name, searchable_id: id).tap do |record| record.update!(search_record_attributes) end end def remove_from_search_index - search_record_class.find_by(searchable_type: self.class.name, searchable_id: id)&.destroy - end - - def search_record_class - Search::Record.for_account(account_id) + Search::Record.find_by(searchable_type: self.class.name, searchable_id: id)&.destroy end def search_record_attributes diff --git a/app/models/search.rb b/app/models/search.rb index 264194f03..68dbbcd21 100644 --- a/app/models/search.rb +++ b/app/models/search.rb @@ -4,6 +4,6 @@ module Search end def self.results(query:, user:) - Record.for_account(user.account_id).search(query: Query.wrap(query), user: user) + Record.search(query: Query.wrap(query), user: user) end end diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index 9517f9219..989f8d834 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -11,10 +11,6 @@ module Search::Record::SQLite end class_methods do - def for_account(account_id) - self - end - def matching_scope(query, account_id) joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") .where("search_records_fts MATCH ?", query) diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index 3620e530d..e9f1d74ab 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -8,11 +8,11 @@ module Search::Record::Trilogy end class_methods do - def table_name + def compute_table_name if Current.account "search_records_#{shard_id_for_account(Current.account.id)}" else - super + raise "Current.account is not set; cannot determine shard for Search::Record" end end @@ -20,10 +20,6 @@ module Search::Record::Trilogy Zlib.crc32(account_id.to_s) % SHARD_COUNT end - def for_account(account_id) - self - end - def matching_scope(query, account_id) stemmed_query = Search::Stemmer.stem(query) account_key = "account#{account_id}" diff --git a/lib/tasks/search.rake b/lib/tasks/search.rake index c1ec0e28a..df39c3a70 100644 --- a/lib/tasks/search.rake +++ b/lib/tasks/search.rake @@ -2,17 +2,24 @@ namespace :search do desc "Reindex all cards and comments in the search index" task reindex: :environment do puts "Clearing search records..." - Search::Record::Trilogy::SHARD_COUNT.times do |shard_id| - ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") + if ActiveRecord::Base.connection.adapter_name == "SQLite" + ActiveRecord::Base.connection.execute("DELETE FROM search_records") + ActiveRecord::Base.connection.execute("DELETE FROM search_records_fts") + else + Search::Record::Trilogy::SHARD_COUNT.times do |shard_id| + ActiveRecord::Base.connection.execute("DELETE FROM search_records_#{shard_id}") + end end puts "Reindexing cards..." Card.find_each do |card| + Current.account = card.account card.reindex end puts "Reindexing comments..." Comment.find_each do |comment| + Current.account = comment.account comment.reindex end diff --git a/test/models/card/searchable_test.rb b/test/models/card/searchable_test.rb index 18845ab7a..2dc8f13a6 100644 --- a/test/models/card/searchable_test.rb +++ b/test/models/card/searchable_test.rb @@ -31,4 +31,14 @@ class Card::SearchableTest < ActiveSupport::TestCase assert_includes results, card_in_board assert_not_includes results, card_in_other_board end + + test "card requires Current.account to be set" do + Current.account = nil + + error = assert_raises(RuntimeError) do + @board.cards.create!(title: "Test card", creator: @user) + end + + assert_match(/Current.account must be set to save Card/, error.message) + end end diff --git a/test/models/comment/searchable_test.rb b/test/models/comment/searchable_test.rb index a8e52bfc2..3c7dd461f 100644 --- a/test/models/comment/searchable_test.rb +++ b/test/models/comment/searchable_test.rb @@ -10,18 +10,18 @@ class Comment::SearchableTest < ActiveSupport::TestCase test "comment search" do # Comment is indexed on create comment = @card.comments.create!(body: "searchable comment text", creator: @user) - record = Search::Record.for_account(@account.id).find_by(searchable_type: "Comment", searchable_id: comment.id) + record = Search::Record.find_by(searchable_type: "Comment", searchable_id: comment.id) assert_not_nil record # Comment is updated in index comment.update!(body: "updated text") - record = Search::Record.for_account(@account.id).find_by(searchable_type: "Comment", searchable_id: comment.id) + record = Search::Record.find_by(searchable_type: "Comment", searchable_id: comment.id) assert_match /updat/, record.content # Comment is removed from index on destroy comment_id = comment.id comment.destroy - record = Search::Record.for_account(@account.id).find_by(searchable_type: "Comment", searchable_id: comment_id) + record = Search::Record.find_by(searchable_type: "Comment", searchable_id: comment_id) assert_nil record # Finding cards via comment search @@ -34,8 +34,18 @@ class Comment::SearchableTest < ActiveSupport::TestCase # Comment stores parent card_id and board_id new_comment = @card.comments.create!(body: "test comment", creator: @user) - record = Search::Record.for_account(@account.id).find_by(searchable_type: "Comment", searchable_id: new_comment.id) + record = Search::Record.find_by(searchable_type: "Comment", searchable_id: new_comment.id) assert_equal @card.id, record.card_id assert_equal @board.id, record.board_id end + + test "comment requires Current.account to be set" do + Current.account = nil + + error = assert_raises(RuntimeError) do + @card.comments.create!(body: "Test comment", creator: @user) + end + + assert_match(/Current.account must be set to save Comment/, error.message) + end end From f91c41d75ae59d36683326d87d19e0a8ee870c75 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 12:11:15 +0000 Subject: [PATCH 26/55] Drop variables --- app/models/search/record/trilogy.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index e9f1d74ab..269bb6dab 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -21,9 +21,7 @@ module Search::Record::Trilogy end def matching_scope(query, account_id) - stemmed_query = Search::Stemmer.stem(query) - account_key = "account#{account_id}" - full_query = "+#{account_key} +(#{stemmed_query})" + full_query = "+account#{account_id} +(#{Search::Stemmer.stem(query)})" where("MATCH(#{table_name}.account_key, #{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", full_query) end From 463a5f089a5851f560aae09f4d821aedb748466c Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 12:31:25 +0000 Subject: [PATCH 27/55] Consolidate scopes and extract search_fields --- app/models/search/record.rb | 9 ++------- app/models/search/record/sqlite.rb | 23 ++++++----------------- app/models/search/record/trilogy.rb | 14 +++++++------- 3 files changed, 15 insertions(+), 31 deletions(-) diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 5f1f19bc2..52bc80522 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -23,20 +23,15 @@ class Search::Record < ApplicationRecord end end - scope :matching, ->(query, account_id) do - matching_scope(query, account_id) - end - scope :for_user, ->(user) do where(account_id: user.account_id, board_id: user.board_ids) end scope :search, ->(query:, user:) do - relation = for_query(query: query, user: user) + for_query(query: query, user: user) .includes(:searchable, card: [ :board, :creator ]) .order(created_at: :desc) - - search_scope(relation, query) + .select(:id, :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :title, :content, :created_at, *(search_fields(query))) end def source diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index 989f8d834..e3ffcf3bf 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -8,34 +8,23 @@ module Search::Record::SQLite after_save :upsert_to_fts5_table after_destroy :delete_from_fts5_table - end - class_methods do - def matching_scope(query, account_id) + scope :matching, ->(query, account_id) do joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") .where("search_records_fts MATCH ?", query) end + end - def search_scope(relation, query) + class_methods do + def search_fields(query) opening_mark = connection.quote(Search::Highlighter::OPENING_MARK) closing_mark = connection.quote(Search::Highlighter::CLOSING_MARK) ellipsis = connection.quote(Search::Highlighter::ELIPSIS) - relation.select( - "#{table_name}.id", - "#{table_name}.account_id", - "#{table_name}.searchable_type", - "#{table_name}.searchable_id", - "#{table_name}.card_id", - "#{table_name}.board_id", - "#{table_name}.title", - "#{table_name}.content", - "#{table_name}.created_at", - "highlight(search_records_fts, 0, #{opening_mark}, #{closing_mark}) AS highlighted_title", + [ "highlight(search_records_fts, 0, #{opening_mark}, #{closing_mark}) AS highlighted_title", "highlight(search_records_fts, 1, #{opening_mark}, #{closing_mark}) AS highlighted_content", "snippet(search_records_fts, 1, #{opening_mark}, #{closing_mark}, #{ellipsis}, 20) AS content_snippet", - "#{connection.quote(query.terms)} AS query" - ) + "#{connection.quote(query.terms)} AS query" ] end end diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index 269bb6dab..e598c62e5 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -5,6 +5,11 @@ module Search::Record::Trilogy included do before_save :set_account_key, :stem_content + + scope :matching, ->(query, account_id) do + full_query = "+account#{account_id} +(#{Search::Stemmer.stem(query)})" + where("MATCH(#{table_name}.account_key, #{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", full_query) + end end class_methods do @@ -20,13 +25,8 @@ module Search::Record::Trilogy Zlib.crc32(account_id.to_s) % SHARD_COUNT end - def matching_scope(query, account_id) - full_query = "+account#{account_id} +(#{Search::Stemmer.stem(query)})" - where("MATCH(#{table_name}.account_key, #{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", full_query) - end - - def search_scope(relation, query) - relation.select(:id, :searchable_type, :searchable_id, :card_id, :board_id, :account_id, :created_at, "#{connection.quote(query.terms)} AS query") + def search_fields(query) + "#{connection.quote(query.terms)} AS query" end end From 02da6dbf8f51dc7e09d86cc7a5f895c1ce41956f Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 12:36:45 +0000 Subject: [PATCH 28/55] Parameterised queries --- app/models/search/record/sqlite.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index e3ffcf3bf..6c72b3e32 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -52,12 +52,18 @@ module Search::Record::SQLite private def upsert_to_fts5_table - self.class.connection.execute( - "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (#{id}, #{self.class.connection.quote(title)}, #{self.class.connection.quote(content)})" + self.class.connection.exec_query( + "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (?, ?, ?)", + "Search::Record Upsert FTS5", + [id, title, content] ) end def delete_from_fts5_table - self.class.connection.execute("DELETE FROM search_records_fts WHERE rowid = #{id}") + self.class.connection.exec_query( + "DELETE FROM search_records_fts WHERE rowid = ?", + "Search::Record Delete FTS5", + [id] + ) end end From aebe3f97edb441fe165a01258067c97742267fb0 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 12:37:29 +0000 Subject: [PATCH 29/55] Drop parentheses --- app/models/search/record.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 52bc80522..861257858 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -31,7 +31,7 @@ class Search::Record < ApplicationRecord for_query(query: query, user: user) .includes(:searchable, card: [ :board, :creator ]) .order(created_at: :desc) - .select(:id, :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :title, :content, :created_at, *(search_fields(query))) + .select(:id, :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :title, :content, :created_at, *search_fields(query)) end def source From 6e749d2b5f7fa60d96ad372becb0747af50f897b Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 12:39:47 +0000 Subject: [PATCH 30/55] Drop for_user --- app/models/search/record.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 861257858..b958a31c7 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -17,16 +17,12 @@ class Search::Record < ApplicationRecord scope :for_query, ->(query:, user:) do if query.valid? && user.board_ids.any? - matching(query.to_s, user.account_id).for_user(user) + matching(query.to_s, user.account_id).where(account_id: user.account_id, board_id: user.board_ids) else none end end - scope :for_user, ->(user) do - where(account_id: user.account_id, board_id: user.board_ids) - end - scope :search, ->(query:, user:) do for_query(query: query, user: user) .includes(:searchable, card: [ :board, :creator ]) From ea06b2d46fe76b59b28de6b670906cbe83df7998 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 14:25:38 +0000 Subject: [PATCH 31/55] Simplify and test highlighing --- app/models/search/record/sqlite.rb | 34 +++++++++++--------- test/controllers/searches_controller_test.rb | 24 ++++++++++++++ 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index 6c72b3e32..cba6a1cde 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -6,6 +6,10 @@ module Search::Record::SQLite # FTS tables require integer rowids attribute :id, :integer, default: nil + # Virtual attributes from FTS5 functions + attribute :result_title, :string + attribute :result_content, :string + after_save :upsert_to_fts5_table after_destroy :delete_from_fts5_table @@ -21,36 +25,34 @@ module Search::Record::SQLite closing_mark = connection.quote(Search::Highlighter::CLOSING_MARK) ellipsis = connection.quote(Search::Highlighter::ELIPSIS) - [ "highlight(search_records_fts, 0, #{opening_mark}, #{closing_mark}) AS highlighted_title", - "highlight(search_records_fts, 1, #{opening_mark}, #{closing_mark}) AS highlighted_content", - "snippet(search_records_fts, 1, #{opening_mark}, #{closing_mark}, #{ellipsis}, 20) AS content_snippet", + [ "highlight(search_records_fts, 0, #{opening_mark}, #{closing_mark}) AS result_title", + "snippet(search_records_fts, 1, #{opening_mark}, #{closing_mark}, #{ellipsis}, 20) AS result_content", "#{connection.quote(query.terms)} AS query" ] end end def card_title - if card_id - if has_attribute?(:highlighted_title) && highlighted_title.present? - highlighted_title.html_safe - else - card.title - end - end + escape_fts_highlight(result_title || card.title) end def card_description - if card_id && has_attribute?(:content_snippet) && content_snippet.present? - content_snippet.html_safe - end + escape_fts_highlight(result_content) unless comment end def comment_body - if comment && has_attribute?(:content_snippet) && content_snippet.present? - content_snippet.html_safe - end + escape_fts_highlight(result_content) if comment end private + def escape_fts_highlight(html) + return nil unless html.present? + + CGI.escapeHTML(html) + .gsub(CGI.escapeHTML(Search::Highlighter::OPENING_MARK), Search::Highlighter::OPENING_MARK.html_safe) + .gsub(CGI.escapeHTML(Search::Highlighter::CLOSING_MARK), Search::Highlighter::CLOSING_MARK.html_safe) + .html_safe + end + def upsert_to_fts5_table self.class.connection.exec_query( "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (?, ?, ?)", diff --git a/test/controllers/searches_controller_test.rb b/test/controllers/searches_controller_test.rb index 12c191567..98ae5cdd3 100644 --- a/test/controllers/searches_controller_test.rb +++ b/test/controllers/searches_controller_test.rb @@ -30,6 +30,7 @@ class SearchesControllerTest < ActionDispatch::IntegrationTest assert_select "li .search__title", text: /Just haggis/, count: 2 # card title shows up in two entries assert_select "li .search__excerpt", text: /More haggis/ # one entry for the card description assert_select "li .search__excerpt--comment", text: /I love haggis/ # one entry for the comment + assert_match(/<\/span>haggis<\/mark>/, response.body) # Searching by card id get search_path(q: @card.id, script_name: "/#{@account.external_account_id}") @@ -40,4 +41,27 @@ class SearchesControllerTest < ActionDispatch::IntegrationTest assert_select "form[data-controller='auto-submit']", count: 0 assert_select ".search__empty", text: "No matches" end + + test "search highlights matched terms with proper HTML marks" do + @board.cards.create!(title: "Testing search highlighting", creator: @user) + + get search_path(q: "highlighting", script_name: "/#{@account.external_account_id}") + assert_response :success + + end + + test "search preserves highlight marks but escapes surrounding HTML" do + @board.cards.create!( + title: "Bold testing content", + creator: @user + ) + + get search_path(q: "testing", script_name: "/#{@account.external_account_id}") + assert_response :success + + # Should escape tags + assert response.body.include?("<b>") + # But should preserve highlight marks around "testing" + assert_match(/<\/span>testing<\/mark>/, response.body) + end end From f384a0d9b82c20a8cc25ccbafd032ba288214dc4 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 14:30:17 +0000 Subject: [PATCH 32/55] Fix database.yml merge error --- config/database.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/database.yml b/config/database.yml index fe34b5eff..ef501a9a2 100644 --- a/config/database.yml +++ b/config/database.yml @@ -27,6 +27,7 @@ default: &default timeout: 5000 variables: transaction_isolation: READ-COMMITTED + <% end %> development: <% if use_sqlite %> From aaf512875e22b67579cd45cce895c9204a79fb07 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 14:34:41 +0000 Subject: [PATCH 33/55] Add search_record has_one --- app/models/concerns/searchable.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 97534bbd7..894c23b94 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -2,6 +2,8 @@ module Searchable extend ActiveSupport::Concern included do + has_one :search_record, class_name: "Search::Record", as: :searchable + before_save :ensure_current_account after_create_commit :create_in_search_index after_update_commit :update_in_search_index @@ -20,17 +22,15 @@ module Searchable end def create_in_search_index - Search::Record.create!(search_record_attributes) + create_search_record!(search_record_attributes) end def update_in_search_index - Search::Record.find_or_initialize_by(searchable_type: self.class.name, searchable_id: id).tap do |record| - record.update!(search_record_attributes) - end + search_record&.update!(search_record_attributes) || create_in_search_index end def remove_from_search_index - Search::Record.find_by(searchable_type: self.class.name, searchable_id: id)&.destroy + search_record&.destroy end def search_record_attributes From bef4b862120b2902dd050daac5da8a3349a9a5d9 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 14:42:09 +0000 Subject: [PATCH 34/55] Use card_search_records instead of join class method --- app/models/card/searchable.rb | 5 +++-- app/models/search/record.rb | 6 ------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb index e2c0c11dc..669dd2bd4 100644 --- a/app/models/card/searchable.rb +++ b/app/models/card/searchable.rb @@ -4,9 +4,10 @@ module Card::Searchable included do include ::Searchable + has_many :card_search_records, class_name: "Search::Record" + scope :mentioning, ->(query, user:) do - joins(Search::Record.card_join) - .merge(Search::Record.for_query(query: Search::Query.wrap(query), user: user)) + joins(:card_search_records).merge(Search::Record.for_query(query: Search::Query.wrap(query), user: user)) end end diff --git a/app/models/search/record.rb b/app/models/search/record.rb index b958a31c7..9ae16125d 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -9,12 +9,6 @@ class Search::Record < ApplicationRecord validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true - class << self - def card_join - "INNER JOIN #{table_name} ON #{table_name}.card_id = cards.id" - end - end - scope :for_query, ->(query:, user:) do if query.valid? && user.board_ids.any? matching(query.to_s, user.account_id).where(account_id: user.account_id, board_id: user.board_ids) From eecdb486b7a60fd653fd80f8cf162e5e98611f2b Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 14:42:28 +0000 Subject: [PATCH 35/55] Remove unused attribute --- app/models/search/record.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 9ae16125d..49b48aa5f 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -4,9 +4,6 @@ class Search::Record < ApplicationRecord belongs_to :searchable, polymorphic: true belongs_to :card - # Virtual attributes from search query - attribute :query, :string - validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true scope :for_query, ->(query:, user:) do From 04c9d9268b59199ff28e0cfff4c6b8d8cba8a198 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 14:53:17 +0000 Subject: [PATCH 36/55] Remove wrapper methods and kwargs --- app/models/card/searchable.rb | 2 +- app/models/search.rb | 4 ---- app/models/search/record.rb | 10 +++++++--- app/models/user/searcher.rb | 2 +- test/models/search_test.rb | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb index 669dd2bd4..caf9b4fea 100644 --- a/app/models/card/searchable.rb +++ b/app/models/card/searchable.rb @@ -7,7 +7,7 @@ module Card::Searchable has_many :card_search_records, class_name: "Search::Record" scope :mentioning, ->(query, user:) do - joins(:card_search_records).merge(Search::Record.for_query(query: Search::Query.wrap(query), user: user)) + joins(:card_search_records).merge(Search::Record.for_query(query, user: user)) end end diff --git a/app/models/search.rb b/app/models/search.rb index 68dbbcd21..f81b9a9d8 100644 --- a/app/models/search.rb +++ b/app/models/search.rb @@ -2,8 +2,4 @@ module Search def self.table_name_prefix "search_" end - - def self.results(query:, user:) - Record.search(query: Query.wrap(query), user: user) - end end diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 49b48aa5f..424a40363 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -6,7 +6,9 @@ class Search::Record < ApplicationRecord validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true - scope :for_query, ->(query:, user:) do + scope :for_query, ->(query, user:) do + query = Search::Query.wrap(query) + if query.valid? && user.board_ids.any? matching(query.to_s, user.account_id).where(account_id: user.account_id, board_id: user.board_ids) else @@ -14,8 +16,10 @@ class Search::Record < ApplicationRecord end end - scope :search, ->(query:, user:) do - for_query(query: query, user: user) + scope :search, ->(query, user:) do + query = Search::Query.wrap(query) + + for_query(query, user: user) .includes(:searchable, card: [ :board, :creator ]) .order(created_at: :desc) .select(:id, :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :title, :content, :created_at, *search_fields(query)) diff --git a/app/models/user/searcher.rb b/app/models/user/searcher.rb index 951717082..60a901ee8 100644 --- a/app/models/user/searcher.rb +++ b/app/models/user/searcher.rb @@ -6,7 +6,7 @@ module User::Searcher end def search(terms) - Search.results(query: terms, user: self) + Search::Record.search(terms, user: self) end def remember_search(terms) diff --git a/test/models/search_test.rb b/test/models/search_test.rb index e27bcbdfe..be4174ff8 100644 --- a/test/models/search_test.rb +++ b/test/models/search_test.rb @@ -9,10 +9,10 @@ class SearchTest < ActiveSupport::TestCase comment_card = @board.cards.create!(title: "Some card", creator: @user) comment_card.comments.create!(body: "overflowing text", creator: @user) - results = Search.results(query: "layout", user: @user) + results = Search::Record.search("layout", user: @user) assert results.find { |it| it.card_id == card.id } - results = Search.results(query: "overflowing", user: @user) + results = Search::Record.search("overflowing", user: @user) assert results.find { |it| it.card_id == comment_card.id && it.searchable_type == "Comment" } # Don't include inaccessible boards @@ -21,13 +21,13 @@ class SearchTest < ActiveSupport::TestCase accessible_card = @board.cards.create!(title: "searchable content", creator: @user) inaccessible_card = inaccessible_board.cards.create!(title: "searchable content", creator: other_user) - results = Search.results(query: "searchable", user: @user) + results = Search::Record.search("searchable", user: @user) assert results.find { |it| it.card_id == accessible_card.id } assert_not results.find { |it| it.card_id == inaccessible_card.id } # Empty board_ids returns no results user_without_access = User.create!(name: "No Access User", account: @account) - results = Search.results(query: "anything", user: user_without_access) + results = Search::Record.search("anything", user: user_without_access) assert_empty results end end From 525e53a2f7b3c7f481f15e40a86c72e2a4624ab2 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 15:21:11 +0000 Subject: [PATCH 37/55] Add FTS ActiveRecord model Needs some special handling for the FTS virtual table format. Set the primary key to rowid and manually specify the columns. --- app/models/search/record/sqlite.rb | 24 ++++++---------------- app/models/search/record/sqlite/fts.rb | 21 +++++++++++++++++++ test/models/card/searchable_test.rb | 28 ++++++++++++++++++++++++++ test/models/comment/searchable_test.rb | 14 +++++++++++++ 4 files changed, 69 insertions(+), 18 deletions(-) create mode 100644 app/models/search/record/sqlite/fts.rb diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index cba6a1cde..006a48a5e 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -2,21 +2,17 @@ module Search::Record::SQLite extend ActiveSupport::Concern included do - # Override the UUID id attribute from ApplicationRecord - # FTS tables require integer rowids + # Override default UUID id attribute, as FTS5 uses rowid integer primary key attribute :id, :integer, default: nil - - # Virtual attributes from FTS5 functions attribute :result_title, :string attribute :result_content, :string + has_one :search_records_fts, -> { with_rowid }, class_name: "Search::Record::SQLite::Fts", foreign_key: :rowid, primary_key: :id + after_save :upsert_to_fts5_table after_destroy :delete_from_fts5_table - scope :matching, ->(query, account_id) do - joins("INNER JOIN search_records_fts ON search_records_fts.rowid = #{table_name}.id") - .where("search_records_fts MATCH ?", query) - end + scope :matching, ->(query, account_id) { joins(:search_records_fts).where("search_records_fts MATCH ?", query) } end class_methods do @@ -54,18 +50,10 @@ module Search::Record::SQLite end def upsert_to_fts5_table - self.class.connection.exec_query( - "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (?, ?, ?)", - "Search::Record Upsert FTS5", - [id, title, content] - ) + Fts.upsert(id, title, content) end def delete_from_fts5_table - self.class.connection.exec_query( - "DELETE FROM search_records_fts WHERE rowid = ?", - "Search::Record Delete FTS5", - [id] - ) + search_records_fts&.destroy end end diff --git a/app/models/search/record/sqlite/fts.rb b/app/models/search/record/sqlite/fts.rb new file mode 100644 index 000000000..fa00dab17 --- /dev/null +++ b/app/models/search/record/sqlite/fts.rb @@ -0,0 +1,21 @@ +class Search::Record::SQLite::Fts < ApplicationRecord + self.table_name = "search_records_fts" + self.primary_key = "rowid" + + # FTS5 virtual table columns + attribute :rowid, :integer + attribute :title, :string + attribute :content, :string + + # FTS5 virtual tables don't expose rowid in the schema by default + # We need to explicitly select it when loading records + scope :with_rowid, -> { select(:rowid, :title, :content) } + + def self.upsert(rowid, title, content) + connection.exec_query( + "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (?, ?, ?)", + "Search::Record::SqliteFts Upsert", + [rowid, title, content] + ) + end +end diff --git a/test/models/card/searchable_test.rb b/test/models/card/searchable_test.rb index 2dc8f13a6..c0a99ab23 100644 --- a/test/models/card/searchable_test.rb +++ b/test/models/card/searchable_test.rb @@ -41,4 +41,32 @@ class Card::SearchableTest < ActiveSupport::TestCase assert_match(/Current.account must be set to save Card/, error.message) end + + test "deleting card removes search record and FTS entry" do + card = @board.cards.create!(title: "Card to delete", creator: @user) + + # Verify search record exists + search_record = Search::Record.find_by(searchable_type: "Card", searchable_id: card.id) + assert_not_nil search_record, "Search record should exist after card creation" + + # For SQLite, verify FTS entry exists + if Search::Record.connection.adapter_name == "SQLite" + fts_entry = search_record.search_records_fts + assert_not_nil fts_entry, "FTS entry should exist" + assert_equal card.title, fts_entry.title + end + + # Delete the card + card.destroy + + # Verify search record is deleted + search_record = Search::Record.find_by(searchable_type: "Card", searchable_id: card.id) + assert_nil search_record, "Search record should be deleted after card deletion" + + # For SQLite, verify FTS entry is deleted + if Search::Record.connection.adapter_name == "SQLite" + fts_count = Search::Record::SQLite::Fts.where(rowid: card.id).count + assert_equal 0, fts_count, "FTS entry should be deleted" + end + end end diff --git a/test/models/comment/searchable_test.rb b/test/models/comment/searchable_test.rb index 3c7dd461f..b8b77a8b0 100644 --- a/test/models/comment/searchable_test.rb +++ b/test/models/comment/searchable_test.rb @@ -20,10 +20,24 @@ class Comment::SearchableTest < ActiveSupport::TestCase # Comment is removed from index on destroy comment_id = comment.id + search_record_id = record.id + + # For SQLite, verify FTS entry exists before deletion + if Search::Record.connection.adapter_name == "SQLite" + fts_entry = record.search_records_fts + assert_not_nil fts_entry, "FTS entry should exist before comment deletion" + end + comment.destroy record = Search::Record.find_by(searchable_type: "Comment", searchable_id: comment_id) assert_nil record + # For SQLite, verify FTS entry is also deleted + if Search::Record.connection.adapter_name == "SQLite" + fts_count = Search::Record::SQLite::Fts.where(rowid: search_record_id).count + assert_equal 0, fts_count, "FTS entry should be deleted after comment deletion" + end + # Finding cards via comment search card_with_comment = @board.cards.create!(title: "Card One", creator: @user) card_with_comment.comments.create!(body: "unique searchable phrase", creator: @user) From be0ce5efa01f9b16a502a63fb30a7f581d89840d Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 15:28:25 +0000 Subject: [PATCH 38/55] Use dependant delete on fts table --- app/models/search/record/sqlite.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index 006a48a5e..03e08cdef 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -7,10 +7,10 @@ module Search::Record::SQLite attribute :result_title, :string attribute :result_content, :string - has_one :search_records_fts, -> { with_rowid }, class_name: "Search::Record::SQLite::Fts", foreign_key: :rowid, primary_key: :id + has_one :search_records_fts, + -> { with_rowid }, class_name: "Search::Record::SQLite::Fts", foreign_key: :rowid, primary_key: :id, dependent: :destroy after_save :upsert_to_fts5_table - after_destroy :delete_from_fts5_table scope :matching, ->(query, account_id) { joins(:search_records_fts).where("search_records_fts MATCH ?", query) } end @@ -52,8 +52,4 @@ module Search::Record::SQLite def upsert_to_fts5_table Fts.upsert(id, title, content) end - - def delete_from_fts5_table - search_records_fts&.destroy - end end From d48eb523b8a7a6a204e984d413ca3a3182586363 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 15:30:14 +0000 Subject: [PATCH 39/55] Fix rubocop --- app/models/search/record/sqlite/fts.rb | 2 +- test/controllers/searches_controller_test.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/models/search/record/sqlite/fts.rb b/app/models/search/record/sqlite/fts.rb index fa00dab17..388884ae4 100644 --- a/app/models/search/record/sqlite/fts.rb +++ b/app/models/search/record/sqlite/fts.rb @@ -15,7 +15,7 @@ class Search::Record::SQLite::Fts < ApplicationRecord connection.exec_query( "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (?, ?, ?)", "Search::Record::SqliteFts Upsert", - [rowid, title, content] + [ rowid, title, content ] ) end end diff --git a/test/controllers/searches_controller_test.rb b/test/controllers/searches_controller_test.rb index 98ae5cdd3..14bd91e64 100644 --- a/test/controllers/searches_controller_test.rb +++ b/test/controllers/searches_controller_test.rb @@ -47,7 +47,6 @@ class SearchesControllerTest < ActionDispatch::IntegrationTest get search_path(q: "highlighting", script_name: "/#{@account.external_account_id}") assert_response :success - end test "search preserves highlight marks but escapes surrounding HTML" do From 0989017f62f4f4641c7899b851c9a8d39b540732 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 15:31:00 +0000 Subject: [PATCH 40/55] Fix rubocop --- app/models/search/record/trilogy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index e598c62e5..c41f30c77 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -17,7 +17,7 @@ module Search::Record::Trilogy if Current.account "search_records_#{shard_id_for_account(Current.account.id)}" else - raise "Current.account is not set; cannot determine shard for Search::Record" + "search_records" end end From d09deca03895e508ed9deb48fb94344c07677888 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 15:35:58 +0000 Subject: [PATCH 41/55] Fallback to search_records_0, so eager loading has a valid table --- app/models/search/record/trilogy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index c41f30c77..dd10334c8 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -17,7 +17,7 @@ module Search::Record::Trilogy if Current.account "search_records_#{shard_id_for_account(Current.account.id)}" else - "search_records" + "search_records_0" end end From 6f02cb40b1f6334584ad4c107e24d81ff5c44883 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 15:46:31 +0000 Subject: [PATCH 42/55] Drop unnecessary html_safes --- app/models/search/record/sqlite.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index 03e08cdef..85aeb59f2 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -7,8 +7,8 @@ module Search::Record::SQLite attribute :result_title, :string attribute :result_content, :string - has_one :search_records_fts, - -> { with_rowid }, class_name: "Search::Record::SQLite::Fts", foreign_key: :rowid, primary_key: :id, dependent: :destroy + has_one :search_records_fts, -> { with_rowid }, + class_name: "Search::Record::SQLite::Fts", foreign_key: :rowid, primary_key: :id, dependent: :destroy after_save :upsert_to_fts5_table @@ -44,8 +44,8 @@ module Search::Record::SQLite return nil unless html.present? CGI.escapeHTML(html) - .gsub(CGI.escapeHTML(Search::Highlighter::OPENING_MARK), Search::Highlighter::OPENING_MARK.html_safe) - .gsub(CGI.escapeHTML(Search::Highlighter::CLOSING_MARK), Search::Highlighter::CLOSING_MARK.html_safe) + .gsub(CGI.escapeHTML(Search::Highlighter::OPENING_MARK), Search::Highlighter::OPENING_MARK) + .gsub(CGI.escapeHTML(Search::Highlighter::CLOSING_MARK), Search::Highlighter::CLOSING_MARK) .html_safe end From 2199dcc0c819ab0335d400ff6fe31566f47eb0bd Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 16:01:05 +0000 Subject: [PATCH 43/55] Fix name in SQL string --- app/models/search/record/sqlite/fts.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/search/record/sqlite/fts.rb b/app/models/search/record/sqlite/fts.rb index 388884ae4..ed7b378c5 100644 --- a/app/models/search/record/sqlite/fts.rb +++ b/app/models/search/record/sqlite/fts.rb @@ -14,7 +14,7 @@ class Search::Record::SQLite::Fts < ApplicationRecord def self.upsert(rowid, title, content) connection.exec_query( "INSERT OR REPLACE INTO search_records_fts(rowid, title, content) VALUES (?, ?, ?)", - "Search::Record::SqliteFts Upsert", + "Search::Record::SQLite::Fts Upsert", [ rowid, title, content ] ) end From 4c4f6957ec6bfec24578a4c958f4cabc182c9fcf Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 21 Nov 2025 17:25:40 +0000 Subject: [PATCH 44/55] Drop sqlite compability, using a separate schema.rb --- config/initializers/sqlite_compatibility.rb | 44 --------------------- 1 file changed, 44 deletions(-) delete mode 100644 config/initializers/sqlite_compatibility.rb diff --git a/config/initializers/sqlite_compatibility.rb b/config/initializers/sqlite_compatibility.rb deleted file mode 100644 index 50d44c28b..000000000 --- a/config/initializers/sqlite_compatibility.rb +++ /dev/null @@ -1,44 +0,0 @@ -# SQLite compatibility layer - filters out MySQL-specific options when using SQLite - -# Define modules outside on_load so they're available immediately -module SQLiteCompatibility - module SQLiteTableDefinitionCompatibility - # Override column method to filter out MySQL-specific options - def column(name, type, **options) - # Check if we're using SQLite by checking the connection adapter - if @conn.is_a?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) - # Remove MySQL-specific options that SQLite doesn't support - options = options.except(:size, :charset, :collation, :unsigned) - end - super(name, type, **options) - end - - # Override index method to filter out MySQL-specific index options - def index(column_name, **options) - if @conn.is_a?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) - # SQLite doesn't support length-limited indexes - options = options.except(:length) - end - super(column_name, **options) - end - end - - module SQLiteSchemaStatementCompatibility - # Override create_table to filter out MySQL-specific table options - def create_table(table_name, **options) - if is_a?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) - # Remove MySQL-specific table options - options = options.except(:charset, :collation) - end - super(table_name, **options) - end - end -end - -# Also run when ActiveRecord loads (for cases where it loads later) -ActiveSupport.on_load(:active_record) do - if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) - ActiveRecord::ConnectionAdapters::TableDefinition.prepend(SQLiteCompatibility::SQLiteTableDefinitionCompatibility) - ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(SQLiteCompatibility::SQLiteSchemaStatementCompatibility) - end -end From b571303385bd635f6133fad916748aea50ea806d Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 25 Nov 2025 11:34:41 +0000 Subject: [PATCH 45/55] Don't use Current.account for search records It doesn't actually work, and even if we could make it work reliably we are better off if the records always know to go to the right shard. It does make the interface a bit more complicated as we need to select the right shard class with `for(account_id)`. --- app/models/card/searchable.rb | 5 ++--- app/models/concerns/searchable.rb | 19 +++++++------------ app/models/search/record.rb | 16 ++++++++++++++++ app/models/search/record/sqlite.rb | 4 ++++ app/models/search/record/trilogy.rb | 23 +++++++++++++++-------- app/models/user/searcher.rb | 2 +- lib/tasks/search.rake | 10 ++-------- test/models/card/searchable_test.rb | 19 +++++-------------- test/models/comment/searchable_test.rb | 23 +++++++---------------- test/models/search_test.rb | 8 ++++---- 10 files changed, 63 insertions(+), 66 deletions(-) diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb index caf9b4fea..8cc108b23 100644 --- a/app/models/card/searchable.rb +++ b/app/models/card/searchable.rb @@ -4,10 +4,9 @@ module Card::Searchable included do include ::Searchable - has_many :card_search_records, class_name: "Search::Record" - scope :mentioning, ->(query, user:) do - joins(:card_search_records).merge(Search::Record.for_query(query, user: user)) + search_record_class = Search::Record.for(user.account_id) + joins(search_record_class.card_join).merge(search_record_class.for_query(query, user: user)) end end diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 894c23b94..8b39504ef 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -2,9 +2,6 @@ module Searchable extend ActiveSupport::Concern included do - has_one :search_record, class_name: "Search::Record", as: :searchable - - before_save :ensure_current_account after_create_commit :create_in_search_index after_update_commit :update_in_search_index after_destroy_commit :remove_from_search_index @@ -15,22 +12,16 @@ module Searchable end private - def ensure_current_account - unless Current.account - raise "Current.account must be set to save #{self.class.name}" - end - end - def create_in_search_index - create_search_record!(search_record_attributes) + search_record_class.create!(search_record_attributes) end def update_in_search_index - search_record&.update!(search_record_attributes) || create_in_search_index + search_record_class.upsert!(search_record_attributes) end def remove_from_search_index - search_record&.destroy + search_record_class.find_by(searchable_type: self.class.name, searchable_id: id)&.destroy end def search_record_attributes @@ -46,6 +37,10 @@ module Searchable } end + def search_record_class + Search::Record.for(account_id) + end + # Models must implement these methods: # - account_id: returns the account id # - search_title: returns title string or nil diff --git a/app/models/search/record.rb b/app/models/search/record.rb index 424a40363..2aaff7c24 100644 --- a/app/models/search/record.rb +++ b/app/models/search/record.rb @@ -6,6 +6,22 @@ class Search::Record < ApplicationRecord validates :account_id, :searchable_type, :searchable_id, :card_id, :board_id, :created_at, presence: true + class << self + def upsert!(attributes) + record = find_by(searchable_type: attributes[:searchable_type], searchable_id: attributes[:searchable_id]) + if record + record.update!(attributes) + record + else + create!(attributes) + end + end + + def card_join + "INNER JOIN #{table_name} ON #{table_name}.card_id = cards.id" + end + end + scope :for_query, ->(query, user:) do query = Search::Query.wrap(query) diff --git a/app/models/search/record/sqlite.rb b/app/models/search/record/sqlite.rb index 85aeb59f2..c3da9131a 100644 --- a/app/models/search/record/sqlite.rb +++ b/app/models/search/record/sqlite.rb @@ -25,6 +25,10 @@ module Search::Record::SQLite "snippet(search_records_fts, 1, #{opening_mark}, #{closing_mark}, #{ellipsis}, 20) AS result_content", "#{connection.quote(query.terms)} AS query" ] end + + def for(account_id) + self + end end def card_title diff --git a/app/models/search/record/trilogy.rb b/app/models/search/record/trilogy.rb index dd10334c8..7705f3b71 100644 --- a/app/models/search/record/trilogy.rb +++ b/app/models/search/record/trilogy.rb @@ -4,23 +4,26 @@ module Search::Record::Trilogy SHARD_COUNT = 16 included do + self.abstract_class = true before_save :set_account_key, :stem_content scope :matching, ->(query, account_id) do full_query = "+account#{account_id} +(#{Search::Stemmer.stem(query)})" where("MATCH(#{table_name}.account_key, #{table_name}.content, #{table_name}.title) AGAINST(? IN BOOLEAN MODE)", full_query) end + + SHARD_CLASSES = SHARD_COUNT.times.map do |shard_id| + Class.new(self) do + self.table_name = "search_records_#{shard_id}" + + def self.name + "Search::Record" + end + end + end.freeze end class_methods do - def compute_table_name - if Current.account - "search_records_#{shard_id_for_account(Current.account.id)}" - else - "search_records_0" - end - end - def shard_id_for_account(account_id) Zlib.crc32(account_id.to_s) % SHARD_COUNT end @@ -28,6 +31,10 @@ module Search::Record::Trilogy def search_fields(query) "#{connection.quote(query.terms)} AS query" end + + def for(account_id) + SHARD_CLASSES[shard_id_for_account(account_id)] + end end def card_title diff --git a/app/models/user/searcher.rb b/app/models/user/searcher.rb index 60a901ee8..090adf4b8 100644 --- a/app/models/user/searcher.rb +++ b/app/models/user/searcher.rb @@ -6,7 +6,7 @@ module User::Searcher end def search(terms) - Search::Record.search(terms, user: self) + Search::Record.for(account_id).search(terms, user: self) end def remember_search(terms) diff --git a/lib/tasks/search.rake b/lib/tasks/search.rake index df39c3a70..6b47dbde3 100644 --- a/lib/tasks/search.rake +++ b/lib/tasks/search.rake @@ -12,16 +12,10 @@ namespace :search do end puts "Reindexing cards..." - Card.find_each do |card| - Current.account = card.account - card.reindex - end + Card.includes(:rich_text_description).find_each(&:reindex) puts "Reindexing comments..." - Comment.find_each do |comment| - Current.account = comment.account - comment.reindex - end + Comment.includes(:rich_text_body, :card).find_each(&:reindex) puts "Done! Reindexed #{Card.count} cards and #{Comment.count} comments." end diff --git a/test/models/card/searchable_test.rb b/test/models/card/searchable_test.rb index c0a99ab23..d6dcdf60c 100644 --- a/test/models/card/searchable_test.rb +++ b/test/models/card/searchable_test.rb @@ -32,25 +32,16 @@ class Card::SearchableTest < ActiveSupport::TestCase assert_not_includes results, card_in_other_board end - test "card requires Current.account to be set" do - Current.account = nil - - error = assert_raises(RuntimeError) do - @board.cards.create!(title: "Test card", creator: @user) - end - - assert_match(/Current.account must be set to save Card/, error.message) - end - test "deleting card removes search record and FTS entry" do + search_record_class = Search::Record.for(@user.account_id) card = @board.cards.create!(title: "Card to delete", creator: @user) # Verify search record exists - search_record = Search::Record.find_by(searchable_type: "Card", searchable_id: card.id) + search_record = search_record_class.find_by(searchable_type: "Card", searchable_id: card.id) assert_not_nil search_record, "Search record should exist after card creation" # For SQLite, verify FTS entry exists - if Search::Record.connection.adapter_name == "SQLite" + if search_record_class.connection.adapter_name == "SQLite" fts_entry = search_record.search_records_fts assert_not_nil fts_entry, "FTS entry should exist" assert_equal card.title, fts_entry.title @@ -60,11 +51,11 @@ class Card::SearchableTest < ActiveSupport::TestCase card.destroy # Verify search record is deleted - search_record = Search::Record.find_by(searchable_type: "Card", searchable_id: card.id) + search_record = search_record_class.find_by(searchable_type: "Card", searchable_id: card.id) assert_nil search_record, "Search record should be deleted after card deletion" # For SQLite, verify FTS entry is deleted - if Search::Record.connection.adapter_name == "SQLite" + if search_record_class.connection.adapter_name == "SQLite" fts_count = Search::Record::SQLite::Fts.where(rowid: card.id).count assert_equal 0, fts_count, "FTS entry should be deleted" end diff --git a/test/models/comment/searchable_test.rb b/test/models/comment/searchable_test.rb index b8b77a8b0..7ba73847c 100644 --- a/test/models/comment/searchable_test.rb +++ b/test/models/comment/searchable_test.rb @@ -8,14 +8,15 @@ class Comment::SearchableTest < ActiveSupport::TestCase end test "comment search" do + search_record_class = Search::Record.for(@user.account_id) # Comment is indexed on create comment = @card.comments.create!(body: "searchable comment text", creator: @user) - record = Search::Record.find_by(searchable_type: "Comment", searchable_id: comment.id) + record = search_record_class.find_by(searchable_type: "Comment", searchable_id: comment.id) assert_not_nil record # Comment is updated in index comment.update!(body: "updated text") - record = Search::Record.find_by(searchable_type: "Comment", searchable_id: comment.id) + record = search_record_class.find_by(searchable_type: "Comment", searchable_id: comment.id) assert_match /updat/, record.content # Comment is removed from index on destroy @@ -23,17 +24,17 @@ class Comment::SearchableTest < ActiveSupport::TestCase search_record_id = record.id # For SQLite, verify FTS entry exists before deletion - if Search::Record.connection.adapter_name == "SQLite" + if search_record_class.connection.adapter_name == "SQLite" fts_entry = record.search_records_fts assert_not_nil fts_entry, "FTS entry should exist before comment deletion" end comment.destroy - record = Search::Record.find_by(searchable_type: "Comment", searchable_id: comment_id) + record = search_record_class.find_by(searchable_type: "Comment", searchable_id: comment_id) assert_nil record # For SQLite, verify FTS entry is also deleted - if Search::Record.connection.adapter_name == "SQLite" + if search_record_class.connection.adapter_name == "SQLite" fts_count = Search::Record::SQLite::Fts.where(rowid: search_record_id).count assert_equal 0, fts_count, "FTS entry should be deleted after comment deletion" end @@ -48,18 +49,8 @@ class Comment::SearchableTest < ActiveSupport::TestCase # Comment stores parent card_id and board_id new_comment = @card.comments.create!(body: "test comment", creator: @user) - record = Search::Record.find_by(searchable_type: "Comment", searchable_id: new_comment.id) + record = search_record_class.find_by(searchable_type: "Comment", searchable_id: new_comment.id) assert_equal @card.id, record.card_id assert_equal @board.id, record.board_id end - - test "comment requires Current.account to be set" do - Current.account = nil - - error = assert_raises(RuntimeError) do - @card.comments.create!(body: "Test comment", creator: @user) - end - - assert_match(/Current.account must be set to save Comment/, error.message) - end end diff --git a/test/models/search_test.rb b/test/models/search_test.rb index be4174ff8..1f4981090 100644 --- a/test/models/search_test.rb +++ b/test/models/search_test.rb @@ -9,10 +9,10 @@ class SearchTest < ActiveSupport::TestCase comment_card = @board.cards.create!(title: "Some card", creator: @user) comment_card.comments.create!(body: "overflowing text", creator: @user) - results = Search::Record.search("layout", user: @user) + results = Search::Record.for(@user.account_id).search("layout", user: @user) assert results.find { |it| it.card_id == card.id } - results = Search::Record.search("overflowing", user: @user) + results = Search::Record.for(@user.account_id).search("overflowing", user: @user) assert results.find { |it| it.card_id == comment_card.id && it.searchable_type == "Comment" } # Don't include inaccessible boards @@ -21,13 +21,13 @@ class SearchTest < ActiveSupport::TestCase accessible_card = @board.cards.create!(title: "searchable content", creator: @user) inaccessible_card = inaccessible_board.cards.create!(title: "searchable content", creator: other_user) - results = Search::Record.search("searchable", user: @user) + results = Search::Record.for(@user.account_id).search("searchable", user: @user) assert results.find { |it| it.card_id == accessible_card.id } assert_not results.find { |it| it.card_id == inaccessible_card.id } # Empty board_ids returns no results user_without_access = User.create!(name: "No Access User", account: @account) - results = Search::Record.search("anything", user: user_without_access) + results = Search::Record.for(user_without_access.account_id).search("anything", user: user_without_access) assert_empty results end end From b96ea6e9d06abb90ebb4d9dd4b9e97d729684254 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 25 Nov 2025 11:38:13 +0000 Subject: [PATCH 46/55] Update config/database.yml Co-authored-by: Mike Dalessio --- config/database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/database.yml b/config/database.yml index ef501a9a2..9177f6c9b 100644 --- a/config/database.yml +++ b/config/database.yml @@ -33,7 +33,7 @@ development: <% if use_sqlite %> primary: <<: *default - database: db/development.sqlite3 + database: storage/development.sqlite3 schema_dump: schema_sqlite.rb <% else %> primary: From e07419e953bf71856cf4ada266e9a2e7f5b70004 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 25 Nov 2025 11:38:21 +0000 Subject: [PATCH 47/55] Update config/database.yml Co-authored-by: Mike Dalessio --- config/database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/database.yml b/config/database.yml index 9177f6c9b..2e89b6969 100644 --- a/config/database.yml +++ b/config/database.yml @@ -69,7 +69,7 @@ test: <% if use_sqlite %> primary: <<: *default - database: db/test.sqlite3 + database: storage/test.sqlite3 schema_dump: schema_sqlite.rb <% else %> primary: From c2eb840e2b5c59667ce34a2c13e3811db9dc59f1 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 25 Nov 2025 13:47:08 +0000 Subject: [PATCH 48/55] Use Uuid type directly --- app/models/board/accessible.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/models/board/accessible.rb b/app/models/board/accessible.rb index e4eb26e32..f17e734ff 100644 --- a/app/models/board/accessible.rb +++ b/app/models/board/accessible.rb @@ -66,8 +66,7 @@ module Board::Accessible # # 1. Mention->Card # 2. Mention->Comment->Card - uuid_type = ActiveRecord::Type.lookup(:uuid, adapter: :trilogy) - board_id_binary = uuid_type.serialize(id) + board_id_binary = ActiveRecord::Type::Uuid.new.serialize(id) user.mentions .joins("LEFT JOIN cards ON mentions.source_id = cards.id AND mentions.source_type = 'Card'") From 67f648f356406fe644ab64df6e586aa365aba53f Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 25 Nov 2025 14:49:34 +0000 Subject: [PATCH 49/55] Enforce MySQL string/text limits in SQLite To ensure consistent column lengths, we'll add limits to string and text columns which match MySQL defaults. --- .../table_definition_column_limits.rb | 49 +++++++++ db/migrate/20251111122540_initial_schema.rb | 86 +++++++-------- .../20251120110206_add_search_records.rb | 4 +- db/schema_sqlite.rb | 102 +++++++++--------- 4 files changed, 145 insertions(+), 96 deletions(-) create mode 100644 config/initializers/table_definition_column_limits.rb diff --git a/config/initializers/table_definition_column_limits.rb b/config/initializers/table_definition_column_limits.rb new file mode 100644 index 000000000..f9a75d174 --- /dev/null +++ b/config/initializers/table_definition_column_limits.rb @@ -0,0 +1,49 @@ +# Apply MySQL-compatible column limits when using SQLite. +# +# For string columns: defaults to 255 (MySQL's VARCHAR default) +# +# For text columns: converts MySQL's `size:` option to equivalent limits: +# - (blank/default): 65,535 (TEXT) +# - size: :tiny: 255 (TINYTEXT) +# - size: :medium: 16,777,215 (MEDIUMTEXT) +# - size: :long: 4,294,967,295 (LONGTEXT) + +module TableDefinitionColumnLimits + # Map MySQL size options to limits + TEXT_SIZE_TO_LIMIT = { + tiny: 255, # TINYTEXT + medium: 16_777_215, # MEDIUMTEXT + long: 4_294_967_295 # LONGTEXT + }.freeze + + TEXT_DEFAULT_LIMIT = 65_535 # TEXT + STRING_DEFAULT_LIMIT = 255 # VARCHAR + + def column(name, type, **options) + if type == :string + options[:limit] ||= STRING_DEFAULT_LIMIT + end + + if type == :text + if options.key?(:size) + size = options.delete(:size) + options[:limit] = TEXT_SIZE_TO_LIMIT.fetch(size) do + raise ArgumentError, "Unknown text size: #{size.inspect}. Use :tiny, :medium, or :long" + end + elsif options.key?(:limit) + valid_limits = [TEXT_DEFAULT_LIMIT] + TEXT_SIZE_TO_LIMIT.values + unless valid_limits.include?(options[:limit]) + raise ArgumentError, "Invalid limit #{options[:limit]} for text column. Use `size:` (:tiny, :medium, :long) or omit for default TEXT." + end + else + options[:limit] = TEXT_DEFAULT_LIMIT + end + end + + super + end +end + +ActiveSupport.on_load(:active_record) do + ActiveRecord::ConnectionAdapters::TableDefinition.prepend(TableDefinitionColumnLimits) +end diff --git a/db/migrate/20251111122540_initial_schema.rb b/db/migrate/20251111122540_initial_schema.rb index 4b736f712..c7afd79ef 100644 --- a/db/migrate/20251111122540_initial_schema.rb +++ b/db/migrate/20251111122540_initial_schema.rb @@ -4,7 +4,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] t.datetime "accessed_at" t.uuid "board_id", null: false t.datetime "created_at", null: false - t.string "involvement", default: "access_only", null: false + t.string "involvement", limit: 255, default: "access_only", null: false t.datetime "updated_at", null: false t.uuid "user_id", null: false t.index ["accessed_at"], name: "index_accesses_on_accessed_at", order: :desc @@ -15,7 +15,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "account_join_codes", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.uuid "account_id" - t.string "code", null: false + t.string "code", limit: 255, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "usage_count", default: 0, null: false @@ -26,7 +26,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "accounts", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.datetime "created_at", null: false t.integer "external_account_id" - t.string "name", null: false + t.string "name", limit: 255, null: false t.datetime "updated_at", null: false t.index ["external_account_id"], name: "index_accounts_on_external_account_id", unique: true end @@ -34,9 +34,9 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "action_text_rich_texts", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.text "body", size: :long t.datetime "created_at", null: false - t.string "name", null: false + t.string "name", limit: 255, null: false t.uuid "record_id", null: false - t.string "record_type", null: false + t.string "record_type", limit: 255, null: false t.datetime "updated_at", null: false t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true end @@ -44,28 +44,28 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "active_storage_attachments", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.uuid "blob_id", null: false t.datetime "created_at", null: false - t.string "name", null: false + t.string "name", limit: 255, null: false t.uuid "record_id", null: false - t.string "record_type", null: false + t.string "record_type", limit: 255, null: false t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true end create_table "active_storage_blobs", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.bigint "byte_size", null: false - t.string "checksum" - t.string "content_type" + t.string "checksum", limit: 255 + t.string "content_type", limit: 255 t.datetime "created_at", null: false - t.string "filename", null: false - t.string "key", null: false + t.string "filename", limit: 255, null: false + t.string "key", limit: 255, null: false t.text "metadata" - t.string "service_name", null: false + t.string "service_name", limit: 255, null: false t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true end create_table "active_storage_variant_records", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.uuid "blob_id", null: false - t.string "variation_digest", null: false + t.string "variation_digest", limit: 255, null: false t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end @@ -96,7 +96,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "board_publications", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.uuid "board_id", null: false t.datetime "created_at", null: false - t.string "key" + t.string "key", limit: 255 t.datetime "updated_at", null: false t.index ["board_id"], name: "index_board_publications_on_board_id" t.index ["key"], name: "index_board_publications_on_key", unique: true @@ -107,7 +107,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] t.boolean "all_access", default: false, null: false t.datetime "created_at", null: false t.uuid "creator_id", null: false - t.string "name", null: false + t.string "name", limit: 255, null: false t.datetime "updated_at", null: false t.index ["creator_id"], name: "index_boards_on_creator_id" end @@ -129,7 +129,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "card_engagements", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.uuid "card_id" t.datetime "created_at", null: false - t.string "status", default: "doing", null: false + t.string "status", limit: 255, default: "doing", null: false t.datetime "updated_at", null: false t.index ["card_id"], name: "index_card_engagements_on_card_id" t.index ["status"], name: "index_card_engagements_on_status" @@ -159,8 +159,8 @@ class InitialSchema < ActiveRecord::Migration[8.2] t.uuid "creator_id", null: false t.date "due_on" t.datetime "last_active_at", null: false - t.string "status", default: "drafted", null: false - t.string "title" + t.string "status", limit: 255, default: "drafted", null: false + t.string "title", limit: 255 t.datetime "updated_at", null: false t.index ["board_id"], name: "index_cards_on_board_id" t.index ["column_id"], name: "index_cards_on_column_id" @@ -187,9 +187,9 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "columns", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.uuid "account_id" t.uuid "board_id", null: false - t.string "color", null: false + t.string "color", limit: 255, null: false t.datetime "created_at", null: false - t.string "name", null: false + t.string "name", limit: 255, null: false t.integer "position", default: 0, null: false t.datetime "updated_at", null: false t.index ["board_id", "position"], name: "index_columns_on_board_id_and_position" @@ -215,7 +215,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "entropies", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.bigint "auto_postpone_period", default: 2592000, null: false t.uuid "container_id", null: false - t.string "container_type", null: false + t.string "container_type", limit: 255, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["container_type", "container_id", "auto_postpone_period"], name: "idx_on_container_type_container_id_auto_postpone_pe_3d79b50517" @@ -224,12 +224,12 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "events", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.uuid "account_id" - t.string "action", null: false + t.string "action", limit: 255, null: false t.uuid "board_id", null: false t.datetime "created_at", null: false t.uuid "creator_id", null: false t.uuid "eventable_id", null: false - t.string "eventable_type", null: false + t.string "eventable_type", limit: 255, null: false t.json "particulars", default: -> { "(json_object())" } t.datetime "updated_at", null: false t.index ["action"], name: "index_events_on_summary_id_and_action" @@ -244,7 +244,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] t.datetime "created_at", null: false t.uuid "creator_id", null: false t.json "fields", default: -> { "(json_object())" }, null: false - t.string "params_digest", null: false + t.string "params_digest", limit: 255, null: false t.datetime "updated_at", null: false t.index ["creator_id", "params_digest"], name: "index_filters_on_creator_id_and_params_digest", unique: true end @@ -258,13 +258,13 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "identities", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.datetime "created_at", null: false - t.string "email_address", null: false + t.string "email_address", limit: 255, null: false t.datetime "updated_at", null: false t.index ["email_address"], name: "index_identities_on_email_address", unique: true end create_table "magic_links", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| - t.string "code", null: false + t.string "code", limit: 255, null: false t.datetime "created_at", null: false t.datetime "expires_at", null: false t.uuid "identity_id" @@ -277,8 +277,8 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "memberships", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.datetime "created_at", null: false t.uuid "identity_id", null: false - t.string "join_code" - t.string "tenant", null: false + t.string "join_code", limit: 255 + t.string "tenant", limit: 255, null: false t.datetime "updated_at", null: false t.index ["identity_id"], name: "index_memberships_on_identity_id" t.index ["tenant", "identity_id"], name: "index_memberships_on_tenant_and_identity_id", unique: true @@ -290,7 +290,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] t.uuid "mentionee_id", null: false t.uuid "mentioner_id", null: false t.uuid "source_id", null: false - t.string "source_type", null: false + t.string "source_type", limit: 255, null: false t.datetime "updated_at", null: false t.index ["mentionee_id"], name: "index_mentions_on_mentionee_id" t.index ["mentioner_id"], name: "index_mentions_on_mentioner_id" @@ -316,7 +316,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] t.uuid "creator_id" t.datetime "read_at" t.uuid "source_id", null: false - t.string "source_type", null: false + t.string "source_type", limit: 255, null: false t.datetime "updated_at", null: false t.uuid "user_id", null: false t.index ["creator_id"], name: "index_notifications_on_creator_id" @@ -337,12 +337,12 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "push_subscriptions", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.uuid "account_id" - t.string "auth_key" + t.string "auth_key", limit: 255 t.datetime "created_at", null: false - t.string "endpoint" - t.string "p256dh_key" + t.text "endpoint" + t.string "p256dh_key", limit: 255 t.datetime "updated_at", null: false - t.string "user_agent" + t.string "user_agent", limit: 255 t.uuid "user_id", null: false t.index ["endpoint", "p256dh_key", "auth_key"], name: "idx_on_endpoint_p256dh_key_auth_key_7553014576" t.index ["endpoint"], name: "index_push_subscriptions_on_endpoint" @@ -379,9 +379,9 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "sessions", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.datetime "created_at", null: false t.uuid "identity_id", null: false - t.string "ip_address" + t.string "ip_address", limit: 255 t.datetime "updated_at", null: false - t.string "user_agent" + t.string "user_agent", limit: 255 t.index ["identity_id"], name: "index_sessions_on_identity_id" end @@ -408,7 +408,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "tags", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.uuid "account_id" t.datetime "created_at", null: false - t.string "title" + t.string "title", limit: 255 t.datetime "updated_at", null: false t.index ["title"], name: "index_tags_on_title", unique: true end @@ -416,7 +416,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] create_table "user_settings", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci" do |t| t.integer "bundle_email_frequency", default: 0, null: false t.datetime "created_at", null: false - t.string "timezone_name" + t.string "timezone_name", limit: 255 t.datetime "updated_at", null: false t.uuid "user_id", null: false t.index ["user_id", "bundle_email_frequency"], name: "index_user_settings_on_user_id_and_bundle_email_frequency" @@ -428,8 +428,8 @@ class InitialSchema < ActiveRecord::Migration[8.2] t.boolean "active", default: true, null: false t.datetime "created_at", null: false t.uuid "membership_id" - t.string "name", null: false - t.string "role", default: "member", null: false + t.string "name", limit: 255, null: false + t.string "role", limit: 255, default: "member", null: false t.datetime "updated_at", null: false t.index ["membership_id"], name: "index_users_on_membership_id" t.index ["role"], name: "index_users_on_role" @@ -460,7 +460,7 @@ class InitialSchema < ActiveRecord::Migration[8.2] t.uuid "event_id", null: false t.text "request" t.text "response" - t.string "state", null: false + t.string "state", limit: 255, null: false t.datetime "updated_at", null: false t.uuid "webhook_id", null: false t.index ["event_id"], name: "index_webhook_deliveries_on_event_id" @@ -472,8 +472,8 @@ class InitialSchema < ActiveRecord::Migration[8.2] t.boolean "active", default: true, null: false t.uuid "board_id", null: false t.datetime "created_at", null: false - t.string "name" - t.string "signing_secret", null: false + t.string "name", limit: 255 + t.string "signing_secret", limit: 255, null: false t.text "subscribed_actions" t.datetime "updated_at", null: false t.text "url", null: false diff --git a/db/migrate/20251120110206_add_search_records.rb b/db/migrate/20251120110206_add_search_records.rb index 866a39b48..638128ec9 100644 --- a/db/migrate/20251120110206_add_search_records.rb +++ b/db/migrate/20251120110206_add_search_records.rb @@ -5,11 +5,11 @@ class AddSearchRecords < ActiveRecord::Migration[8.2] # Create regular table with integer primary key for FTS5 rowid compatibility create_table :search_records do |t| t.uuid :account_id, null: false - t.string :searchable_type, null: false + t.string :searchable_type, limit: 255, null: false t.uuid :searchable_id, null: false t.uuid :card_id, null: false t.uuid :board_id, null: false - t.string :title + t.string :title, limit: 255 t.text :content t.datetime :created_at, null: false diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb index e3444f3fe..bc40a0f42 100644 --- a/db/schema_sqlite.rb +++ b/db/schema_sqlite.rb @@ -16,7 +16,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "account_id", null: false t.uuid "board_id", null: false t.datetime "created_at", null: false - t.string "involvement", default: "access_only", null: false + t.string "involvement", limit: 255, default: "access_only", null: false t.datetime "updated_at", null: false t.uuid "user_id", null: false t.index ["account_id", "accessed_at"], name: "index_accesses_on_account_id_and_accessed_at" @@ -27,7 +27,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "account_join_codes", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false - t.string "code", null: false + t.string "code", limit: 255, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.bigint "usage_count", default: 0, null: false @@ -39,18 +39,18 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.bigint "cards_count", default: 0, null: false t.datetime "created_at", null: false t.bigint "external_account_id" - t.string "name", null: false + t.string "name", limit: 255, null: false t.datetime "updated_at", null: false t.index ["external_account_id"], name: "index_accounts_on_external_account_id", unique: true end create_table "action_text_rich_texts", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false - t.text "body" + t.text "body", limit: 65535 t.datetime "created_at", null: false - t.string "name", null: false + t.string "name", limit: 255, null: false t.uuid "record_id", null: false - t.string "record_type", null: false + t.string "record_type", limit: 255, null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_action_text_rich_texts_on_account_id" t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true @@ -60,9 +60,9 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "account_id", null: false t.uuid "blob_id", null: false t.datetime "created_at", null: false - t.string "name", null: false + t.string "name", limit: 255, null: false t.uuid "record_id", null: false - t.string "record_type", null: false + t.string "record_type", limit: 255, null: false t.index ["account_id"], name: "index_active_storage_attachments_on_account_id" t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true @@ -71,13 +71,13 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "active_storage_blobs", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false t.bigint "byte_size", null: false - t.string "checksum" - t.string "content_type" + t.string "checksum", limit: 255 + t.string "content_type", limit: 255 t.datetime "created_at", null: false - t.string "filename", null: false - t.string "key", null: false - t.text "metadata" - t.string "service_name", null: false + t.string "filename", limit: 255, null: false + t.string "key", limit: 255, null: false + t.text "metadata", limit: 65535 + t.string "service_name", limit: 255, null: false t.index ["account_id"], name: "index_active_storage_blobs_on_account_id" t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true end @@ -85,7 +85,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "active_storage_variant_records", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false t.uuid "blob_id", null: false - t.string "variation_digest", null: false + t.string "variation_digest", limit: 255, null: false t.index ["account_id"], name: "index_active_storage_variant_records_on_account_id" t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end @@ -120,7 +120,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "account_id", null: false t.uuid "board_id", null: false t.datetime "created_at", null: false - t.string "key" + t.string "key", limit: 255 t.datetime "updated_at", null: false t.index ["account_id", "key"], name: "index_board_publications_on_account_id_and_key" t.index ["board_id"], name: "index_board_publications_on_board_id" @@ -131,7 +131,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.boolean "all_access", default: false, null: false t.datetime "created_at", null: false t.uuid "creator_id", null: false - t.string "name", null: false + t.string "name", limit: 255, null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_boards_on_account_id" t.index ["creator_id"], name: "index_boards_on_creator_id" @@ -157,7 +157,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "account_id", null: false t.uuid "card_id" t.datetime "created_at", null: false - t.string "status", default: "doing", null: false + t.string "status", limit: 255, default: "doing", null: false t.datetime "updated_at", null: false t.index ["account_id", "status"], name: "index_card_engagements_on_account_id_and_status" t.index ["card_id"], name: "index_card_engagements_on_card_id" @@ -192,8 +192,8 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.date "due_on" t.datetime "last_active_at", null: false t.bigint "number", null: false - t.string "status", default: "drafted", null: false - t.string "title" + t.string "status", limit: 255, default: "drafted", null: false + t.string "title", limit: 255 t.datetime "updated_at", null: false t.index ["account_id", "last_active_at", "status"], name: "index_cards_on_account_id_and_last_active_at_and_status" t.index ["account_id", "number"], name: "index_cards_on_account_id_and_number", unique: true @@ -223,9 +223,9 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "columns", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false t.uuid "board_id", null: false - t.string "color", null: false + t.string "color", limit: 255, null: false t.datetime "created_at", null: false - t.string "name", null: false + t.string "name", limit: 255, null: false t.integer "position", default: 0, null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_columns_on_account_id" @@ -254,7 +254,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "account_id", null: false t.bigint "auto_postpone_period", default: 2592000, null: false t.uuid "container_id", null: false - t.string "container_type", null: false + t.string "container_type", limit: 255, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_entropies_on_account_id" @@ -264,12 +264,12 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "events", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false - t.string "action", null: false + t.string "action", limit: 255, null: false t.uuid "board_id", null: false t.datetime "created_at", null: false t.uuid "creator_id", null: false t.uuid "eventable_id", null: false - t.string "eventable_type", null: false + t.string "eventable_type", limit: 255, null: false t.json "particulars", default: -> { "json_object()" } t.datetime "updated_at", null: false t.index ["account_id", "action"], name: "index_events_on_account_id_and_action" @@ -284,7 +284,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.datetime "created_at", null: false t.uuid "creator_id", null: false t.json "fields", default: -> { "json_object()" }, null: false - t.string "params_digest", null: false + t.string "params_digest", limit: 255, null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_filters_on_account_id" t.index ["creator_id", "params_digest"], name: "index_filters_on_creator_id_and_params_digest", unique: true @@ -299,13 +299,13 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "identities", id: :uuid, force: :cascade do |t| t.datetime "created_at", null: false - t.string "email_address", null: false + t.string "email_address", limit: 255, null: false t.datetime "updated_at", null: false t.index ["email_address"], name: "index_identities_on_email_address", unique: true end create_table "magic_links", id: :uuid, force: :cascade do |t| - t.string "code", null: false + t.string "code", limit: 255, null: false t.datetime "created_at", null: false t.datetime "expires_at", null: false t.uuid "identity_id" @@ -321,7 +321,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "mentionee_id", null: false t.uuid "mentioner_id", null: false t.uuid "source_id", null: false - t.string "source_type", null: false + t.string "source_type", limit: 255, null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_mentions_on_account_id" t.index ["mentionee_id"], name: "index_mentions_on_mentionee_id" @@ -349,7 +349,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "creator_id" t.datetime "read_at" t.uuid "source_id", null: false - t.string "source_type", null: false + t.string "source_type", limit: 255, null: false t.datetime "updated_at", null: false t.uuid "user_id", null: false t.index ["account_id"], name: "index_notifications_on_account_id" @@ -373,12 +373,12 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "push_subscriptions", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false - t.string "auth_key" + t.string "auth_key", limit: 255 t.datetime "created_at", null: false - t.text "endpoint" - t.string "p256dh_key" + t.text "endpoint", limit: 65535 + t.string "p256dh_key", limit: 255 t.datetime "updated_at", null: false - t.string "user_agent" + t.string "user_agent", limit: 255 t.uuid "user_id", null: false t.index ["account_id"], name: "index_push_subscriptions_on_account_id" t.index ["user_id", "endpoint"], name: "index_push_subscriptions_on_user_id_and_endpoint", unique: true @@ -412,11 +412,11 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "account_id", null: false t.uuid "board_id", null: false t.uuid "card_id", null: false - t.text "content" + t.text "content", limit: 65535 t.datetime "created_at", null: false t.uuid "searchable_id", null: false - t.string "searchable_type", null: false - t.string "title" + t.string "searchable_type", limit: 255, null: false + t.string "title", limit: 255 t.index ["account_id"], name: "index_search_records_on_account_id" t.index ["searchable_type", "searchable_id"], name: "index_search_records_on_searchable_type_and_searchable_id", unique: true end @@ -424,9 +424,9 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "sessions", id: :uuid, force: :cascade do |t| t.datetime "created_at", null: false t.uuid "identity_id", null: false - t.string "ip_address" + t.string "ip_address", limit: 255 t.datetime "updated_at", null: false - t.string "user_agent" + t.string "user_agent", limit: 255 t.index ["identity_id"], name: "index_sessions_on_identity_id" end @@ -434,7 +434,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "account_id", null: false t.uuid "card_id", null: false t.boolean "completed", default: false, null: false - t.text "content", null: false + t.text "content", limit: 65535, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_steps_on_account_id" @@ -456,7 +456,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "tags", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false t.datetime "created_at", null: false - t.string "title" + t.string "title", limit: 255 t.datetime "updated_at", null: false t.index ["account_id", "title"], name: "index_tags_on_account_id_and_title", unique: true end @@ -465,7 +465,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "account_id", null: false t.integer "bundle_email_frequency", default: 0, null: false t.datetime "created_at", null: false - t.string "timezone_name" + t.string "timezone_name", limit: 255 t.datetime "updated_at", null: false t.uuid "user_id", null: false t.index ["account_id"], name: "index_user_settings_on_account_id" @@ -478,8 +478,8 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.boolean "active", default: true, null: false t.datetime "created_at", null: false t.uuid "identity_id" - t.string "name", null: false - t.string "role", default: "member", null: false + t.string "name", limit: 255, null: false + t.string "role", limit: 255, default: "member", null: false t.datetime "updated_at", null: false t.index ["account_id", "identity_id"], name: "index_users_on_account_id_and_identity_id", unique: true t.index ["account_id", "role"], name: "index_users_on_account_id_and_role" @@ -514,9 +514,9 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "account_id", null: false t.datetime "created_at", null: false t.uuid "event_id", null: false - t.text "request" - t.text "response" - t.string "state", null: false + t.text "request", limit: 65535 + t.text "response", limit: 65535 + t.string "state", limit: 255, null: false t.datetime "updated_at", null: false t.uuid "webhook_id", null: false t.index ["account_id"], name: "index_webhook_deliveries_on_account_id" @@ -529,11 +529,11 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.boolean "active", default: true, null: false t.uuid "board_id", null: false t.datetime "created_at", null: false - t.string "name" - t.string "signing_secret", null: false - t.text "subscribed_actions" + t.string "name", limit: 255 + t.string "signing_secret", limit: 255, null: false + t.text "subscribed_actions", limit: 65535 t.datetime "updated_at", null: false - t.text "url", null: false + t.text "url", limit: 65535, null: false t.index ["account_id"], name: "index_webhooks_on_account_id" t.index ["board_id", "subscribed_actions"], name: "index_webhooks_on_board_id_and_subscribed_actions" end From 97bcdf1853ac57a72ec4febd7fe4f775b327afff Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 25 Nov 2025 15:29:42 +0000 Subject: [PATCH 50/55] Enforce column limits via concern SQLite columns lengths are purely informational, so we'll enforce the limits via a concern that checks the lengths from the schema. --- app/models/application_record.rb | 2 + app/models/concerns/column_limits.rb | 46 ++++++++++++++ .../table_definition_column_limits.rb | 37 ++++++----- db/schema_sqlite.rb | 2 +- test/models/column_limits_test.rb | 62 +++++++++++++++++++ 5 files changed, 134 insertions(+), 15 deletions(-) create mode 100644 app/models/concerns/column_limits.rb create mode 100644 test/models/column_limits_test.rb diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 45e9c2f21..84fdb190c 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -1,6 +1,8 @@ class ApplicationRecord < ActiveRecord::Base primary_abstract_class + include ColumnLimits + configure_replica_connections attribute :id, :uuid, default: -> { ActiveRecord::Type::Uuid.generate } diff --git a/app/models/concerns/column_limits.rb b/app/models/concerns/column_limits.rb new file mode 100644 index 000000000..de179c34b --- /dev/null +++ b/app/models/concerns/column_limits.rb @@ -0,0 +1,46 @@ +# Validates string and text column lengths to match MySQL limits. +# SQLite doesn't enforce column limits, so we need validations to ensure +# data portability between databases. +# +# Usage: +# class Card < ApplicationRecord +# include ColumnLimits +# end +# +# This will automatically add length validations for all string and text +# columns based on their defined limits in the database schema. +# +# MySQL VARCHAR limits are in characters, not bytes. +# MySQL TEXT limits are in bytes. + +module ColumnLimits + extend ActiveSupport::Concern + + included do + validate :validate_column_limits + end + + private + def validate_column_limits + self.class.columns.each do |column| + next unless column.limit + next unless %i[string text].include?(column.type) + + value = read_attribute(column.name) + next if value.nil? + next unless value.is_a?(String) + + if column.type == :string + # VARCHAR limits are in characters + if value.length > column.limit + errors.add(column.name, "is too long (maximum is #{column.limit} characters)") + end + else + # TEXT limits are in bytes + if value.bytesize > column.limit + errors.add(column.name, "is too long (maximum is #{column.limit} bytes)") + end + end + end + end +end diff --git a/config/initializers/table_definition_column_limits.rb b/config/initializers/table_definition_column_limits.rb index f9a75d174..373f19138 100644 --- a/config/initializers/table_definition_column_limits.rb +++ b/config/initializers/table_definition_column_limits.rb @@ -1,4 +1,4 @@ -# Apply MySQL-compatible column limits when using SQLite. +# Apply MySQL-compatible column limits when defining tables. # # For string columns: defaults to 255 (MySQL's VARCHAR default) # @@ -7,17 +7,19 @@ # - size: :tiny: 255 (TINYTEXT) # - size: :medium: 16,777,215 (MEDIUMTEXT) # - size: :long: 4,294,967,295 (LONGTEXT) +# +# This ensures the SQLite schema records the same limits as MySQL, +# which the ColumnLimits validation concern uses to enforce limits. module TableDefinitionColumnLimits - # Map MySQL size options to limits TEXT_SIZE_TO_LIMIT = { - tiny: 255, # TINYTEXT - medium: 16_777_215, # MEDIUMTEXT - long: 4_294_967_295 # LONGTEXT + tiny: 255, + medium: 16_777_215, + long: 4_294_967_295 }.freeze - TEXT_DEFAULT_LIMIT = 65_535 # TEXT - STRING_DEFAULT_LIMIT = 255 # VARCHAR + TEXT_DEFAULT_LIMIT = 65_535 + STRING_DEFAULT_LIMIT = 255 def column(name, type, **options) if type == :string @@ -27,16 +29,11 @@ module TableDefinitionColumnLimits if type == :text if options.key?(:size) size = options.delete(:size) - options[:limit] = TEXT_SIZE_TO_LIMIT.fetch(size) do + options[:limit] ||= TEXT_SIZE_TO_LIMIT.fetch(size) do raise ArgumentError, "Unknown text size: #{size.inspect}. Use :tiny, :medium, or :long" end - elsif options.key?(:limit) - valid_limits = [TEXT_DEFAULT_LIMIT] + TEXT_SIZE_TO_LIMIT.values - unless valid_limits.include?(options[:limit]) - raise ArgumentError, "Invalid limit #{options[:limit]} for text column. Use `size:` (:tiny, :medium, :long) or omit for default TEXT." - end else - options[:limit] = TEXT_DEFAULT_LIMIT + options[:limit] ||= TEXT_DEFAULT_LIMIT end end @@ -47,3 +44,15 @@ end ActiveSupport.on_load(:active_record) do ActiveRecord::ConnectionAdapters::TableDefinition.prepend(TableDefinitionColumnLimits) end + +ActiveSupport.on_load(:action_text_rich_text) do + include ColumnLimits +end + +ActiveSupport.on_load(:active_storage_blob) do + include ColumnLimits +end + +ActiveSupport.on_load(:active_storage_attachment) do + include ColumnLimits +end diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb index bc40a0f42..d3533e53d 100644 --- a/db/schema_sqlite.rb +++ b/db/schema_sqlite.rb @@ -46,7 +46,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "action_text_rich_texts", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false - t.text "body", limit: 65535 + t.text "body", limit: 4294967295 t.datetime "created_at", null: false t.string "name", limit: 255, null: false t.uuid "record_id", null: false diff --git a/test/models/column_limits_test.rb b/test/models/column_limits_test.rb new file mode 100644 index 000000000..68144e99c --- /dev/null +++ b/test/models/column_limits_test.rb @@ -0,0 +1,62 @@ +require "test_helper" + +class ColumnLimitsTest < ActiveSupport::TestCase + test "account name rejects strings over 255 characters" do + account = Account.new(name: "a" * 256) + assert_not account.valid? + assert_includes account.errors[:name], "is too long (maximum is 255 characters)" + end + + test "account name accepts strings up to 255 characters" do + account = Account.new(name: "a" * 255) + assert account.valid? + assert_not_includes account.errors[:name], "is too long (maximum is 255 characters)" + end + + test "account name accepts 255 emoji characters" do + account = Account.new(name: "🎉" * 255) + assert account.valid? + assert_not_includes account.errors[:name], "is too long (maximum is 255 characters)" + end + + test "account name rejects 256 emoji characters" do + account = Account.new(name: "🎉" * 256) + assert_not account.valid? + assert_includes account.errors[:name], "is too long (maximum is 255 characters)" + end + + # Test text column limits (65535 bytes for TEXT) + test "step content rejects text over 65535 bytes" do + step = Step.new(content: "a" * 65536, card: cards(:logo)) + assert_not step.valid? + assert_includes step.errors[:content], "is too long (maximum is 65535 bytes)" + end + + test "step content accepts text up to 65535 bytes" do + step = Step.new(content: "a" * 65535, card: cards(:logo)) + assert step.valid? + assert_not_includes step.errors[:content], "is too long (maximum is 65535 bytes)" + end + + test "step content counts bytes not characters for text columns" do + # 20000 emoji = 20000 chars but 80000 bytes (over 65535 limit) + step = Step.new(content: "🎉" * 20000, card: cards(:logo)) + assert_not step.valid? + assert_includes step.errors[:content], "is too long (maximum is 65535 bytes)" + end + + # Test ActionText::RichText (inherits from ActionText::Record, not ApplicationRecord) + test "ActionText::RichText name rejects strings over 255 characters" do + rich_text = ActionText::RichText.new(name: "a" * 256, record: cards(:logo)) + assert_not rich_text.valid? + assert_includes rich_text.errors[:name], "is too long (maximum is 255 characters)" + end + + # Test ActiveStorage::Blob (inherits from ActiveStorage::Record, not ApplicationRecord) + test "ActiveStorage::Blob filename rejects strings over 255 characters" do + Current.account = accounts(:"37s") + blob = ActiveStorage::Blob.new(filename: "a" * 256) + assert_not blob.valid? + assert_includes blob.errors[:filename], "is too long (maximum is 255 characters)" + end +end From 9c862055108fe72f4d3ce47d25a8a14074e2e52a Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 26 Nov 2025 09:50:29 +0000 Subject: [PATCH 51/55] Enforce SQLite column limits via CHECK constraints Patch the sqlite adapter to add CHECK constraints for string and text column limits. We'll do them inline, so that any column changes automatically update the constraints. --- app/models/application_record.rb | 2 - app/models/concerns/column_limits.rb | 46 ------------------- .../table_definition_column_limits.rb | 32 +++++++++---- test/models/column_limits_test.rb | 36 ++++++--------- 4 files changed, 37 insertions(+), 79 deletions(-) delete mode 100644 app/models/concerns/column_limits.rb diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 84fdb190c..45e9c2f21 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -1,8 +1,6 @@ class ApplicationRecord < ActiveRecord::Base primary_abstract_class - include ColumnLimits - configure_replica_connections attribute :id, :uuid, default: -> { ActiveRecord::Type::Uuid.generate } diff --git a/app/models/concerns/column_limits.rb b/app/models/concerns/column_limits.rb deleted file mode 100644 index de179c34b..000000000 --- a/app/models/concerns/column_limits.rb +++ /dev/null @@ -1,46 +0,0 @@ -# Validates string and text column lengths to match MySQL limits. -# SQLite doesn't enforce column limits, so we need validations to ensure -# data portability between databases. -# -# Usage: -# class Card < ApplicationRecord -# include ColumnLimits -# end -# -# This will automatically add length validations for all string and text -# columns based on their defined limits in the database schema. -# -# MySQL VARCHAR limits are in characters, not bytes. -# MySQL TEXT limits are in bytes. - -module ColumnLimits - extend ActiveSupport::Concern - - included do - validate :validate_column_limits - end - - private - def validate_column_limits - self.class.columns.each do |column| - next unless column.limit - next unless %i[string text].include?(column.type) - - value = read_attribute(column.name) - next if value.nil? - next unless value.is_a?(String) - - if column.type == :string - # VARCHAR limits are in characters - if value.length > column.limit - errors.add(column.name, "is too long (maximum is #{column.limit} characters)") - end - else - # TEXT limits are in bytes - if value.bytesize > column.limit - errors.add(column.name, "is too long (maximum is #{column.limit} bytes)") - end - end - end - end -end diff --git a/config/initializers/table_definition_column_limits.rb b/config/initializers/table_definition_column_limits.rb index 373f19138..77dd1ded2 100644 --- a/config/initializers/table_definition_column_limits.rb +++ b/config/initializers/table_definition_column_limits.rb @@ -8,8 +8,6 @@ # - size: :medium: 16,777,215 (MEDIUMTEXT) # - size: :long: 4,294,967,295 (LONGTEXT) # -# This ensures the SQLite schema records the same limits as MySQL, -# which the ColumnLimits validation concern uses to enforce limits. module TableDefinitionColumnLimits TEXT_SIZE_TO_LIMIT = { @@ -45,14 +43,28 @@ ActiveSupport.on_load(:active_record) do ActiveRecord::ConnectionAdapters::TableDefinition.prepend(TableDefinitionColumnLimits) end -ActiveSupport.on_load(:action_text_rich_text) do - include ColumnLimits +# For SQLite: append inline CHECK constraints to enforce string/text length limits. +# since SQLite doesn't natively enforce VARCHAR/TEXT length limits. +module SQLiteColumnLimitCheckConstraints + def add_column_options!(sql, options) + super + + column = options[:column] + if column && column.limit && %i[string text].include?(column.type) + check_expr = if column.type == :string + # VARCHAR limits are in characters + %(length("#{column.name}") <= #{column.limit}) + else + # TEXT limits are in bytes + %(length(CAST("#{column.name}" AS BLOB)) <= #{column.limit}) + end + sql << " CHECK(#{check_expr})" + end + + sql + end end -ActiveSupport.on_load(:active_storage_blob) do - include ColumnLimits -end - -ActiveSupport.on_load(:active_storage_attachment) do - include ColumnLimits +ActiveSupport.on_load(:active_record_sqlite3adapter) do + ActiveRecord::ConnectionAdapters::SQLite3::SchemaCreation.prepend(SQLiteColumnLimitCheckConstraints) end diff --git a/test/models/column_limits_test.rb b/test/models/column_limits_test.rb index 68144e99c..ba5f67b92 100644 --- a/test/models/column_limits_test.rb +++ b/test/models/column_limits_test.rb @@ -1,62 +1,56 @@ require "test_helper" class ColumnLimitsTest < ActiveSupport::TestCase + # Database errors for exceeding column limits: + # - MySQL: ActiveRecord::ValueTooLong + # - SQLite: ActiveRecord::CheckViolation + COLUMN_LIMIT_ERRORS = [ ActiveRecord::ValueTooLong, ActiveRecord::CheckViolation ] + test "account name rejects strings over 255 characters" do account = Account.new(name: "a" * 256) - assert_not account.valid? - assert_includes account.errors[:name], "is too long (maximum is 255 characters)" + assert_raises(*COLUMN_LIMIT_ERRORS) { account.save! } end test "account name accepts strings up to 255 characters" do account = Account.new(name: "a" * 255) - assert account.valid? - assert_not_includes account.errors[:name], "is too long (maximum is 255 characters)" + assert account.save end test "account name accepts 255 emoji characters" do account = Account.new(name: "🎉" * 255) - assert account.valid? - assert_not_includes account.errors[:name], "is too long (maximum is 255 characters)" + assert account.save end test "account name rejects 256 emoji characters" do account = Account.new(name: "🎉" * 256) - assert_not account.valid? - assert_includes account.errors[:name], "is too long (maximum is 255 characters)" + assert_raises(*COLUMN_LIMIT_ERRORS) { account.save! } end # Test text column limits (65535 bytes for TEXT) test "step content rejects text over 65535 bytes" do step = Step.new(content: "a" * 65536, card: cards(:logo)) - assert_not step.valid? - assert_includes step.errors[:content], "is too long (maximum is 65535 bytes)" + assert_raises(*COLUMN_LIMIT_ERRORS) { step.save! } end test "step content accepts text up to 65535 bytes" do step = Step.new(content: "a" * 65535, card: cards(:logo)) - assert step.valid? - assert_not_includes step.errors[:content], "is too long (maximum is 65535 bytes)" + assert step.save end test "step content counts bytes not characters for text columns" do # 20000 emoji = 20000 chars but 80000 bytes (over 65535 limit) step = Step.new(content: "🎉" * 20000, card: cards(:logo)) - assert_not step.valid? - assert_includes step.errors[:content], "is too long (maximum is 65535 bytes)" + assert_raises(*COLUMN_LIMIT_ERRORS) { step.save! } end - # Test ActionText::RichText (inherits from ActionText::Record, not ApplicationRecord) test "ActionText::RichText name rejects strings over 255 characters" do rich_text = ActionText::RichText.new(name: "a" * 256, record: cards(:logo)) - assert_not rich_text.valid? - assert_includes rich_text.errors[:name], "is too long (maximum is 255 characters)" + assert_raises(*COLUMN_LIMIT_ERRORS) { rich_text.save! } end - # Test ActiveStorage::Blob (inherits from ActiveStorage::Record, not ApplicationRecord) test "ActiveStorage::Blob filename rejects strings over 255 characters" do Current.account = accounts(:"37s") - blob = ActiveStorage::Blob.new(filename: "a" * 256) - assert_not blob.valid? - assert_includes blob.errors[:filename], "is too long (maximum is 255 characters)" + blob = ActiveStorage::Blob.new(filename: "a" * 256, key: "test-key", byte_size: 0, checksum: "test", service_name: "local") + assert_raises(*COLUMN_LIMIT_ERRORS) { blob.save! } end end From b839340cf2a1ef20d20af3ce4f94c2fb7d0b2dc5 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 26 Nov 2025 10:02:32 +0000 Subject: [PATCH 52/55] Tidy up modules and on_load points --- config/initializers/sqlite_schema_dumper.rb | 6 +- .../table_definition_column_limits.rb | 8 +- config/initializers/uuid_primary_keys.rb | 134 ++++++++---------- 3 files changed, 67 insertions(+), 81 deletions(-) diff --git a/config/initializers/sqlite_schema_dumper.rb b/config/initializers/sqlite_schema_dumper.rb index b3d76b186..782c0b6b2 100644 --- a/config/initializers/sqlite_schema_dumper.rb +++ b/config/initializers/sqlite_schema_dumper.rb @@ -17,8 +17,6 @@ module SQLiteFTS5SchemaDumperFix end end -ActiveSupport.on_load(:active_record) do - if defined?(ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper) - ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper.prepend(SQLiteFTS5SchemaDumperFix) - end +ActiveSupport.on_load(:active_record_sqlite3adapter) do + ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper.prepend(SQLiteFTS5SchemaDumperFix) end diff --git a/config/initializers/table_definition_column_limits.rb b/config/initializers/table_definition_column_limits.rb index 77dd1ded2..ff01c5e3a 100644 --- a/config/initializers/table_definition_column_limits.rb +++ b/config/initializers/table_definition_column_limits.rb @@ -39,10 +39,6 @@ module TableDefinitionColumnLimits end end -ActiveSupport.on_load(:active_record) do - ActiveRecord::ConnectionAdapters::TableDefinition.prepend(TableDefinitionColumnLimits) -end - # For SQLite: append inline CHECK constraints to enforce string/text length limits. # since SQLite doesn't natively enforce VARCHAR/TEXT length limits. module SQLiteColumnLimitCheckConstraints @@ -65,6 +61,10 @@ module SQLiteColumnLimitCheckConstraints end end +ActiveSupport.on_load(:active_record) do + ActiveRecord::ConnectionAdapters::TableDefinition.prepend(TableDefinitionColumnLimits) +end + ActiveSupport.on_load(:active_record_sqlite3adapter) do ActiveRecord::ConnectionAdapters::SQLite3::SchemaCreation.prepend(SQLiteColumnLimitCheckConstraints) end diff --git a/config/initializers/uuid_primary_keys.rb b/config/initializers/uuid_primary_keys.rb index 698da184f..56ff7f3af 100644 --- a/config/initializers/uuid_primary_keys.rb +++ b/config/initializers/uuid_primary_keys.rb @@ -1,95 +1,83 @@ # Automatically use UUID type for all binary(16) columns -ActiveSupport.on_load(:active_record) do - module MysqlUuidAdapter - # Add UUID to MySQL's native database types +module MysqlUuidAdapter + extend ActiveSupport::Concern + + # Override lookup_cast_type to recognize binary(16) as UUID type + def lookup_cast_type(sql_type) + if sql_type == "binary(16)" + ActiveRecord::Type.lookup(:uuid, adapter: :trilogy) + else + super + end + end + + class_methods do def native_database_types @native_database_types_with_uuid ||= super.merge(uuid: { name: "binary", limit: 16 }) end + end +end - # Override lookup_cast_type to recognize binary(16) as UUID type - def lookup_cast_type(sql_type) - if sql_type == "binary(16)" - ActiveRecord::Type.lookup(:uuid, adapter: :trilogy) - else - super - end +module SqliteUuidAdapter + extend ActiveSupport::Concern + + # Override lookup_cast_type to recognize BLOB as UUID type + def lookup_cast_type(sql_type) + if sql_type == "blob(16)" + ActiveRecord::Type.lookup(:uuid, adapter: :sqlite3) + else + super end end - if defined?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter) - ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlUuidAdapter) + # Override fetch_type_metadata to preserve UUID type and limit + def fetch_type_metadata(sql_type) + if sql_type == "blob(16)" + ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new( + sql_type: sql_type, + type: :uuid, + limit: 16 + ) + else + super + end end - module SqliteUuidAdapter + class_methods do def native_database_types @native_database_types_with_uuid ||= super.merge(uuid: { name: "blob", limit: 16 }) end + end +end - # Override lookup_cast_type to recognize BLOB as UUID type - def lookup_cast_type(sql_type) - if sql_type == "blob(16)" - ActiveRecord::Type.lookup(:uuid, adapter: :sqlite3) - else - super - end - end - - # Override fetch_type_metadata to preserve UUID type and limit - def fetch_type_metadata(sql_type) - if sql_type == "blob(16)" - ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new( - sql_type: sql_type, - type: :uuid, - limit: 16 - ) - else - super - end +module SchemaDumperUuidType + # Map binary(16) and blob(16) columns to :uuid type in schema.rb + def schema_type(column) + if column.sql_type == "binary(16)" || column.sql_type == "blob(16)" + :uuid + else + super end end +end - if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) - ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(SqliteUuidAdapter) - - # Also add UUID to class-level native_database_types - ActiveRecord::ConnectionAdapters::SQLite3Adapter.class_eval do - @native_database_types = nil # Clear cache - - class << self - alias_method :original_native_database_types, :native_database_types - - def native_database_types - @native_database_types_with_uuid ||= original_native_database_types.merge(uuid: { name: "blob", limit: 16 }) - end - end - end - end - - module SchemaDumperUuidType - # Map binary(16) and blob(16) columns to :uuid type in schema.rb - def schema_type(column) - if column.sql_type == "binary(16)" || column.sql_type == "blob(16)" - :uuid - else - super - end - end - end - - if defined?(ActiveRecord::ConnectionAdapters::MySQL::SchemaDumper) - ActiveRecord::ConnectionAdapters::MySQL::SchemaDumper.prepend(SchemaDumperUuidType) - end - - if defined?(ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper) - ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper.prepend(SchemaDumperUuidType) - end - - module TableDefinitionUuidSupport - def uuid(name, **options) - column(name, :uuid, **options) - end +module TableDefinitionUuidSupport + def uuid(name, **options) + column(name, :uuid, **options) end +end +ActiveSupport.on_load(:active_record) do ActiveRecord::ConnectionAdapters::TableDefinition.prepend(TableDefinitionUuidSupport) end + +ActiveSupport.on_load(:active_record_trilogyadapter) do + ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlUuidAdapter) + ActiveRecord::ConnectionAdapters::MySQL::SchemaDumper.prepend(SchemaDumperUuidType) +end + +ActiveSupport.on_load(:active_record_sqlite3adapter) do + ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(SqliteUuidAdapter) + ActiveRecord::ConnectionAdapters::SQLite3::SchemaDumper.prepend(SchemaDumperUuidType) +end From a5fc6eeaec54bb4234966844a1fe8c591835df71 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 26 Nov 2025 10:18:55 +0000 Subject: [PATCH 53/55] Update SQLite schema --- db/schema_sqlite.rb | 7 ++++--- gems/fizzy-saas/lib/fizzy/saas/engine.rb | 10 ++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb index d3533e53d..fe2aad115 100644 --- a/db/schema_sqlite.rb +++ b/db/schema_sqlite.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do +ActiveRecord::Schema[8.2].define(version: 2025_11_25_130010) do create_table "accesses", id: :uuid, force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false @@ -300,6 +300,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do create_table "identities", id: :uuid, force: :cascade do |t| t.datetime "created_at", null: false t.string "email_address", limit: 255, null: false + t.boolean "staff", default: false, null: false t.datetime "updated_at", null: false t.index ["email_address"], name: "index_identities_on_email_address", unique: true end @@ -378,7 +379,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.text "endpoint", limit: 65535 t.string "p256dh_key", limit: 255 t.datetime "updated_at", null: false - t.string "user_agent", limit: 255 + t.string "user_agent", limit: 4096 t.uuid "user_id", null: false t.index ["account_id"], name: "index_push_subscriptions_on_account_id" t.index ["user_id", "endpoint"], name: "index_push_subscriptions_on_user_id_and_endpoint", unique: true @@ -426,7 +427,7 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_21_112416) do t.uuid "identity_id", null: false t.string "ip_address", limit: 255 t.datetime "updated_at", null: false - t.string "user_agent", limit: 255 + t.string "user_agent", limit: 4096 t.index ["identity_id"], name: "index_sessions_on_identity_id" end diff --git a/gems/fizzy-saas/lib/fizzy/saas/engine.rb b/gems/fizzy-saas/lib/fizzy/saas/engine.rb index 8db0f40bf..14ce03759 100644 --- a/gems/fizzy-saas/lib/fizzy/saas/engine.rb +++ b/gems/fizzy-saas/lib/fizzy/saas/engine.rb @@ -8,10 +8,12 @@ module Fizzy Queenbee.host_app = Fizzy initializer "fizzy_saas.transaction_pinning" do |app| - app.config.middleware.insert_after( - ActiveRecord::Middleware::DatabaseSelector, - TransactionPinning::Middleware - ) + if ActiveRecord::Base.replica_configured? + app.config.middleware.insert_after( + ActiveRecord::Middleware::DatabaseSelector, + TransactionPinning::Middleware + ) + end end config.to_prepare do From 18fa8778825f4478ff46e14a4d93f6a471419704 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 26 Nov 2025 10:27:06 +0000 Subject: [PATCH 54/55] Switch to a pool size of 5 --- config/database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/database.yml b/config/database.yml index 95992c2d0..913ba7cba 100644 --- a/config/database.yml +++ b/config/database.yml @@ -19,7 +19,7 @@ default: &default <% if use_sqlite %> adapter: sqlite3 - pool: 50 + pool: 5 timeout: 5000 <% else %> adapter: trilogy From 761f86d78d9db701f06b46238a43e4f9c2eaf202 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 26 Nov 2025 10:30:02 +0000 Subject: [PATCH 55/55] Revert unnecessary fixture change --- test/fixtures/sessions.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/fixtures/sessions.yml b/test/fixtures/sessions.yml index cf6abd120..a062f53fd 100644 --- a/test/fixtures/sessions.yml +++ b/test/fixtures/sessions.yml @@ -7,5 +7,5 @@ kevin: jz: identity: jz -mike: - identity: mike +jason: + identity: jason