diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb index af2d3dbc9..d9c84709f 100644 --- a/app/models/card/searchable.rb +++ b/app/models/card/searchable.rb @@ -4,48 +4,22 @@ module Card::Searchable included do include ::Searchable - searchable_by :title_and_description, using: :cards_search_index, as: :title + searchable_by :title, :description, using: :cards_search_index scope :mentioning, ->(query, by_similarity: false) do method = by_similarity ? :search_similar : :search - if query = sanitize_query_syntax(query) - cards = Card.public_send(method, query).select(:id).to_sql - comments = Comment.public_send(method, query).select(:id).to_sql + cards = Card.public_send(method, query).select(:id).to_sql + comments = Comment.public_send(method, query).select(:id).to_sql - left_joins(:comments).where("cards.id in (#{cards}) or comments.id in (#{comments})").distinct - else - none - end + left_joins(:comments).where("cards.id in (#{cards}) or comments.id in (#{comments})").distinct end end - class_methods do - def sanitize_query_syntax(terms) - terms = terms.to_s - terms = remove_invalid_search_characters(terms) - terms = remove_unbalanced_quotes(terms) - terms.presence - end - - private - def remove_invalid_search_characters(terms) - terms.gsub(/[^\w"]/, " ") - end - - def remove_unbalanced_quotes(terms) - if terms.count("\"").even? - terms - else - terms.gsub("\"", " ") - end - end - end - private # TODO: Temporary until we stabilize the search API def title_and_description - [ title, description.to_plain_text ].join(" ") + [title, description.to_plain_text].join(" ") end def search_embedding_content diff --git a/app/models/comment.rb b/app/models/comment.rb index f62f457a0..5b7cd26ec 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -6,7 +6,7 @@ class Comment < ApplicationRecord has_many :reactions, dependent: :delete_all has_rich_text :body - searchable_by :body_plain_text, using: :comments_search_index, as: :body + searchable_by :body, using: :comments_search_index scope :chronologically, -> { order created_at: :asc, id: :desc } @@ -19,10 +19,6 @@ class Comment < ApplicationRecord end private - def body_plain_text - body.to_plain_text - end - def watch_card_by_creator card.watch_by creator end diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 5946212dd..c869c5c05 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -4,22 +4,39 @@ module Searchable included do has_one :search_embedding, as: :record, dependent: :destroy, class_name: "Search::Embedding" - after_create_commit :refresh_search_embedding_later - after_update_commit :refresh_search_embedding_later + after_create_commit :refresh_search_embedding_later + after_update_commit :refresh_search_embedding_later after_destroy_commit :remove_search_embedding end class_methods do - def searchable_by(field, using:, as: field) - define_method :search_value do send(field); end - define_method :search_field do as; end - define_method :search_table do using; end + def searchable_by(*fields, using:) + define_method :search_fields do + fields + end - after_create_commit :create_in_search_index - after_update_commit :update_in_search_index + define_method :search_values do + fields.map do + value = send(it) + value.respond_to?(:to_plain_text) ? value.to_plain_text : value + end + end + + define_method :search_table do + using + end + + after_create_commit :create_in_search_index + after_update_commit :update_in_search_index after_destroy_commit :remove_from_search_index - scope :search, ->(query) { joins("join #{using} idx on #{table_name}.id = idx.rowid").where("idx.#{as} match ?", query) } + scope :search, ->(query) do + if query = sanitize_query_syntax(query) + joins("join #{using} idx on #{table_name}.id = idx.rowid").where("#{using} match ?", query) + else + none + end + end scope :search_similar, ->(query) do query_embedding = Rails.cache.fetch("embed-search:#{query}") { RubyLLM.embed(Ai::Tokenizer.truncate(query)) } joins(:search_embedding) @@ -27,6 +44,26 @@ module Searchable .order(:distance) end end + + def sanitize_query_syntax(terms) + terms = terms.to_s + terms = remove_invalid_search_characters(terms) + terms = remove_unbalanced_quotes(terms) + terms.presence + end + + private + def remove_invalid_search_characters(terms) + terms.gsub(/[^\w"]/, " ") + end + + def remove_unbalanced_quotes(terms) + if terms.count("\"").even? + terms + else + terms.gsub("\"", " ") + end + end end def reindex @@ -41,12 +78,26 @@ module Searchable private def create_in_search_index - execute_sql_with_binds "insert into #{search_table}(rowid, #{search_field}) values (?, ?)", id, search_value + fields_sql = ["rowid", *search_fields].join(", ") + placeholders = (["?"] * (search_fields.size + 1)).join(", ") + values = [id, *search_values] + + execute_sql_with_binds( + "insert into #{search_table}(#{fields_sql}) values (#{placeholders})", + *values + ) end def update_in_search_index transaction do - updated = execute_sql_with_binds "update #{search_table} set #{search_field} = ? where rowid = ?", search_value, id + set_clause = search_fields.map { |field| "#{field} = ?" }.join(", ") + binds = search_values + [id] + + updated = execute_sql_with_binds( + "update #{search_table} set #{set_clause} where rowid = ?", + *binds + ) + create_in_search_index unless updated end end diff --git a/db/migrate/20250624055720_add_description_to_cards_search_index.rb b/db/migrate/20250624055720_add_description_to_cards_search_index.rb new file mode 100644 index 000000000..2f58e6c02 --- /dev/null +++ b/db/migrate/20250624055720_add_description_to_cards_search_index.rb @@ -0,0 +1,6 @@ +class AddDescriptionToCardsSearchIndex < ActiveRecord::Migration[8.1] + def change + drop_virtual_table :cards_search_index, "fts5", ["title"] + create_virtual_table :cards_search_index, "fts5", ["title", "description"] + end +end diff --git a/db/schema.rb b/db/schema.rb index 0c7504173..993ce6ab7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2025_06_12_163028) do +ActiveRecord::Schema[8.1].define(version: 2025_06_24_055720) do create_table "accesses", force: :cascade do |t| t.integer "collection_id", null: false t.datetime "created_at", null: false @@ -326,7 +326,6 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_12_163028) do t.datetime "created_at", null: false t.string "title" t.datetime "updated_at", null: false - t.index ["title"], name: "index_tags_on_account_id_and_title", unique: true end create_table "users", force: :cascade do |t| @@ -396,7 +395,7 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_12_163028) do # Virtual tables defined in this database. # Note that virtual tables may not work with other database engines. Be careful if changing database. - create_virtual_table "cards_search_index", "fts5", ["title"] + create_virtual_table "cards_search_index", "fts5", ["title", "description"] create_virtual_table "comments_search_index", "fts5", ["body"] create_virtual_table "search_embeddings", "vec0", ["id INTEGER PRIMARY KEY", "record_type TEXT NOT NULL", "record_id INTEGER NOT NULL", "embedding FLOAT[1536] distance_metric=cosine"] end diff --git a/db/schema_cache.yml b/db/schema_cache.yml index 2c6a24735..de274674c 100644 --- a/db/schema_cache.yml +++ b/db/schema_cache.yml @@ -2236,23 +2236,7 @@ indexes: nulls_not_distinct: comment: valid: true - tags: - - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition - table: tags - name: index_tags_on_account_id_and_title - unique: true - columns: - - title - lengths: {} - orders: {} - opclasses: {} - where: - type: - using: - include: - nulls_not_distinct: - comment: - valid: true + tags: [] users: - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition table: users @@ -2353,4 +2337,4 @@ indexes: comment: valid: true workflows: [] -version: 20250612163028 +version: 20250624055720