diff --git a/.gitignore b/.gitignore index 5fb66c9f0..7805910bf 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ # Ignore master key for decrypting credentials and more. /config/master.key + +/config/credentials/development.key diff --git a/Gemfile b/Gemfile index edb02dcf8..d8ee4f09a 100644 --- a/Gemfile +++ b/Gemfile @@ -33,6 +33,9 @@ gem "sentry-ruby" gem "sentry-rails" gem "rails_structured_logging", bc: "rails-structured-logging" +gem "ruby_llm", git: "https://github.com/crmne/ruby_llm.git" +gem 'sqlite-vec', '0.1.7.alpha.2' + group :development, :test do gem "debug" gem "bundler-audit", require: false diff --git a/Gemfile.lock b/Gemfile.lock index fda80efb5..0cb8c35f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,6 +16,19 @@ GIT json rails (>= 6.0.0) +GIT + remote: https://github.com/crmne/ruby_llm.git + revision: aa64332d72bb216a486512dc84b7833b00e7f0a0 + specs: + ruby_llm (1.2.0) + base64 + event_stream_parser (~> 1) + faraday (~> 2) + faraday-multipart (~> 1) + faraday-net_http (~> 3) + faraday-retry (~> 2) + zeitwerk (~> 2) + GIT remote: https://github.com/rails/rails.git revision: bc7f77b32d7f3f26db654cae506c26a959852506 @@ -157,6 +170,17 @@ GEM erubi (1.13.1) et-orbi (1.2.11) tzinfo + event_stream_parser (1.0.0) + faraday (2.13.1) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-multipart (1.1.0) + multipart-post (~> 2.0) + faraday-net_http (3.4.0) + net-http (>= 0.5.0) + faraday-retry (2.3.1) + faraday (~> 2.0) fugit (1.11.1) et-orbi (~> 1, >= 1.2.11) raabro (~> 1.4) @@ -205,9 +229,11 @@ GEM marcel (1.0.4) matrix (0.4.2) mini_mime (1.1.5) - mini_portile2 (2.8.8) minitest (5.25.5) msgpack (1.8.0) + multipart-post (2.4.1) + net-http (0.6.0) + uri net-imap (0.5.7) date net-protocol @@ -223,25 +249,12 @@ GEM net-protocol net-ssh (7.3.0) nio4r (2.7.4) - nokogiri (1.18.8) - mini_portile2 (~> 2.8.2) - racc (~> 1.4) - nokogiri (1.18.8-aarch64-linux-gnu) - racc (~> 1.4) - nokogiri (1.18.8-aarch64-linux-musl) - racc (~> 1.4) - nokogiri (1.18.8-arm-linux-gnu) - racc (~> 1.4) - nokogiri (1.18.8-arm-linux-musl) - racc (~> 1.4) nokogiri (1.18.8-arm64-darwin) racc (~> 1.4) nokogiri (1.18.8-x86_64-darwin) racc (~> 1.4) nokogiri (1.18.8-x86_64-linux-gnu) racc (~> 1.4) - nokogiri (1.18.8-x86_64-linux-musl) - racc (~> 1.4) ostruct (0.6.1) parallel (1.27.0) parser (3.3.8.0) @@ -354,16 +367,12 @@ GEM fugit (~> 1.11.0) railties (>= 7.1) thor (~> 1.3.1) - sqlite3 (2.6.0-aarch64-linux-gnu) - sqlite3 (2.6.0-aarch64-linux-musl) - sqlite3 (2.6.0-arm-linux-gnu) - sqlite3 (2.6.0-arm-linux-musl) + sqlite-vec (0.1.7.alpha.2-arm64-darwin) + sqlite-vec (0.1.7.alpha.2-x86_64-darwin) + sqlite-vec (0.1.7.alpha.2-x86_64-linux) sqlite3 (2.6.0-arm64-darwin) - sqlite3 (2.6.0-x86-linux-gnu) - sqlite3 (2.6.0-x86-linux-musl) sqlite3 (2.6.0-x86_64-darwin) sqlite3 (2.6.0-x86_64-linux-gnu) - sqlite3 (2.6.0-x86_64-linux-musl) sshkit (1.24.0) base64 logger @@ -375,8 +384,6 @@ GEM railties (>= 6.0.0) stringio (3.1.7) thor (1.3.2) - thruster (0.1.13) - thruster (0.1.13-aarch64-linux) thruster (0.1.13-arm64-darwin) thruster (0.1.13-x86_64-darwin) thruster (0.1.13-x86_64-linux) @@ -401,20 +408,9 @@ GEM zeitwerk (2.7.2) PLATFORMS - aarch64-linux - aarch64-linux-gnu - aarch64-linux-musl - arm-linux - arm-linux-gnu - arm-linux-musl arm64-darwin - x86-linux - x86-linux-gnu - x86-linux-musl x86_64-darwin x86_64-linux - x86_64-linux-gnu - x86_64-linux-musl DEPENDENCIES active_record-tenanted! @@ -436,12 +432,14 @@ DEPENDENCIES rouge rqrcode rubocop-rails-omakase + ruby_llm! selenium-webdriver sentry-rails sentry-ruby solid_cable (>= 3.0) solid_cache (~> 1.0) solid_queue (~> 1.1) + sqlite-vec (= 0.1.7.alpha.2) sqlite3 (>= 2.0) stimulus-rails thruster diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb index f28e964cf..b2de0c1c8 100644 --- a/app/models/card/searchable.rb +++ b/app/models/card/searchable.rb @@ -45,4 +45,13 @@ module Card::Searchable def title_and_description [ title, description.to_plain_text ].join(" ") end + + def search_embedding_content + <<~CONTENT + Title: #{title} + Description: #{description.to_plain_text} + Created by: #{creator.name}} + Assigned to: #{assignees.map(&:name).join(", ")}} + CONTENT + end end diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 1631ab9ab..4ba10d046 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -1,6 +1,14 @@ module Searchable extend ActiveSupport::Concern + included do + has_one :search_embedding, as: :record, dependent: :destroy, class_name: "Search::Embedding" + + after_create_commit :refresh_search_embedding + after_update_commit :refresh_search_embedding + after_destroy_commit :remove_search_embedding + end + class_methods do def searchable_by(field, using:, as: field) define_method :search_value do send(field); end @@ -12,6 +20,12 @@ module Searchable after_destroy_commit :remove_from_search_index scope :search, ->(query) { joins("join #{using} idx on #{table_name}.id = idx.rowid").where("idx.#{as} match ?", query) } + scope :search_similar, ->(query) do + query_embedding = RubyLLM.embed(query) + joins(:search_embedding) + .where("embedding MATCH ? AND k = ?", query_embedding.vectors.to_json, 3) + .order(created_at: :desc) + end end end @@ -19,6 +33,12 @@ module Searchable update_in_search_index end + def refresh_search_embedding + embedding = RubyLLM.embed(search_embedding_content) + search_embedding = self.search_embedding || build_search_embedding + search_embedding.update! embedding: embedding.vectors.to_json + end + private def create_in_search_index execute_sql_with_binds "insert into #{search_table}(rowid, #{search_field}) values (?, ?)", id, search_value @@ -39,4 +59,8 @@ module Searchable self.class.connection.execute self.class.sanitize_sql(statement) self.class.connection.raw_connection.changes.nonzero? end + + def remove_search_embedding + search_embedding&.destroy + end end diff --git a/app/models/search/embedding.rb b/app/models/search/embedding.rb new file mode 100644 index 000000000..32ee2786c --- /dev/null +++ b/app/models/search/embedding.rb @@ -0,0 +1,5 @@ +class Search::Embedding < ApplicationRecord + self.table_name = "search_embeddings" + + belongs_to :record, polymorphic: true +end diff --git a/config/credentials/development.yml.enc b/config/credentials/development.yml.enc new file mode 100644 index 000000000..f8caaf960 --- /dev/null +++ b/config/credentials/development.yml.enc @@ -0,0 +1 @@ +tt9xtdgcURdAJblqdkQ9LiJQCxcvrVseNgG53XcR3KMk/yiJ3TIl5AarU0tFFLyCXJbrHmMpkdrKFM/coC/GOsrnl9vvR6PBlyzuRY0MdGvOf7iRT/Ox9RL3wtlYuoSZ0mQ1RPPiK1MYpiDUXJ6qAnSusslIBoOWNHE3v5jmyzR8vBZO/8ioeLMYvF0dPBx2+MuMA5/+ulCPMjeY4ET96V3NF1vl+FiHVlnDlSrJCA4i1uno6hO2YkFa--Whv5bB+DLpCGmo5S--jbLTpg+Kmoj8lJpc73sJ0A== \ No newline at end of file diff --git a/config/initializers/ruby_llm.rb b/config/initializers/ruby_llm.rb new file mode 100644 index 000000000..37c38dca9 --- /dev/null +++ b/config/initializers/ruby_llm.rb @@ -0,0 +1,3 @@ +RubyLLM.configure do |config| + config.openai_api_key = Rails.application.credentials.openai_api_key +end diff --git a/db/migrate/20250510034936_create_search_embeddings.rb b/db/migrate/20250510034936_create_search_embeddings.rb new file mode 100644 index 000000000..688d4a082 --- /dev/null +++ b/db/migrate/20250510034936_create_search_embeddings.rb @@ -0,0 +1,10 @@ +class CreateSearchEmbeddings < ActiveRecord::Migration[7.1] + def change + create_virtual_table :search_embeddings, :vec0, [ + "id INTEGER PRIMARY KEY", + "record_type TEXT NOT NULL", + "record_id INTEGER NOT NULL", + "embedding FLOAT[1536] distance_metric=cosine" + ] + end +end diff --git a/db/schema.rb b/db/schema.rb index ddcbf6956..5cd0c8e30 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2025_05_07_095113) do +ActiveRecord::Schema[8.1].define(version: 2025_05_10_034936) do create_table "accesses", force: :cascade do |t| t.integer "collection_id", null: false t.datetime "created_at", null: false @@ -269,6 +269,10 @@ ActiveRecord::Schema[8.1].define(version: 2025_05_07_095113) do t.index ["reacter_id"], name: "index_reactions_on_reacter_id" end +# Could not dump table "search_embeddings_vector_chunks00" because of following StandardError +# Unknown type '' for column 'rowid' + + create_table "sessions", force: :cascade do |t| t.datetime "created_at", null: false t.string "ip_address" @@ -291,6 +295,7 @@ ActiveRecord::Schema[8.1].define(version: 2025_05_07_095113) do t.datetime "created_at", null: false t.string "title" t.datetime "updated_at", null: false + t.index ["title"], name: "index_tags_on_account_id_and_title", unique: true end create_table "users", force: :cascade do |t| @@ -357,4 +362,5 @@ ActiveRecord::Schema[8.1].define(version: 2025_05_07_095113) do # Note that virtual tables may not work with other database engines. Be careful if changing database. create_virtual_table "cards_search_index", "fts5", ["title"] create_virtual_table "comments_search_index", "fts5", ["body"] + create_virtual_table "search_embeddings", "vec0", ["id INTEGER PRIMARY KEY", "record_type TEXT NOT NULL", "record_id INTEGER NOT NULL", "embedding FLOAT[1536] distance_metric=cosine"] end diff --git a/db/schema_cache.yml b/db/schema_cache.yml index 1ae033f77..63c05a1e2 100644 --- a/db/schema_cache.yml +++ b/db/schema_cache.yml @@ -811,6 +811,43 @@ columns: default_function: collation: comment: + search_embeddings_vector_chunks00: + - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column + auto_increment: + name: rowid + cast_type: !ruby/object:ActiveModel::Type::Value + precision: + scale: + limit: + sql_type_metadata: !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata + sql_type: '' + type: + limit: + precision: + scale: + 'null': true + default: + default_function: + collation: + comment: + - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column + auto_increment: + name: vectors + cast_type: !ruby/object:ActiveModel::Type::Binary + precision: + scale: + limit: + sql_type_metadata: !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata + sql_type: BLOB + type: :binary + limit: + precision: + scale: + 'null': false + default: + default_function: + collation: + comment: sessions: - *5 - *6 @@ -968,6 +1005,7 @@ primary_keys: pins: id reactions: id schema_migrations: version + search_embeddings_vector_chunks00: rowid sessions: id taggings: id tags: id @@ -1005,6 +1043,7 @@ data_sources: pins: true reactions: true schema_migrations: true + search_embeddings_vector_chunks00: true sessions: true taggings: true tags: true @@ -1925,6 +1964,7 @@ indexes: comment: valid: true schema_migrations: [] + search_embeddings_vector_chunks00: [] sessions: - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition table: sessions @@ -1976,7 +2016,23 @@ indexes: nulls_not_distinct: comment: valid: true - tags: [] + tags: + - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition + table: tags + name: index_tags_on_account_id_and_title + unique: true + columns: + - title + lengths: {} + orders: {} + opclasses: {} + where: + type: + using: + include: + nulls_not_distinct: + comment: + valid: true users: - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition table: users @@ -2061,4 +2117,4 @@ indexes: comment: valid: true workflows: [] -version: 20250507095113 +version: 20250510034936 diff --git a/lib/rails_ext/sqlite_vec.rb b/lib/rails_ext/sqlite_vec.rb new file mode 100644 index 000000000..071ef44fb --- /dev/null +++ b/lib/rails_ext/sqlite_vec.rb @@ -0,0 +1,14 @@ +require "sqlite_vec" +require "active_record/connection_adapters/sqlite3_adapter" + +module SqliteVecExtension + def configure_connection + super + db = @raw_connection + db.enable_load_extension(true) + SqliteVec.load(db) + db.enable_load_extension(false) + end +end + +ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(SqliteVecExtension)