diff --git a/Gemfile b/Gemfile index e5e3637e9..2f56fa81d 100644 --- a/Gemfile +++ b/Gemfile @@ -50,7 +50,6 @@ gem "rails_structured_logging", bc: "rails-structured-logging" # AI gem "ruby_llm", git: "https://github.com/crmne/ruby_llm.git" -gem "sqlite-vec", "0.1.7.alpha.2" gem "tiktoken_ruby" group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index f18b08187..871dc9cc4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -519,11 +519,6 @@ GEM fugit (~> 1.11.0) railties (>= 7.1) thor (~> 1.3.1) - sqlite-vec (0.1.7.alpha.2-arm64-darwin) - sqlite-vec (0.1.7.alpha.2-arm64-linux) - sqlite-vec (0.1.7.alpha.2-x86_64-darwin) - sqlite-vec (0.1.7.alpha.2-x86_64-linux) - sqlite-vec (0.1.7.alpha.2-x86_64-mingw32) sqlite3 (2.7.0) mini_portile2 (~> 2.8.0) sqlite3 (2.7.0-arm64-darwin) @@ -623,7 +618,6 @@ DEPENDENCIES solid_cable (>= 3.0) solid_cache (~> 1.0) solid_queue (~> 1.1) - sqlite-vec (= 0.1.7.alpha.2) sqlite3 (>= 2.0) stimulus-rails thruster diff --git a/app/jobs/search/refresh_embedding_job.rb b/app/jobs/search/refresh_embedding_job.rb deleted file mode 100644 index cef2397d1..000000000 --- a/app/jobs/search/refresh_embedding_job.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Search::RefreshEmbeddingJob < ApplicationJob - def perform(record) - record.refresh_search_embedding - end -end diff --git a/app/models/command/parser/context.rb b/app/models/command/parser/context.rb index 3624ba43d..0efcf0fa9 100644 --- a/app/models/command/parser/context.rb +++ b/app/models/command/parser/context.rb @@ -12,7 +12,7 @@ class Command::Parser::Context if viewing_card_contents? user.accessible_cards.where id: params[:id] elsif viewing_list_of_cards? - filter.cards.published + filtered_cards else Card.none end @@ -33,6 +33,9 @@ class Command::Parser::Context private attr_reader :controller, :action, :params + MAX_CARDS = 20 + MAX_CLOSED_CARDS = 10 + def extract_url_components uri = URI.parse(url || "") route = Rails.application.routes.recognize_path(uri.path) @@ -40,4 +43,11 @@ class Command::Parser::Context @action = route[:action] @params = ActionController::Parameters.new(Rack::Utils.parse_nested_query(uri.query).merge(route.except(:controller, :action))) end + + def filtered_cards + open_cards = filter.cards.published.limit(MAX_CARDS) + closed_cards = filter.indexed_by.stalled? || filter.indexed_by.closed? ? Card.none : filter.with(indexed_by: "closed").cards.limit(MAX_CLOSED_CARDS) + Rails.logger.info "CLOSED CARDS: #{closed_cards.collect(&:title)}" + user.accessible_cards.where(id: open_cards.ids + closed_cards.ids) + end end diff --git a/app/models/concerns/searchable.rb b/app/models/concerns/searchable.rb index 8fcede1ec..5b63c7f09 100644 --- a/app/models/concerns/searchable.rb +++ b/app/models/concerns/searchable.rb @@ -1,14 +1,6 @@ module Searchable extend ActiveSupport::Concern - included do - has_one :search_embedding, as: :record, dependent: :destroy, class_name: "Search::Embedding" - - after_create_commit :refresh_search_embedding_later - after_update_commit :refresh_search_embedding_later - after_destroy_commit :remove_search_embedding - end - class_methods do def searchable_by(*fields, using:) define_method :search_fields do @@ -38,13 +30,6 @@ module Searchable none end end - - scope :search_similar, ->(query) do - query_embedding = Rails.cache.fetch("embed-search:#{query}") { RubyLLM.embed(Ai::Tokenizer.truncate(query)) } - joins(:search_embedding) - .where("embedding MATCH ? AND k = ?", query_embedding.vectors.to_json, 20) - .order(:distance) - end end end @@ -52,12 +37,6 @@ module Searchable update_in_search_index end - def refresh_search_embedding - embedding = RubyLLM.embed(Ai::Tokenizer.truncate(search_embedding_content)) - search_embedding = self.search_embedding || build_search_embedding - search_embedding.update! embedding: embedding.vectors.to_json - end - private def create_in_search_index fields_sql = [ "rowid", *search_fields ].join(", ") @@ -88,16 +67,8 @@ module Searchable execute_sql_with_binds "delete from #{search_table} where rowid = ?", id end - def refresh_search_embedding_later - Search::RefreshEmbeddingJob.perform_later(self) - end - def execute_sql_with_binds(*statement) self.class.connection.execute self.class.sanitize_sql(statement) self.class.connection.raw_connection.changes.nonzero? end - - def remove_search_embedding - search_embedding&.destroy - end end diff --git a/app/models/filter.rb b/app/models/filter.rb index 7e6e932c3..a92b3a402 100644 --- a/app/models/filter.rb +++ b/app/models/filter.rb @@ -29,7 +29,7 @@ class Filter < ApplicationRecord result = result.in_stage(stages.ids) if stages.present? result = result.tagged_with(tags.ids) if tags.present? result = terms.reduce(result) do |result, term| - result.similar_to(term) + result.mentioning(term) end result diff --git a/app/models/search/embedding.rb b/app/models/search/embedding.rb deleted file mode 100644 index 32ee2786c..000000000 --- a/app/models/search/embedding.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Search::Embedding < ApplicationRecord - self.table_name = "search_embeddings" - - belongs_to :record, polymorphic: true -end diff --git a/config/application.rb b/config/application.rb index 76851bea2..765a43731 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,7 +1,6 @@ require_relative "boot" require "rails/all" -require "sqlite_vec" # Referenced in database.yml # Require the gems listed in Gemfile, including any gems # you've limited to :test, :development, or :production. diff --git a/config/database.yml b/config/database.yml index 05bf8e6e4..1eb359d73 100644 --- a/config/database.yml +++ b/config/database.yml @@ -8,8 +8,6 @@ default: &default adapter: sqlite3 pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> timeout: 5000 - extensions: - - <%= SqliteVec.loadable_path %> development: primary: @@ -42,8 +40,6 @@ production: &production <<: *default database: file:storage/tenants/<%= Rails.env %>/%{tenant}/db/main.sqlite3?vfs=beamer tenanted: true - extensions: - - <%= SqliteVec.loadable_path %> primary_original: <<: *default database: storage/production.sqlite3 diff --git a/db/migrate/20250625073215_remove_search_embeddings.rb b/db/migrate/20250625073215_remove_search_embeddings.rb new file mode 100644 index 000000000..0452650b2 --- /dev/null +++ b/db/migrate/20250625073215_remove_search_embeddings.rb @@ -0,0 +1,5 @@ +class RemoveSearchEmbeddings < ActiveRecord::Migration[8.1] + def change + drop_table :search_embeddings + end +end diff --git a/db/schema.rb b/db/schema.rb index 825a6dbef..eb894a811 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do +ActiveRecord::Schema[8.1].define(version: 2025_06_25_073215) do create_table "accesses", force: :cascade do |t| t.integer "collection_id", null: false t.datetime "created_at", null: false @@ -300,17 +300,11 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do t.index ["reacter_id"], name: "index_reactions_on_reacter_id" end -# Could not dump table "search_embeddings_vector_chunks00" because of following StandardError -# Unknown type '' for column 'rowid' - - create_table "search_queries", force: :cascade do |t| t.datetime "created_at", null: false t.string "terms", limit: 2000, null: false t.datetime "updated_at", null: false - t.integer "user_id", null: false - t.index ["user_id", "terms"], name: "index_search_queries_on_user_id_and_terms" - t.index ["user_id"], name: "index_search_queries_on_user_id" + t.index ["terms"], name: "index_search_queries_on_terms" end create_table "search_results", force: :cascade do |t| @@ -340,7 +334,6 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do t.datetime "created_at", null: false t.string "title" t.datetime "updated_at", null: false - t.index ["title"], name: "index_tags_on_account_id_and_title", unique: true end create_table "users", force: :cascade do |t| @@ -401,7 +394,6 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do add_foreign_key "notifications", "users", column: "creator_id" add_foreign_key "pins", "cards" add_foreign_key "pins", "users" - add_foreign_key "search_queries", "users" add_foreign_key "sessions", "users" add_foreign_key "taggings", "cards" add_foreign_key "taggings", "tags" @@ -413,5 +405,4 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do # Note that virtual tables may not work with other database engines. Be careful if changing database. create_virtual_table "cards_search_index", "fts5", ["title", "description"] create_virtual_table "comments_search_index", "fts5", ["body"] - create_virtual_table "search_embeddings", "vec0", ["id INTEGER PRIMARY KEY", "record_type TEXT NOT NULL", "record_id INTEGER NOT NULL", "embedding FLOAT[1536] distance_metric=cosine"] end diff --git a/db/schema_cache.yml b/db/schema_cache.yml index f37790897..84621ba81 100644 --- a/db/schema_cache.yml +++ b/db/schema_cache.yml @@ -876,43 +876,6 @@ columns: default_function: collation: comment: - search_embeddings_vector_chunks00: - - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column - auto_increment: - name: rowid - cast_type: !ruby/object:ActiveModel::Type::Value - precision: - scale: - limit: - sql_type_metadata: !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata - sql_type: '' - type: - limit: - precision: - scale: - 'null': true - default: - default_function: - collation: - comment: - - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column - auto_increment: - name: vectors - cast_type: !ruby/object:ActiveModel::Type::Binary - precision: - scale: - limit: - sql_type_metadata: !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata - sql_type: BLOB - type: :binary - limit: - precision: - scale: - 'null': false - default: - default_function: - collation: - comment: search_queries: - *5 - *6 @@ -937,7 +900,6 @@ columns: collation: comment: - *9 - - *25 search_results: - *5 - *6 @@ -1112,7 +1074,6 @@ primary_keys: pins: id reactions: id schema_migrations: version - search_embeddings_vector_chunks00: rowid search_queries: id search_results: id sessions: id @@ -1155,7 +1116,6 @@ data_sources: pins: true reactions: true schema_migrations: true - search_embeddings_vector_chunks00: true search_queries: true search_results: true sessions: true @@ -2217,30 +2177,12 @@ indexes: comment: valid: true schema_migrations: [] - search_embeddings_vector_chunks00: [] search_queries: - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition table: search_queries - name: index_search_queries_on_user_id + name: index_search_queries_on_terms unique: false columns: - - user_id - lengths: {} - orders: {} - opclasses: {} - where: - type: - using: - include: - nulls_not_distinct: - comment: - valid: true - - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition - table: search_queries - name: index_search_queries_on_user_id_and_terms - unique: false - columns: - - user_id - terms lengths: {} orders: {} @@ -2304,23 +2246,7 @@ indexes: nulls_not_distinct: comment: valid: true - tags: - - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition - table: tags - name: index_tags_on_account_id_and_title - unique: true - columns: - - title - lengths: {} - orders: {} - opclasses: {} - where: - type: - using: - include: - nulls_not_distinct: - comment: - valid: true + tags: [] users: - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition table: users @@ -2421,4 +2347,4 @@ indexes: comment: valid: true workflows: [] -version: 20250624080408 +version: 20250625073215