Remove semantic searches

We may revisit in the future but we'll go with keyword search only for launch

Gemfile

@@ -50,7 +50,6 @@ gem "rails_structured_logging", bc: "rails-structured-logging"
 
 # AI
 gem "ruby_llm", git: "https://github.com/crmne/ruby_llm.git"
-gem "sqlite-vec", "0.1.7.alpha.2"
 gem "tiktoken_ruby"
 
 group :development, :test do

Gemfile.lock

@@ -519,11 +519,6 @@ GEM
       fugit (~> 1.11.0)
       railties (>= 7.1)
       thor (~> 1.3.1)
-    sqlite-vec (0.1.7.alpha.2-arm64-darwin)
-    sqlite-vec (0.1.7.alpha.2-arm64-linux)
-    sqlite-vec (0.1.7.alpha.2-x86_64-darwin)
-    sqlite-vec (0.1.7.alpha.2-x86_64-linux)
-    sqlite-vec (0.1.7.alpha.2-x86_64-mingw32)
     sqlite3 (2.7.0)
       mini_portile2 (~> 2.8.0)
     sqlite3 (2.7.0-arm64-darwin)
@@ -623,7 +618,6 @@ DEPENDENCIES
   solid_cable (>= 3.0)
   solid_cache (~> 1.0)
   solid_queue (~> 1.1)
-  sqlite-vec (= 0.1.7.alpha.2)
   sqlite3 (>= 2.0)
   stimulus-rails
   thruster

app/jobs/search/refresh_embedding_job.rb

@@ -1,5 +0,0 @@
-class Search::RefreshEmbeddingJob < ApplicationJob
-  def perform(record)
-    record.refresh_search_embedding
-  end
-end

app/models/command/parser/context.rb

@@ -12,7 +12,7 @@ class Command::Parser::Context
     if viewing_card_contents?
       user.accessible_cards.where id: params[:id]
     elsif viewing_list_of_cards?
-      filter.cards.published
+      filtered_cards
     else
       Card.none
     end
@@ -33,6 +33,9 @@ class Command::Parser::Context
   private
     attr_reader :controller, :action, :params
 
+    MAX_CARDS = 20
+    MAX_CLOSED_CARDS = 10
+
     def extract_url_components
       uri = URI.parse(url || "")
       route = Rails.application.routes.recognize_path(uri.path)
@@ -40,4 +43,11 @@ class Command::Parser::Context
       @action = route[:action]
       @params =  ActionController::Parameters.new(Rack::Utils.parse_nested_query(uri.query).merge(route.except(:controller, :action)))
     end
+
+    def filtered_cards
+      open_cards = filter.cards.published.limit(MAX_CARDS)
+      closed_cards = filter.indexed_by.stalled? || filter.indexed_by.closed? ? Card.none : filter.with(indexed_by: "closed").cards.limit(MAX_CLOSED_CARDS)
+      Rails.logger.info "CLOSED CARDS: #{closed_cards.collect(&:title)}"
+      user.accessible_cards.where(id: open_cards.ids + closed_cards.ids)
+    end
 end

app/models/concerns/searchable.rb

@@ -1,14 +1,6 @@
 module Searchable
   extend ActiveSupport::Concern
 
-  included do
-    has_one :search_embedding, as: :record, dependent: :destroy, class_name: "Search::Embedding"
-
-    after_create_commit :refresh_search_embedding_later
-    after_update_commit :refresh_search_embedding_later
-    after_destroy_commit :remove_search_embedding
-  end
-
   class_methods do
     def searchable_by(*fields, using:)
       define_method :search_fields do
@@ -38,13 +30,6 @@ module Searchable
           none
         end
       end
-
-      scope :search_similar, ->(query) do
-        query_embedding = Rails.cache.fetch("embed-search:#{query}") { RubyLLM.embed(Ai::Tokenizer.truncate(query)) }
-        joins(:search_embedding)
-          .where("embedding MATCH ? AND k = ?", query_embedding.vectors.to_json, 20)
-          .order(:distance)
-      end
     end
   end
 
@@ -52,12 +37,6 @@ module Searchable
     update_in_search_index
   end
 
-  def refresh_search_embedding
-    embedding = RubyLLM.embed(Ai::Tokenizer.truncate(search_embedding_content))
-    search_embedding = self.search_embedding || build_search_embedding
-    search_embedding.update! embedding: embedding.vectors.to_json
-  end
-
   private
     def create_in_search_index
       fields_sql = [ "rowid", *search_fields ].join(", ")
@@ -88,16 +67,8 @@ module Searchable
       execute_sql_with_binds "delete from #{search_table} where rowid = ?", id
     end
 
-    def refresh_search_embedding_later
-      Search::RefreshEmbeddingJob.perform_later(self)
-    end
-
     def execute_sql_with_binds(*statement)
       self.class.connection.execute self.class.sanitize_sql(statement)
       self.class.connection.raw_connection.changes.nonzero?
     end
-
-    def remove_search_embedding
-      search_embedding&.destroy
-    end
 end

app/models/filter.rb

@@ -29,7 +29,7 @@ class Filter < ApplicationRecord
       result = result.in_stage(stages.ids) if stages.present?
       result = result.tagged_with(tags.ids) if tags.present?
       result = terms.reduce(result) do |result, term|
-        result.similar_to(term)
+        result.mentioning(term)
       end
 
       result

app/models/search/embedding.rb

@@ -1,5 +0,0 @@
-class Search::Embedding < ApplicationRecord
-  self.table_name = "search_embeddings"
-
-  belongs_to :record, polymorphic: true
-end

config/application.rb

@@ -1,7 +1,6 @@
 require_relative "boot"
 
 require "rails/all"
-require "sqlite_vec" # Referenced in database.yml
 
 # Require the gems listed in Gemfile, including any gems
 # you've limited to :test, :development, or :production.

config/database.yml

@@ -8,8 +8,6 @@ default: &default
   adapter: sqlite3
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
   timeout: 5000
-  extensions:
-    - <%= SqliteVec.loadable_path %>
 
 development:
   primary:
@@ -42,8 +40,6 @@ production: &production
     <<: *default
     database: file:storage/tenants/<%= Rails.env %>/%{tenant}/db/main.sqlite3?vfs=beamer
     tenanted: true
-    extensions:
-      - <%= SqliteVec.loadable_path %>
   primary_original:
     <<: *default
     database: storage/production.sqlite3

db/migrate/20250625073215_remove_search_embeddings.rb

@@ -0,0 +1,5 @@
+class RemoveSearchEmbeddings < ActiveRecord::Migration[8.1]
+  def change
+    drop_table :search_embeddings
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do
+ActiveRecord::Schema[8.1].define(version: 2025_06_25_073215) do
   create_table "accesses", force: :cascade do |t|
     t.integer "collection_id", null: false
     t.datetime "created_at", null: false
@@ -300,17 +300,11 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do
     t.index ["reacter_id"], name: "index_reactions_on_reacter_id"
   end
 
-# Could not dump table "search_embeddings_vector_chunks00" because of following StandardError
-#   Unknown type '' for column 'rowid'
-
-
   create_table "search_queries", force: :cascade do |t|
     t.datetime "created_at", null: false
     t.string "terms", limit: 2000, null: false
     t.datetime "updated_at", null: false
-    t.integer "user_id", null: false
-    t.index ["user_id", "terms"], name: "index_search_queries_on_user_id_and_terms"
-    t.index ["user_id"], name: "index_search_queries_on_user_id"
+    t.index ["terms"], name: "index_search_queries_on_terms"
   end
 
   create_table "search_results", force: :cascade do |t|
@@ -340,7 +334,6 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do
     t.datetime "created_at", null: false
     t.string "title"
     t.datetime "updated_at", null: false
-    t.index ["title"], name: "index_tags_on_account_id_and_title", unique: true
   end
 
   create_table "users", force: :cascade do |t|
@@ -401,7 +394,6 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do
   add_foreign_key "notifications", "users", column: "creator_id"
   add_foreign_key "pins", "cards"
   add_foreign_key "pins", "users"
-  add_foreign_key "search_queries", "users"
   add_foreign_key "sessions", "users"
   add_foreign_key "taggings", "cards"
   add_foreign_key "taggings", "tags"
@@ -413,5 +405,4 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_24_080408) do
   # Note that virtual tables may not work with other database engines. Be careful if changing database.
   create_virtual_table "cards_search_index", "fts5", ["title", "description"]
   create_virtual_table "comments_search_index", "fts5", ["body"]
-  create_virtual_table "search_embeddings", "vec0", ["id INTEGER PRIMARY KEY", "record_type TEXT NOT NULL", "record_id INTEGER NOT NULL", "embedding FLOAT[1536] distance_metric=cosine"]
 end

db/schema_cache.yml

@@ -876,43 +876,6 @@ columns:
     default_function:
     collation:
     comment:
-  search_embeddings_vector_chunks00:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: rowid
-    cast_type: !ruby/object:ActiveModel::Type::Value
-      precision:
-      scale:
-      limit:
-    sql_type_metadata: !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata
-      sql_type: ''
-      type:
-      limit:
-      precision:
-      scale:
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: vectors
-    cast_type: !ruby/object:ActiveModel::Type::Binary
-      precision:
-      scale:
-      limit:
-    sql_type_metadata: !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata
-      sql_type: BLOB
-      type: :binary
-      limit:
-      precision:
-      scale:
-    'null': false
-    default:
-    default_function:
-    collation:
-    comment:
   search_queries:
   - *5
   - *6
@@ -937,7 +900,6 @@ columns:
     collation:
     comment:
   - *9
-  - *25
   search_results:
   - *5
   - *6
@@ -1112,7 +1074,6 @@ primary_keys:
   pins: id
   reactions: id
   schema_migrations: version
-  search_embeddings_vector_chunks00: rowid
   search_queries: id
   search_results: id
   sessions: id
@@ -1155,7 +1116,6 @@ data_sources:
   pins: true
   reactions: true
   schema_migrations: true
-  search_embeddings_vector_chunks00: true
   search_queries: true
   search_results: true
   sessions: true
@@ -2217,30 +2177,12 @@ indexes:
     comment:
     valid: true
   schema_migrations: []
-  search_embeddings_vector_chunks00: []
   search_queries:
   - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition
     table: search_queries
-    name: index_search_queries_on_user_id
+    name: index_search_queries_on_terms
     unique: false
     columns:
-    - user_id
-    lengths: {}
-    orders: {}
-    opclasses: {}
-    where:
-    type:
-    using:
-    include:
-    nulls_not_distinct:
-    comment:
-    valid: true
-  - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition
-    table: search_queries
-    name: index_search_queries_on_user_id_and_terms
-    unique: false
-    columns:
-    - user_id
     - terms
     lengths: {}
     orders: {}
@@ -2304,23 +2246,7 @@ indexes:
     nulls_not_distinct:
     comment:
     valid: true
-  tags:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition
-    table: tags
-    name: index_tags_on_account_id_and_title
-    unique: true
-    columns:
-    - title
-    lengths: {}
-    orders: {}
-    opclasses: {}
-    where:
-    type:
-    using:
-    include:
-    nulls_not_distinct:
-    comment:
-    valid: true
+  tags: []
   users:
   - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition
     table: users
@@ -2421,4 +2347,4 @@ indexes:
     comment:
     valid: true
   workflows: []
-version: 20250624080408
+version: 20250625073215