Add stemming to search tables

db/migrate/20250625111932_add_stemming.rb

@@ -0,0 +1,9 @@
+class AddStemming < ActiveRecord::Migration[8.1]
+  def change
+    drop_table :cards_search_index
+    drop_table :comments_search_index
+
+    create_virtual_table "cards_search_index", "fts5", [ "title", "description", "tokenize='porter'" ]
+    create_virtual_table "comments_search_index", "fts5", [ "body", "tokenize='porter'" ]
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.1].define(version: 2025_06_25_073215) do
+ActiveRecord::Schema[8.1].define(version: 2025_06_25_111932) do
   create_table "accesses", force: :cascade do |t|
     t.integer "collection_id", null: false
     t.datetime "created_at", null: false
@@ -406,6 +406,6 @@ ActiveRecord::Schema[8.1].define(version: 2025_06_25_073215) do
 
   # Virtual tables defined in this database.
   # Note that virtual tables may not work with other database engines. Be careful if changing database.
-  create_virtual_table "cards_search_index", "fts5", ["title", "description"]
-  create_virtual_table "comments_search_index", "fts5", ["body"]
+  create_virtual_table "cards_search_index", "fts5", ["title", "description", "tokenize='porter'"]
+  create_virtual_table "comments_search_index", "fts5", ["body", "tokenize='porter'"]
 end

db/schema_cache.yml

@@ -441,7 +441,7 @@ columns:
     default_function:
     collation:
     comment:
-  - &37 !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
+  - &31 !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
     auto_increment:
     name: title
     cast_type: *7
@@ -510,11 +510,11 @@ columns:
   - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
     auto_increment:
     name: all_access
-    cast_type: &38 !ruby/object:ActiveModel::Type::Boolean
+    cast_type: &32 !ruby/object:ActiveModel::Type::Boolean
       precision:
       scale:
       limit:
-    sql_type_metadata: &39 !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata
+    sql_type_metadata: &33 !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata
       sql_type: boolean
       type: :boolean
       limit:
@@ -734,7 +734,7 @@ columns:
     comment:
   filters_tags:
   - *19
-  - &36 !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
+  - &30 !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
     auto_increment:
     name: tag_id
     cast_type: *1
@@ -876,178 +876,6 @@ columns:
     default_function:
     collation:
     comment:
-  search_embeddings_chunks:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: chunk_id
-    cast_type: *1
-    sql_type_metadata: *2
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: rowids
-    cast_type: &30 !ruby/object:ActiveModel::Type::Binary
-      precision:
-      scale:
-      limit:
-    sql_type_metadata: &31 !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata
-      sql_type: BLOB
-      type: :binary
-      limit:
-      precision:
-      scale:
-    'null': false
-    default:
-    default_function:
-    collation:
-    comment:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: size
-    cast_type: *1
-    sql_type_metadata: *2
-    'null': false
-    default:
-    default_function:
-    collation:
-    comment:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: validity
-    cast_type: *30
-    sql_type_metadata: *31
-    'null': false
-    default:
-    default_function:
-    collation:
-    comment:
-  search_embeddings_info:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: key
-    cast_type: *15
-    sql_type_metadata: *16
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: value
-    cast_type: &32 !ruby/object:ActiveModel::Type::Value
-      precision:
-      scale:
-      limit:
-    sql_type_metadata: !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata
-      sql_type: ANY
-      type:
-      limit:
-      precision:
-      scale:
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  search_embeddings_metadatachunks00:
-  - &33 !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: data
-    cast_type: *30
-    sql_type_metadata: *31
-    'null': false
-    default:
-    default_function:
-    collation:
-    comment:
-  - &34 !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: rowid
-    cast_type: *32
-    sql_type_metadata: &35 !ruby/object:ActiveRecord::ConnectionAdapters::SqlTypeMetadata
-      sql_type: ''
-      type:
-      limit:
-      precision:
-      scale:
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  search_embeddings_metadatachunks01:
-  - *33
-  - *34
-  search_embeddings_metadatatext00:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: data
-    cast_type: *15
-    sql_type_metadata: *16
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  - *34
-  search_embeddings_rowids:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: chunk_id
-    cast_type: *1
-    sql_type_metadata: *2
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: chunk_offset
-    cast_type: *1
-    sql_type_metadata: *2
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: id
-    cast_type: *32
-    sql_type_metadata: *35
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: rowid
-    cast_type: *1
-    sql_type_metadata: *2
-    'null': true
-    default:
-    default_function:
-    collation:
-    comment:
-  search_embeddings_vector_chunks00:
-  - *34
-  - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
-    auto_increment:
-    name: vectors
-    cast_type: *30
-    sql_type_metadata: *31
-    'null': false
-    default:
-    default_function:
-    collation:
-    comment:
   search_queries:
   - *5
   - *6
@@ -1106,19 +934,19 @@ columns:
   - *22
   - *5
   - *6
-  - *36
+  - *30
   - *9
   tags:
   - *5
   - *6
-  - *37
+  - *31
   - *9
   users:
   - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
     auto_increment:
     name: active
-    cast_type: *38
-    sql_type_metadata: *39
+    cast_type: *32
+    sql_type_metadata: *33
     'null': false
     default: true
     default_function:
@@ -1177,8 +1005,8 @@ columns:
   - !ruby/object:ActiveRecord::ConnectionAdapters::SQLite3::Column
     auto_increment:
     name: watching
-    cast_type: *38
-    sql_type_metadata: *39
+    cast_type: *32
+    sql_type_metadata: *33
     'null': false
     default: true
     default_function:
@@ -1247,13 +1075,6 @@ primary_keys:
   pins: id
   reactions: id
   schema_migrations: version
-  search_embeddings_chunks: chunk_id
-  search_embeddings_info: key
-  search_embeddings_metadatachunks00: rowid
-  search_embeddings_metadatachunks01: rowid
-  search_embeddings_metadatatext00: rowid
-  search_embeddings_rowids: rowid
-  search_embeddings_vector_chunks00: rowid
   search_queries: id
   search_results: id
   sessions: id
@@ -1296,13 +1117,6 @@ data_sources:
   pins: true
   reactions: true
   schema_migrations: true
-  search_embeddings_chunks: true
-  search_embeddings_info: true
-  search_embeddings_metadatachunks00: true
-  search_embeddings_metadatachunks01: true
-  search_embeddings_metadatatext00: true
-  search_embeddings_rowids: true
-  search_embeddings_vector_chunks00: true
   search_queries: true
   search_results: true
   sessions: true
@@ -2364,13 +2178,6 @@ indexes:
     comment:
     valid: true
   schema_migrations: []
-  search_embeddings_chunks: []
-  search_embeddings_info: []
-  search_embeddings_metadatachunks00: []
-  search_embeddings_metadatachunks01: []
-  search_embeddings_metadatatext00: []
-  search_embeddings_rowids: []
-  search_embeddings_vector_chunks00: []
   search_queries:
   - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition
     table: search_queries
@@ -2457,23 +2264,7 @@ indexes:
     nulls_not_distinct:
     comment:
     valid: true
-  tags:
-  - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition
-    table: tags
-    name: index_tags_on_account_id_and_title
-    unique: true
-    columns:
-    - title
-    lengths: {}
-    orders: {}
-    opclasses: {}
-    where:
-    type:
-    using:
-    include:
-    nulls_not_distinct:
-    comment:
-    valid: true
+  tags: []
   users:
   - !ruby/object:ActiveRecord::ConnectionAdapters::IndexDefinition
     table: users
@@ -2574,4 +2365,4 @@ indexes:
     comment:
     valid: true
   workflows: []
-version: 20250625073215
+version: 20250625111932