Speedy, auditable, deadlock-resistant storage tracking (#2026)
An append-only storage ledger replaces deadlock-prone synchronous counter updates and drift-prone async updates. All content storage (card images, card/comment/board description embeds) is tracked. Account exports (which expire) and avatars are not tracked. Storage is accounted for by Account and Board. Both consume the same event stream independently: no bubble-up storage bumps triggering deadlocks due to lock sequencing. Each calculates its own total from the underlying ledger with independent cursors and materialization. * Storage::Entry: Append-only ledger recording attach/detach/transfer events with delta bytes. Single event stream indexed for both Account and Board cursor queries. * Storage::Total: Polymorphic snapshot cache with cursor (last_entry_id) tracking which entries have been materialized. * Storage::Totaled concern: Provides bytes_used (fast snapshot) and bytes_used_exact (snapshot + pending) query modes, plus materialize_storage! to roll up pending entries. * Storage::Tracked concern: For models owning attachments (Card, Comment, Board). Provides board_for_storage_tracking for models where board is determined differently (Board returns self). Handles board transfers by recording transfer_out/transfer_in entries. * Storage::AttachmentTracking: Hooks ActiveStorage::Attachment lifecycle to record attach/detach entries. Handles ActionText::RichText embeds by traversing to the actual model. Snapshots context in before_destroy to handle cascading deletes where parent record may be gone by after_destroy_commit. * MaterializeJob: Rolls up pending entries into snapshot. Concurrency limited per owner to prevent duplicate work. * ReconcileJob: On-demand reconciliation against actual attachment storage for support/debugging. Compares ledger total to real bytes from card images, card embeds, comment embeds, and board embeds. Usage: * account.bytes_used / board.bytes_used: fast, slightly stale bytesize * account.bytes_used_exact / board.bytes_used_exact: real-time bytesize * Storage::Entry: audit trail for debugging and point-in-time queries
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
class CreateStorageTables < ActiveRecord::Migration[8.0]
|
||||
def change
|
||||
# Storage ledger: debit/credit event stream
|
||||
create_table :storage_entries, id: :uuid do |t|
|
||||
t.references :account, type: :uuid, null: false
|
||||
t.references :board, type: :uuid, null: true
|
||||
|
||||
t.references :recordable, type: :uuid, polymorphic: true, null: true
|
||||
|
||||
t.bigint :delta, null: false
|
||||
t.string :operation, null: false
|
||||
|
||||
t.datetime :created_at, null: false
|
||||
end
|
||||
|
||||
# Storage totals: cached snapshots
|
||||
create_table :storage_totals, id: :uuid do |t|
|
||||
t.references :owner, type: :uuid, polymorphic: true, null: false, index: false
|
||||
|
||||
t.bigint :bytes_stored, null: false, default: 0
|
||||
t.uuid :last_entry_id # Cursor: includes all entries <= this ID
|
||||
|
||||
t.timestamps
|
||||
t.index %i[ owner_type owner_id ], unique: true
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,9 @@
|
||||
class AddBlobIdAndAuditContextToStorageEntries < ActiveRecord::Migration[8.2]
|
||||
def change
|
||||
change_table :storage_entries do |t|
|
||||
t.references :blob, type: :uuid, foreign_key: false, index: true
|
||||
t.references :user, type: :uuid, foreign_key: false, index: true
|
||||
t.string :request_id, index: true
|
||||
end
|
||||
end
|
||||
end
|
||||
Generated
+30
-1
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do
|
||||
ActiveRecord::Schema[8.2].define(version: 2025_12_10_054934) do
|
||||
create_table "accesses", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||
t.datetime "accessed_at"
|
||||
t.uuid "account_id", null: false
|
||||
@@ -697,6 +697,35 @@ ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do
|
||||
t.index ["card_id"], name: "index_steps_on_card_id"
|
||||
end
|
||||
|
||||
create_table "storage_entries", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||
t.uuid "account_id", null: false
|
||||
t.uuid "blob_id"
|
||||
t.uuid "board_id"
|
||||
t.datetime "created_at", null: false
|
||||
t.bigint "delta", null: false
|
||||
t.string "operation", null: false
|
||||
t.uuid "recordable_id"
|
||||
t.string "recordable_type"
|
||||
t.string "request_id"
|
||||
t.uuid "user_id"
|
||||
t.index ["account_id"], name: "index_storage_entries_on_account_id"
|
||||
t.index ["blob_id"], name: "index_storage_entries_on_blob_id"
|
||||
t.index ["board_id"], name: "index_storage_entries_on_board_id"
|
||||
t.index ["recordable_type", "recordable_id"], name: "index_storage_entries_on_recordable"
|
||||
t.index ["request_id"], name: "index_storage_entries_on_request_id"
|
||||
t.index ["user_id"], name: "index_storage_entries_on_user_id"
|
||||
end
|
||||
|
||||
create_table "storage_totals", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||
t.bigint "bytes_stored", default: 0, null: false
|
||||
t.datetime "created_at", null: false
|
||||
t.uuid "last_entry_id"
|
||||
t.uuid "owner_id", null: false
|
||||
t.string "owner_type", null: false
|
||||
t.datetime "updated_at", null: false
|
||||
t.index ["owner_type", "owner_id"], name: "index_storage_totals_on_owner_type_and_owner_id", unique: true
|
||||
end
|
||||
|
||||
create_table "taggings", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||
t.uuid "account_id", null: false
|
||||
t.uuid "card_id", null: false
|
||||
|
||||
+30
-1
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do
|
||||
ActiveRecord::Schema[8.2].define(version: 2025_12_10_054934) do
|
||||
create_table "accesses", id: :uuid, force: :cascade do |t|
|
||||
t.datetime "accessed_at"
|
||||
t.uuid "account_id", null: false
|
||||
@@ -470,6 +470,35 @@ ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do
|
||||
t.index ["card_id"], name: "index_steps_on_card_id"
|
||||
end
|
||||
|
||||
create_table "storage_entries", id: :uuid, force: :cascade do |t|
|
||||
t.uuid "account_id", null: false
|
||||
t.uuid "blob_id"
|
||||
t.uuid "board_id"
|
||||
t.datetime "created_at", null: false
|
||||
t.bigint "delta", null: false
|
||||
t.string "operation", limit: 255, null: false
|
||||
t.uuid "recordable_id"
|
||||
t.string "recordable_type", limit: 255
|
||||
t.string "request_id"
|
||||
t.uuid "user_id"
|
||||
t.index ["account_id"], name: "index_storage_entries_on_account_id"
|
||||
t.index ["blob_id"], name: "index_storage_entries_on_blob_id"
|
||||
t.index ["board_id"], name: "index_storage_entries_on_board_id"
|
||||
t.index ["recordable_type", "recordable_id"], name: "index_storage_entries_on_recordable"
|
||||
t.index ["request_id"], name: "index_storage_entries_on_request_id"
|
||||
t.index ["user_id"], name: "index_storage_entries_on_user_id"
|
||||
end
|
||||
|
||||
create_table "storage_totals", id: :uuid, force: :cascade do |t|
|
||||
t.bigint "bytes_stored", default: 0, null: false
|
||||
t.datetime "created_at", null: false
|
||||
t.uuid "last_entry_id"
|
||||
t.uuid "owner_id", null: false
|
||||
t.string "owner_type", limit: 255, null: false
|
||||
t.datetime "updated_at", null: false
|
||||
t.index ["owner_type", "owner_id"], name: "index_storage_totals_on_owner_type_and_owner_id", unique: true
|
||||
end
|
||||
|
||||
create_table "taggings", id: :uuid, force: :cascade do |t|
|
||||
t.uuid "account_id", null: false
|
||||
t.uuid "card_id", null: false
|
||||
|
||||
Reference in New Issue
Block a user