From 761d0b4a769ade6ae0e9f7532617e2adb66b535b Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Wed, 10 Dec 2025 16:04:30 -0800 Subject: [PATCH] Speedy, auditable, deadlock-resistant storage tracking (#2026) An append-only storage ledger replaces deadlock-prone synchronous counter updates and drift-prone async updates. All content storage (card images, card/comment/board description embeds) is tracked. Account exports (which expire) and avatars are not tracked. Storage is accounted for by Account and Board. Both consume the same event stream independently: no bubble-up storage bumps triggering deadlocks due to lock sequencing. Each calculates its own total from the underlying ledger with independent cursors and materialization. * Storage::Entry: Append-only ledger recording attach/detach/transfer events with delta bytes. Single event stream indexed for both Account and Board cursor queries. * Storage::Total: Polymorphic snapshot cache with cursor (last_entry_id) tracking which entries have been materialized. * Storage::Totaled concern: Provides bytes_used (fast snapshot) and bytes_used_exact (snapshot + pending) query modes, plus materialize_storage! to roll up pending entries. * Storage::Tracked concern: For models owning attachments (Card, Comment, Board). Provides board_for_storage_tracking for models where board is determined differently (Board returns self). Handles board transfers by recording transfer_out/transfer_in entries. * Storage::AttachmentTracking: Hooks ActiveStorage::Attachment lifecycle to record attach/detach entries. Handles ActionText::RichText embeds by traversing to the actual model. Snapshots context in before_destroy to handle cascading deletes where parent record may be gone by after_destroy_commit. * MaterializeJob: Rolls up pending entries into snapshot. Concurrency limited per owner to prevent duplicate work. * ReconcileJob: On-demand reconciliation against actual attachment storage for support/debugging. Compares ledger total to real bytes from card images, card embeds, comment embeds, and board embeds. Usage: * account.bytes_used / board.bytes_used: fast, slightly stale bytesize * account.bytes_used_exact / board.bytes_used_exact: real-time bytesize * Storage::Entry: audit trail for debugging and point-in-time queries --- app/jobs/storage/materialize_job.rb | 8 + app/jobs/storage/reconcile_job.rb | 7 + app/models/account.rb | 2 +- app/models/account/storage.rb | 9 + app/models/board.rb | 2 +- app/models/board/storage.rb | 56 ++++ app/models/card.rb | 2 +- app/models/comment.rb | 2 +- app/models/concerns/storage/totaled.rb | 70 +++++ app/models/concerns/storage/tracked.rb | 57 ++++ app/models/storage.rb | 5 + app/models/storage/attachment_tracking.rb | 53 ++++ app/models/storage/entry.rb | 35 +++ app/models/storage/total.rb | 12 + config/initializers/action_text.rb | 5 + config/initializers/active_storage.rb | 4 + .../20251205205826_create_storage_tables.rb | 27 ++ ...id_and_audit_context_to_storage_entries.rb | 9 + db/schema.rb | 31 +- db/schema_sqlite.rb | 31 +- .../active_record_replica_support.rb | 12 + script/migrations/backfill-storage-ledger.rb | 73 +++++ test/jobs/storage/materialize_job_test.rb | 58 ++++ test/jobs/storage/reconcile_job_test.rb | 48 +++ .../storage/attachment_tracking_test.rb | 263 +++++++++++++++++ test/models/storage/entry_test.rb | 168 +++++++++++ test/models/storage/total_test.rb | 83 ++++++ test/models/storage/totaled_test.rb | 276 ++++++++++++++++++ test/models/storage/tracked_test.rb | 119 ++++++++ 29 files changed, 1521 insertions(+), 6 deletions(-) create mode 100644 app/jobs/storage/materialize_job.rb create mode 100644 app/jobs/storage/reconcile_job.rb create mode 100644 app/models/account/storage.rb create mode 100644 app/models/board/storage.rb create mode 100644 app/models/concerns/storage/totaled.rb create mode 100644 app/models/concerns/storage/tracked.rb create mode 100644 app/models/storage.rb create mode 100644 app/models/storage/attachment_tracking.rb create mode 100644 app/models/storage/entry.rb create mode 100644 app/models/storage/total.rb create mode 100644 db/migrate/20251205205826_create_storage_tables.rb create mode 100644 db/migrate/20251210054934_add_blob_id_and_audit_context_to_storage_entries.rb create mode 100644 script/migrations/backfill-storage-ledger.rb create mode 100644 test/jobs/storage/materialize_job_test.rb create mode 100644 test/jobs/storage/reconcile_job_test.rb create mode 100644 test/models/storage/attachment_tracking_test.rb create mode 100644 test/models/storage/entry_test.rb create mode 100644 test/models/storage/total_test.rb create mode 100644 test/models/storage/totaled_test.rb create mode 100644 test/models/storage/tracked_test.rb diff --git a/app/jobs/storage/materialize_job.rb b/app/jobs/storage/materialize_job.rb new file mode 100644 index 000000000..b2e494977 --- /dev/null +++ b/app/jobs/storage/materialize_job.rb @@ -0,0 +1,8 @@ +class Storage::MaterializeJob < ApplicationJob + queue_as :backend + limits_concurrency to: 1, key: ->(owner) { owner } + + def perform(owner) + owner.materialize_storage + end +end diff --git a/app/jobs/storage/reconcile_job.rb b/app/jobs/storage/reconcile_job.rb new file mode 100644 index 000000000..dde6dcb2e --- /dev/null +++ b/app/jobs/storage/reconcile_job.rb @@ -0,0 +1,7 @@ +class Storage::ReconcileJob < ApplicationJob + queue_as :backend + + def perform(owner) + owner.reconcile_storage + end +end diff --git a/app/models/account.rb b/app/models/account.rb index 0bb4d4111..34aaf64ae 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -1,5 +1,5 @@ class Account < ApplicationRecord - include Entropic, Seedeable + include Account::Storage, Entropic, Seedeable has_one :join_code has_many :users, dependent: :destroy diff --git a/app/models/account/storage.rb b/app/models/account/storage.rb new file mode 100644 index 000000000..8b432e8ce --- /dev/null +++ b/app/models/account/storage.rb @@ -0,0 +1,9 @@ +module Account::Storage + extend ActiveSupport::Concern + include Storage::Totaled + + private + def calculate_real_storage_bytes + boards.sum { |board| board.send(:calculate_real_storage_bytes) } + end +end diff --git a/app/models/board.rb b/app/models/board.rb index 9f1bcd5d7..ada49f1f2 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -1,5 +1,5 @@ class Board < ApplicationRecord - include Accessible, AutoPostponing, Broadcastable, Cards, Entropic, Filterable, Publishable, Triageable + include Accessible, AutoPostponing, Board::Storage, Broadcastable, Cards, Entropic, Filterable, Publishable, ::Storage::Tracked, Triageable belongs_to :creator, class_name: "User", default: -> { Current.user } belongs_to :account, default: -> { creator.account } diff --git a/app/models/board/storage.rb b/app/models/board/storage.rb new file mode 100644 index 000000000..64dde9985 --- /dev/null +++ b/app/models/board/storage.rb @@ -0,0 +1,56 @@ +module Board::Storage + extend ActiveSupport::Concern + include Storage::Totaled + + # Board's own embeds (public_description) count toward itself + def board_for_storage_tracking + self + end + + private + BATCH_SIZE = 1000 + + # Calculate actual storage by summing blob sizes. + # + # Uses batched pluck queries to avoid loading huge ID arrays, and avoids + # ActiveRecord model queries on ActiveStorage tables to sidestep cross-pool + # issues when ActiveStorage uses separate connection pools (e.g., with replicas). + def calculate_real_storage_bytes + card_image_bytes + card_embed_bytes + comment_embed_bytes + board_embed_bytes + end + + def card_image_bytes + sum_blob_bytes_in_batches \ + ActiveStorage::Attachment.where(record_type: "Card", name: "image"), + cards.pluck(:id) + end + + def card_embed_bytes + sum_embed_bytes_for "Card", cards.pluck(:id) + end + + def comment_embed_bytes + sum_embed_bytes_for "Comment", Comment.where(card_id: cards.pluck(:id)).pluck(:id) + end + + def board_embed_bytes + sum_embed_bytes_for "Board", [ id ] + end + + def sum_embed_bytes_for(record_type, record_ids) + rich_text_ids = ActionText::RichText \ + .where(record_type: record_type, record_id: record_ids) + .pluck(:id) + + sum_blob_bytes_in_batches \ + ActiveStorage::Attachment.where(record_type: "ActionText::RichText", name: "embeds"), + rich_text_ids + end + + def sum_blob_bytes_in_batches(base_scope, record_ids) + record_ids.each_slice(BATCH_SIZE).sum do |batch_ids| + blob_ids = base_scope.where(record_id: batch_ids).pluck(:blob_id) + ActiveStorage::Blob.where(id: blob_ids).sum(:byte_size) + end + end +end diff --git a/app/models/card.rb b/app/models/card.rb index dfad00cb8..33c96e502 100644 --- a/app/models/card.rb +++ b/app/models/card.rb @@ -1,7 +1,7 @@ class Card < ApplicationRecord include Assignable, Attachments, Broadcastable, Closeable, Colored, Entropic, Eventable, Exportable, Golden, Mentions, Multistep, Pinnable, Postponable, Promptable, - Readable, Searchable, Stallable, Statuses, Taggable, Triageable, Watchable + Readable, Searchable, Stallable, Statuses, Storage::Tracked, Taggable, Triageable, Watchable belongs_to :account, default: -> { board.account } belongs_to :board diff --git a/app/models/comment.rb b/app/models/comment.rb index f81c465ad..a396a904f 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -1,5 +1,5 @@ class Comment < ApplicationRecord - include Attachments, Eventable, Mentions, Promptable, Searchable + include Attachments, Eventable, Mentions, Promptable, Searchable, Storage::Tracked belongs_to :account, default: -> { card.account } belongs_to :card, touch: true diff --git a/app/models/concerns/storage/totaled.rb b/app/models/concerns/storage/totaled.rb new file mode 100644 index 000000000..3f220f55f --- /dev/null +++ b/app/models/concerns/storage/totaled.rb @@ -0,0 +1,70 @@ +module Storage::Totaled + extend ActiveSupport::Concern + + included do + has_one :storage_total, as: :owner, class_name: "Storage::Total", dependent: :destroy + has_many :storage_entries, class_name: "Storage::Entry", foreign_key: foreign_key_for_storage + end + + class_methods do + def foreign_key_for_storage + "#{model_name.singular}_id" + end + end + + # Fast: materialized snapshot (may be slightly stale) + def bytes_used + storage_total&.bytes_stored || 0 + end + + # Exact: snapshot + pending entries + def bytes_used_exact + (storage_total || create_storage_total!).current_usage + end + + def materialize_storage_later + Storage::MaterializeJob.perform_later(self) + end + + # Materialize all pending entries into snapshot + def materialize_storage + total = storage_total || create_storage_total! + + total.with_lock do + latest_entry_id = storage_entries.maximum(:id) + + if latest_entry_id && total.last_entry_id != latest_entry_id + scope = storage_entries.where(id: ..latest_entry_id) + scope = scope.where.not(id: ..total.last_entry_id) if total.last_entry_id + delta_sum = scope.sum(:delta) + + total.update! bytes_stored: total.bytes_stored + delta_sum, last_entry_id: latest_entry_id + end + end + end + + # Reconcile ledger against actual attachment storage. + # Uses cursor to ensure consistency: captures max entry ID first, then calculates + # real bytes, then sums only entries up to that cursor. Concurrent uploads during + # calculation will have entries with IDs beyond the cursor, avoiding double-count. + def reconcile_storage + max_entry_id = storage_entries.maximum(:id) + real_bytes = calculate_real_storage_bytes + ledger_bytes = max_entry_id ? storage_entries.where(id: ..max_entry_id).sum(:delta) : 0 + diff = real_bytes - ledger_bytes + + if diff.nonzero? + Storage::Entry.record \ + account: is_a?(Account) ? self : account, + board: is_a?(Board) ? self : nil, + recordable: nil, + delta: diff, + operation: "reconcile" + end + end + + private + def calculate_real_storage_bytes + raise NotImplementedError, "Subclass must implement calculate_real_storage_bytes" + end +end diff --git a/app/models/concerns/storage/tracked.rb b/app/models/concerns/storage/tracked.rb new file mode 100644 index 000000000..7b96e95c0 --- /dev/null +++ b/app/models/concerns/storage/tracked.rb @@ -0,0 +1,57 @@ +module Storage::Tracked + extend ActiveSupport::Concern + + included do + before_update :track_board_transfer, if: :board_transfer? + end + + # Return self as the trackable record for storage entries + def storage_tracked_record + self + end + + # Override in models where board is determined differently (e.g., Board itself) + def board_for_storage_tracking + board + end + + # Total bytes for all attachments on this record + def storage_bytes + attachments_for_storage.sum { |a| a.blob.byte_size } + end + + private + def board_transfer? + respond_to?(:board_id_changed?) && board_id_changed? + end + + def track_board_transfer + old_board_id = board_id_was + current_bytes = storage_bytes + + if current_bytes.positive? + # Debit old board + if old_board_id + Storage::Entry.record \ + account: account, + board_id: old_board_id, + recordable: self, + delta: -current_bytes, + operation: "transfer_out" + end + + # Credit new board + Storage::Entry.record \ + account: account, + board: board, + recordable: self, + delta: current_bytes, + operation: "transfer_in" + end + end + + # Override if needed. Default = all direct attachments + def attachments_for_storage + ActiveStorage::Attachment.where(record: self) + end +end diff --git a/app/models/storage.rb b/app/models/storage.rb new file mode 100644 index 000000000..9c17b7cd9 --- /dev/null +++ b/app/models/storage.rb @@ -0,0 +1,5 @@ +module Storage + def self.table_name_prefix + "storage_" + end +end diff --git a/app/models/storage/attachment_tracking.rb b/app/models/storage/attachment_tracking.rb new file mode 100644 index 000000000..d68f5845d --- /dev/null +++ b/app/models/storage/attachment_tracking.rb @@ -0,0 +1,53 @@ +module Storage::AttachmentTracking + extend ActiveSupport::Concern + + included do + # Snapshot IDs in before_destroy since parent record may be deleted + # by the time after_destroy_commit runs + before_destroy :snapshot_storage_context + after_create_commit :record_storage_attach + after_destroy_commit :record_storage_detach + end + + private + def record_storage_attach + return unless storage_tracked_record + + Storage::Entry.record \ + account: storage_tracked_record.account, + board: storage_tracked_record.board_for_storage_tracking, + recordable: storage_tracked_record, + blob: blob, + delta: blob.byte_size, + operation: "attach" + end + + def record_storage_detach + return unless @storage_snapshot + + Storage::Entry.record \ + account_id: @storage_snapshot[:account_id], + board_id: @storage_snapshot[:board_id], + recordable_type: @storage_snapshot[:recordable_type], + recordable_id: @storage_snapshot[:recordable_id], + blob_id: @storage_snapshot[:blob_id], + delta: -blob.byte_size, + operation: "detach" + end + + def snapshot_storage_context + return unless storage_tracked_record + + @storage_snapshot = { + account_id: storage_tracked_record.account.id, + board_id: storage_tracked_record.board_for_storage_tracking&.id, + recordable_type: storage_tracked_record.class.name, + recordable_id: storage_tracked_record.id, + blob_id: blob.id + } + end + + def storage_tracked_record + record.try(:storage_tracked_record) + end +end diff --git a/app/models/storage/entry.rb b/app/models/storage/entry.rb new file mode 100644 index 000000000..a2f6e726b --- /dev/null +++ b/app/models/storage/entry.rb @@ -0,0 +1,35 @@ +class Storage::Entry < ApplicationRecord + belongs_to :account + belongs_to :board, optional: true + belongs_to :recordable, polymorphic: true, optional: true + + scope :pending, ->(last_entry_id) { where.not(id: ..last_entry_id) if last_entry_id } + + # Accepts either objects or _id params (for after_destroy_commit snapshots) + def self.record(delta:, operation:, account: nil, account_id: nil, board: nil, board_id: nil, + recordable: nil, recordable_type: nil, recordable_id: nil, blob: nil, blob_id: nil) + return if delta.zero? + + account_id ||= account&.id + board_id ||= board&.id + blob_id ||= blob&.id + + entry = create! \ + account_id: account_id, + board_id: board_id, + recordable_type: recordable_type || recordable&.class&.name, + recordable_id: recordable_id || recordable&.id, + blob_id: blob_id, + delta: delta, + operation: operation, + user_id: Current.user&.id, + request_id: Current.request_id + + # Enqueue materialization - use find_by to handle cascading deletes + # (Account/Board may be destroyed while attachments are still being cleaned up) + Account.find_by(id: account_id)&.materialize_storage_later + Board.find_by(id: board_id)&.materialize_storage_later if board_id + + entry + end +end diff --git a/app/models/storage/total.rb b/app/models/storage/total.rb new file mode 100644 index 000000000..7e1651d79 --- /dev/null +++ b/app/models/storage/total.rb @@ -0,0 +1,12 @@ +class Storage::Total < ApplicationRecord + belongs_to :owner, polymorphic: true + + def pending_entries + owner.storage_entries.pending(last_entry_id) + end + + # Exact current usage (snapshot + pending) + def current_usage + bytes_stored + pending_entries.sum(:delta) + end +end diff --git a/config/initializers/action_text.rb b/config/initializers/action_text.rb index eee87de0d..0f804e57b 100644 --- a/config/initializers/action_text.rb +++ b/config/initializers/action_text.rb @@ -11,6 +11,11 @@ module ActionText end end end + + # Delegate storage tracking to the parent record (Card, Comment, Board, etc.) + def storage_tracked_record + record.try(:storage_tracked_record) + end end end end diff --git a/config/initializers/active_storage.rb b/config/initializers/active_storage.rb index 74461c795..e647a72cf 100644 --- a/config/initializers/active_storage.rb +++ b/config/initializers/active_storage.rb @@ -1,3 +1,7 @@ +ActiveSupport.on_load(:active_storage_attachment) do + include Storage::AttachmentTracking +end + ActiveSupport.on_load(:active_storage_blob) do ActiveStorage::DiskController.after_action only: :show do expires_in 5.minutes, public: true diff --git a/db/migrate/20251205205826_create_storage_tables.rb b/db/migrate/20251205205826_create_storage_tables.rb new file mode 100644 index 000000000..878b6dd72 --- /dev/null +++ b/db/migrate/20251205205826_create_storage_tables.rb @@ -0,0 +1,27 @@ +class CreateStorageTables < ActiveRecord::Migration[8.0] + def change + # Storage ledger: debit/credit event stream + create_table :storage_entries, id: :uuid do |t| + t.references :account, type: :uuid, null: false + t.references :board, type: :uuid, null: true + + t.references :recordable, type: :uuid, polymorphic: true, null: true + + t.bigint :delta, null: false + t.string :operation, null: false + + t.datetime :created_at, null: false + end + + # Storage totals: cached snapshots + create_table :storage_totals, id: :uuid do |t| + t.references :owner, type: :uuid, polymorphic: true, null: false, index: false + + t.bigint :bytes_stored, null: false, default: 0 + t.uuid :last_entry_id # Cursor: includes all entries <= this ID + + t.timestamps + t.index %i[ owner_type owner_id ], unique: true + end + end +end diff --git a/db/migrate/20251210054934_add_blob_id_and_audit_context_to_storage_entries.rb b/db/migrate/20251210054934_add_blob_id_and_audit_context_to_storage_entries.rb new file mode 100644 index 000000000..42fa66a77 --- /dev/null +++ b/db/migrate/20251210054934_add_blob_id_and_audit_context_to_storage_entries.rb @@ -0,0 +1,9 @@ +class AddBlobIdAndAuditContextToStorageEntries < ActiveRecord::Migration[8.2] + def change + change_table :storage_entries do |t| + t.references :blob, type: :uuid, foreign_key: false, index: true + t.references :user, type: :uuid, foreign_key: false, index: true + t.string :request_id, index: true + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 85a8f8064..645cb7a96 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do +ActiveRecord::Schema[8.2].define(version: 2025_12_10_054934) do create_table "accesses", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false @@ -697,6 +697,35 @@ ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do t.index ["card_id"], name: "index_steps_on_card_id" end + create_table "storage_entries", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "blob_id" + t.uuid "board_id" + t.datetime "created_at", null: false + t.bigint "delta", null: false + t.string "operation", null: false + t.uuid "recordable_id" + t.string "recordable_type" + t.string "request_id" + t.uuid "user_id" + t.index ["account_id"], name: "index_storage_entries_on_account_id" + t.index ["blob_id"], name: "index_storage_entries_on_blob_id" + t.index ["board_id"], name: "index_storage_entries_on_board_id" + t.index ["recordable_type", "recordable_id"], name: "index_storage_entries_on_recordable" + t.index ["request_id"], name: "index_storage_entries_on_request_id" + t.index ["user_id"], name: "index_storage_entries_on_user_id" + end + + create_table "storage_totals", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.bigint "bytes_stored", default: 0, null: false + t.datetime "created_at", null: false + t.uuid "last_entry_id" + t.uuid "owner_id", null: false + t.string "owner_type", null: false + t.datetime "updated_at", null: false + t.index ["owner_type", "owner_id"], name: "index_storage_totals_on_owner_type_and_owner_id", unique: true + end + create_table "taggings", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.uuid "account_id", null: false t.uuid "card_id", null: false diff --git a/db/schema_sqlite.rb b/db/schema_sqlite.rb index e2f65dccc..84fd827bb 100644 --- a/db/schema_sqlite.rb +++ b/db/schema_sqlite.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do +ActiveRecord::Schema[8.2].define(version: 2025_12_10_054934) do create_table "accesses", id: :uuid, force: :cascade do |t| t.datetime "accessed_at" t.uuid "account_id", null: false @@ -470,6 +470,35 @@ ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do t.index ["card_id"], name: "index_steps_on_card_id" end + create_table "storage_entries", id: :uuid, force: :cascade do |t| + t.uuid "account_id", null: false + t.uuid "blob_id" + t.uuid "board_id" + t.datetime "created_at", null: false + t.bigint "delta", null: false + t.string "operation", limit: 255, null: false + t.uuid "recordable_id" + t.string "recordable_type", limit: 255 + t.string "request_id" + t.uuid "user_id" + t.index ["account_id"], name: "index_storage_entries_on_account_id" + t.index ["blob_id"], name: "index_storage_entries_on_blob_id" + t.index ["board_id"], name: "index_storage_entries_on_board_id" + t.index ["recordable_type", "recordable_id"], name: "index_storage_entries_on_recordable" + t.index ["request_id"], name: "index_storage_entries_on_request_id" + t.index ["user_id"], name: "index_storage_entries_on_user_id" + end + + create_table "storage_totals", id: :uuid, force: :cascade do |t| + t.bigint "bytes_stored", default: 0, null: false + t.datetime "created_at", null: false + t.uuid "last_entry_id" + t.uuid "owner_id", null: false + t.string "owner_type", limit: 255, null: false + t.datetime "updated_at", null: false + t.index ["owner_type", "owner_id"], name: "index_storage_totals_on_owner_type_and_owner_id", unique: true + end + create_table "taggings", id: :uuid, force: :cascade do |t| t.uuid "account_id", null: false t.uuid "card_id", null: false diff --git a/lib/rails_ext/active_record_replica_support.rb b/lib/rails_ext/active_record_replica_support.rb index 2625a1ac9..8b9e63a98 100644 --- a/lib/rails_ext/active_record_replica_support.rb +++ b/lib/rails_ext/active_record_replica_support.rb @@ -34,6 +34,18 @@ module ActiveRecordReplicaSupport def replica_configured? configurations.find_db_config("replica").present? end + + # Execute block using read replica if available, otherwise use primary. + # + # Example: + # ApplicationRecord.with_reading_role { User.count } + def with_reading_role(&block) + if replica_configured? + connected_to(role: :reading, &block) + else + yield + end + end end end diff --git a/script/migrations/backfill-storage-ledger.rb b/script/migrations/backfill-storage-ledger.rb new file mode 100644 index 000000000..9ad0b20b4 --- /dev/null +++ b/script/migrations/backfill-storage-ledger.rb @@ -0,0 +1,73 @@ +#!/usr/bin/env ruby + +# Backfill storage ledger with attach entries for all existing attachments. +# +# Run locally: +# bin/rails runner script/migrations/backfill-storage-ledger.rb +# +# Run via Kamal: +# kamal app exec -d -p --reuse "bin/rails runner script/migrations/backfill-storage-ledger.rb" +# +# Safe to re-run: skips attachments that already have entries (by blob_id). +class BackfillStorageLedger + def run + puts "Backfilling storage entries…" + backfill_entries + + puts "\nMaterializing totals…" + materialize_totals + end + + private + def backfill_entries + created = 0 + skipped = 0 + + ActiveStorage::Attachment.includes(:blob).find_each do |attachment| + record = attachment.record.try(:storage_tracked_record) + + if record.nil? || Storage::Entry.exists?(blob_id: attachment.blob_id) + skipped += 1 + next + end + + Storage::Entry.create! \ + account_id: record.account.id, + board_id: record.board_for_storage_tracking&.id, + recordable_type: record.class.name, + recordable_id: record.id, + blob_id: attachment.blob_id, + delta: attachment.blob.byte_size, + operation: "attach" + created += 1 + + print "." if created % 100 == 0 + end + + puts "\n\nBackfill complete!" + puts " Entries created: #{created}" + puts " Attachments skipped: #{skipped}" + end + + def materialize_totals + boards_materialized = 0 + accounts_materialized = 0 + + Board.find_each do |board| + board.materialize_storage + boards_materialized += 1 + print "." if boards_materialized % 100 == 0 + end + + Account.find_each do |account| + account.materialize_storage + accounts_materialized += 1 + end + + puts "\n\nMaterialization complete!" + puts " Boards: #{boards_materialized}" + puts " Accounts: #{accounts_materialized}" + end +end + +BackfillStorageLedger.new.run diff --git a/test/jobs/storage/materialize_job_test.rb b/test/jobs/storage/materialize_job_test.rb new file mode 100644 index 000000000..5a54dfc62 --- /dev/null +++ b/test/jobs/storage/materialize_job_test.rb @@ -0,0 +1,58 @@ +require "test_helper" + +class Storage::MaterializeJobTest < ActiveJob::TestCase + setup do + @account = accounts("37s") + @board = boards(:writebook) + end + + test "calls materialize_storage on account" do + Storage::Entry.record(account: @account, delta: 1024, operation: "attach") + + Storage::MaterializeJob.perform_now(@account) + + assert_not_nil @account.storage_total + assert_equal 1024, @account.bytes_used + end + + test "calls materialize_storage on board" do + Storage::Entry.record(account: @account, board: @board, delta: 2048, operation: "attach") + + Storage::MaterializeJob.perform_now(@board) + + assert_not_nil @board.storage_total + assert_equal 2048, @board.bytes_used + end + + test "job is idempotent" do + Storage::Entry.record(account: @account, delta: 1024, operation: "attach") + + 3.times { Storage::MaterializeJob.perform_now(@account) } + + assert_equal 1024, @account.bytes_used + end + + test "job processes entries added between runs" do + Storage::Entry.record(account: @account, delta: 1000, operation: "attach") + Storage::MaterializeJob.perform_now(@account) + + # Small delay to ensure UUIDv7 timestamp advances + travel 1.second + + Storage::Entry.record(account: @account, delta: 500, operation: "attach") + Storage::MaterializeJob.perform_now(@account) + + assert_equal 1500, @account.bytes_used + end + + test "job queued to backend queue" do + assert_equal "backend", Storage::MaterializeJob.new.queue_name + end + + test "job has concurrency limit by owner" do + job = Storage::MaterializeJob.new(@account) + # limits_concurrency is a Solid Queue feature + # Just verify the job can be instantiated and has the correct queue + assert_equal "backend", job.queue_name + end +end diff --git a/test/jobs/storage/reconcile_job_test.rb b/test/jobs/storage/reconcile_job_test.rb new file mode 100644 index 000000000..038dfd89f --- /dev/null +++ b/test/jobs/storage/reconcile_job_test.rb @@ -0,0 +1,48 @@ +require "test_helper" + +class Storage::ReconcileJobTest < ActiveJob::TestCase + setup do + Current.session = sessions(:david) + @account = accounts("37s") + @board = @account.boards.create!(name: "Test Board", creator: users(:david)) + @card = @board.cards.create!(title: "Test Card", creator: users(:david)) + end + + test "reconcile_storage corrects drift when ledger undercounts" do + @card.image.attach io: StringIO.new("x" * 1000), filename: "test.png", content_type: "image/png" + Storage::Entry.where(board: @board).delete_all + + Storage::ReconcileJob.perform_now(@board) + + entry = Storage::Entry.find_by(board: @board, operation: "reconcile") + assert_not_nil entry + assert_equal 1000, entry.delta + end + + test "reconcile_storage corrects drift when ledger overcounts" do + Storage::Entry.create! \ + account_id: @account.id, + board_id: @board.id, + delta: 5000, + operation: "attach" + + Storage::ReconcileJob.perform_now(@board) + + entry = Storage::Entry.find_by(board: @board, operation: "reconcile") + assert_not_nil entry + assert_equal(-5000, entry.delta) + end + + test "reconcile_storage creates no entry when ledger matches reality" do + @card.image.attach io: StringIO.new("x" * 1000), filename: "test.png", content_type: "image/png" + initial_count = Storage::Entry.where(board: @board).count + + Storage::ReconcileJob.perform_now(@board) + + assert_equal initial_count, Storage::Entry.where(board: @board).count + end + + test "job queued to backend queue" do + assert_equal "backend", Storage::ReconcileJob.new.queue_name + end +end diff --git a/test/models/storage/attachment_tracking_test.rb b/test/models/storage/attachment_tracking_test.rb new file mode 100644 index 000000000..55407630e --- /dev/null +++ b/test/models/storage/attachment_tracking_test.rb @@ -0,0 +1,263 @@ +require "test_helper" + +class Storage::AttachmentTrackingTest < ActiveSupport::TestCase + setup do + Current.session = sessions(:david) + Current.request_id = "test-request-123" + @account = accounts("37s") + @board = boards(:writebook) + @card = cards(:logo) + end + + + # Attachment Creation + + test "attaching file creates storage entry with positive delta" do + assert_difference "Storage::Entry.count", +1 do + @card.image.attach io: StringIO.new("x" * 2048), filename: "test.png", content_type: "image/png" + end + + entry = Storage::Entry.last + assert_equal 2048, entry.delta + assert_equal "attach", entry.operation + assert_equal @account.id, entry.account_id + assert_equal @board.id, entry.board_id + assert_equal @card.class.name, entry.recordable_type + assert_equal @card.id, entry.recordable_id + assert_equal @card.image.blob.id, entry.blob_id + assert_equal Current.user.id, entry.user_id + assert_equal Current.request_id, entry.request_id + end + + test "attaching file enqueues MaterializeJob for account" do + assert_enqueued_with job: Storage::MaterializeJob, args: [ @account ] do + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + end + end + + test "attaching file enqueues MaterializeJob for board" do + assert_enqueued_with job: Storage::MaterializeJob, args: [ @board ] do + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + end + end + + + # Attachment Deletion + + test "destroying attachment creates storage entry with negative delta" do + @card.image.attach io: StringIO.new("x" * 2048), filename: "test.png", content_type: "image/png" + attachment = @card.image.attachment + blob_id = attachment.blob_id + + # Destroy the attachment directly to trigger callbacks + attachment.destroy! + + entry = Storage::Entry.find_by(operation: "detach", recordable: @card) + assert_not_nil entry, "Expected detach entry to be created" + assert_equal -2048, entry.delta + assert_equal "detach", entry.operation + assert_equal blob_id, entry.blob_id + end + + test "destroying attachment uses snapshotted IDs from before_destroy" do + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + + # Capture expected values before destroy + expected_account_id = @account.id + expected_board_id = @board.id + expected_recordable_type = @card.class.name + expected_recordable_id = @card.id + + attachment = @card.image.attachment + attachment.destroy! + + entry = Storage::Entry.find_by(operation: "detach", recordable_id: expected_recordable_id) + assert_not_nil entry, "Expected detach entry to be created" + assert_equal expected_account_id, entry.account_id + assert_equal expected_board_id, entry.board_id + assert_equal expected_recordable_type, entry.recordable_type + assert_equal expected_recordable_id, entry.recordable_id + end + + + # Non-Trackable Records + + test "does not track attachments on records without account method" do + # Account uploads are not trackable (Account.account returns self, but + # uploads on Account are not board-scoped in the same way) + # This test verifies the guard clause works + + # Create a model that doesn't respond to :board + identity = identities(:david) + + # Identity doesn't have :account or :board, so attachments shouldn't be tracked + # (Though in practice, Identity may not have attachments in this codebase) + # We test the guard by checking that the tracking module handles non-trackable records + assert_respond_to @card, :account + assert_respond_to @card, :board + end + + + # Edge Cases + + test "attachment tracking handles nil board gracefully" do + # Create a card with nil board association won't happen in practice + # but test that entry creation handles nil board_id + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + entry = Storage::Entry.last + assert_not_nil entry.account_id + # board_id should be present for cards + assert_not_nil entry.board_id + end + + test "replacing attachment creates detach and attach entries" do + # First attachment + @card.image.attach io: StringIO.new("x" * 1024), filename: "first.png", content_type: "image/png" + initial_count = Storage::Entry.count + + # Replace with new attachment + @card.image.attach io: StringIO.new("x" * 2048), filename: "second.png", content_type: "image/png" + + # Should have detach (-1024) and attach (+2048) entries + # Note: depending on purge_later vs purge, the detach might be async + entries = Storage::Entry.where(recordable: @card).order(:id).last(2) + + # At minimum, we should have the new attach entry + attach_entry = entries.find { |e| e.operation == "attach" && e.delta == 2048 } + assert_not_nil attach_entry + end + + + # Rich Text Embeds + # + # ActionText embeds are automatically extracted from body content that contains + # tags referencing ActiveStorage::Blob objects. + # The embeds association is populated during before_validation callback. + + test "card description embed creates storage entry" do + blob = ActiveStorage::Blob.create_and_upload! \ + io: file_fixture("moon.jpg").open, + filename: "card_embed.jpg", + content_type: "image/jpeg" + + # Create rich text content with embedded blob attachment + attachment_html = ActionText::Attachment.from_attachable(blob).to_html + + assert_difference "Storage::Entry.count", +1 do + @card.update!(description: "

Description with image: #{attachment_html}

") + end + + entry = Storage::Entry.last + assert_equal blob.byte_size, entry.delta + assert_equal "attach", entry.operation + assert_equal "Card", entry.recordable_type + assert_equal @card.id, entry.recordable_id + end + + test "comment embed creates storage entry via rich text body" do + blob = ActiveStorage::Blob.create_and_upload! \ + io: file_fixture("moon.jpg").open, + filename: "comment_image.jpg", + content_type: "image/jpeg" + + attachment_html = ActionText::Attachment.from_attachable(blob).to_html + + assert_difference "Storage::Entry.count", +1 do + @card.comments.create!(body: "

Comment with image: #{attachment_html}

") + end + + entry = Storage::Entry.last + assert_equal blob.byte_size, entry.delta + assert_equal "attach", entry.operation + assert_equal @account.id, entry.account_id + assert_equal @board.id, entry.board_id + assert_equal "Comment", entry.recordable_type + end + + test "comment embed uses card's board for tracking" do + blob = ActiveStorage::Blob.create_and_upload! \ + io: file_fixture("moon.jpg").open, + filename: "test.jpg", + content_type: "image/jpeg" + + attachment_html = ActionText::Attachment.from_attachable(blob).to_html + comment = @card.comments.create!(body: "

Comment: #{attachment_html}

") + + entry = Storage::Entry.last + assert_equal @card.board_id, entry.board_id + assert_equal comment.id, entry.recordable_id + end + + test "board public_description embed creates storage entry" do + blob = ActiveStorage::Blob.create_and_upload! \ + io: file_fixture("moon.jpg").open, + filename: "board_image.jpg", + content_type: "image/jpeg" + + attachment_html = ActionText::Attachment.from_attachable(blob).to_html + + assert_difference "Storage::Entry.count", +1 do + @board.update!(public_description: "

Board description: #{attachment_html}

") + end + + entry = Storage::Entry.last + assert_equal blob.byte_size, entry.delta + assert_equal "attach", entry.operation + assert_equal @account.id, entry.account_id + assert_equal @board.id, entry.board_id + assert_equal "Board", entry.recordable_type + assert_equal @board.id, entry.recordable_id + end + + + # Reconciliation includes all attachment types + + test "board calculate_real_storage_bytes includes comment embeds" do + blob = ActiveStorage::Blob.create_and_upload! \ + io: file_fixture("moon.jpg").open, + filename: "comment_embed.jpg", + content_type: "image/jpeg" + + attachment_html = ActionText::Attachment.from_attachable(blob).to_html + @card.comments.create!(body: "

Comment: #{attachment_html}

") + + board_bytes = @board.send(:calculate_real_storage_bytes) + + assert board_bytes >= blob.byte_size, "board bytes should include comment embed bytes" + end + + test "account calculate_real_storage_bytes includes comment embeds via boards" do + blob = ActiveStorage::Blob.create_and_upload! \ + io: file_fixture("moon.jpg").open, + filename: "comment_embed.jpg", + content_type: "image/jpeg" + + attachment_html = ActionText::Attachment.from_attachable(blob).to_html + @card.comments.create!(body: "

Comment: #{attachment_html}

") + + account_bytes = @account.send(:calculate_real_storage_bytes) + + assert account_bytes >= blob.byte_size, "account bytes should include comment embed bytes" + end + + + # Cascading Deletes + + test "attachment tracking handles card deletion gracefully" do + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + card_id = @card.id + + # Delete the card - this should trigger attachment purge + # The before_destroy snapshot should capture IDs before card is gone + perform_enqueued_jobs do + assert_nothing_raised do + @card.destroy! + end + end + + # Should have detach entry with snapshotted IDs + detach_entry = Storage::Entry.find_by(recordable_id: card_id, operation: "detach") + assert_not_nil detach_entry, "Expected detach entry for destroyed card" + assert_equal -1024, detach_entry.delta + end +end diff --git a/test/models/storage/entry_test.rb b/test/models/storage/entry_test.rb new file mode 100644 index 000000000..cf5befd52 --- /dev/null +++ b/test/models/storage/entry_test.rb @@ -0,0 +1,168 @@ +require "test_helper" + +class Storage::EntryTest < ActiveSupport::TestCase + setup do + @account = accounts("37s") + @board = boards(:writebook) + @card = cards(:logo) + end + + test "record! creates entry with positive delta" do + assert_difference "Storage::Entry.count", +1 do + entry = Storage::Entry.record \ + account: @account, + board: @board, + recordable: @card, + delta: 1024, + operation: "attach" + + assert_equal @account.id, entry.account_id + assert_equal @board.id, entry.board_id + assert_equal @card.class.name, entry.recordable_type + assert_equal @card.id, entry.recordable_id + assert_equal 1024, entry.delta + assert_equal "attach", entry.operation + end + end + + test "record! creates entry with negative delta" do + entry = Storage::Entry.record \ + account: @account, + board: @board, + recordable: @card, + delta: -512, + operation: "detach" + + assert_equal -512, entry.delta + assert_equal "detach", entry.operation + end + + test "record! returns nil and creates no entry when delta is zero" do + assert_no_difference "Storage::Entry.count" do + result = Storage::Entry.record \ + account: @account, + board: @board, + recordable: @card, + delta: 0, + operation: "attach" + + assert_nil result + end + end + + test "record! accepts _id params for after_destroy_commit snapshots" do + entry = Storage::Entry.record \ + account_id: @account.id, + board_id: @board.id, + recordable_type: "Card", + recordable_id: @card.id, + delta: 2048, + operation: "detach" + + assert_equal @account.id, entry.account_id + assert_equal @board.id, entry.board_id + assert_equal "Card", entry.recordable_type + assert_equal @card.id, entry.recordable_id + end + + test "record! creates entry without board" do + entry = Storage::Entry.record \ + account: @account, + board: nil, + recordable: @card, + delta: 1024, + operation: "attach" + + assert_nil entry.board_id + end + + test "record! creates entry without recordable" do + entry = Storage::Entry.record \ + account: @account, + board: @board, + recordable: nil, + delta: 1024, + operation: "reconcile" + + assert_nil entry.recordable_type + assert_nil entry.recordable_id + end + + test "record! enqueues MaterializeJob for account" do + assert_enqueued_with job: Storage::MaterializeJob, args: [ @account ] do + Storage::Entry.record \ + account: @account, + board: nil, + recordable: nil, + delta: 1024, + operation: "attach" + end + end + + test "record! enqueues MaterializeJob for board when board_id present" do + assert_enqueued_with job: Storage::MaterializeJob, args: [ @board ] do + Storage::Entry.record \ + account: @account, + board: @board, + recordable: nil, + delta: 1024, + operation: "attach" + end + end + + test "record! does not enqueue job when account is deleted" do + # The graceful handling is that find_by returns nil, so no job is enqueued + # for a non-existent account. We can't test with a fake ID due to FK constraints, + # but we can verify the find_by behavior by stubbing. + Account.stubs(:find_by).returns(nil) + + assert_no_enqueued_jobs only: Storage::MaterializeJob do + Storage::Entry.record \ + account: @account, + delta: 1024, + operation: "attach" + end + end + + test "record! does not enqueue board job when board is deleted" do + Board.stubs(:find_by).returns(nil) + + # Account job still enqueued, but board job is not + entry = Storage::Entry.record \ + account: @account, + board: @board, + delta: 1024, + operation: "attach" + + assert_not_nil entry + end + + test "entries belong to account" do + entry = Storage::Entry.record \ + account: @account, + delta: 1024, + operation: "attach" + + assert_equal @account, entry.account + end + + test "entries belong to board (optional)" do + entry = Storage::Entry.record \ + account: @account, + board: @board, + delta: 1024, + operation: "attach" + + assert_equal @board, entry.board + end + + test "entries belong to recordable (polymorphic, optional)" do + entry = Storage::Entry.record \ + account: @account, + recordable: @card, + delta: 1024, + operation: "attach" + + assert_equal @card, entry.recordable + end +end diff --git a/test/models/storage/total_test.rb b/test/models/storage/total_test.rb new file mode 100644 index 000000000..c57ee6e4f --- /dev/null +++ b/test/models/storage/total_test.rb @@ -0,0 +1,83 @@ +require "test_helper" + +class Storage::TotalTest < ActiveSupport::TestCase + setup do + @account = accounts("37s") + @board = boards(:writebook) + end + + test "pending_entries returns all entries when no cursor" do + # Create some entries + 3.times do |i| + Storage::Entry.record \ + account: @account, + delta: 1024 * (i + 1), + operation: "attach" + end + + total = @account.create_storage_total! + assert_nil total.last_entry_id + + assert_equal 3, total.pending_entries.count + end + + test "pending_entries returns only entries after cursor" do + # Create first entry and set cursor + entry1 = Storage::Entry.record(account: @account, delta: 1024, operation: "attach") + total = @account.create_storage_total!(last_entry_id: entry1.id, bytes_stored: 1024) + + # Advance time to ensure UUIDv7 timestamps sort correctly + travel 1.second + + # Create more entries AFTER cursor is set + entry2 = Storage::Entry.record(account: @account, delta: 2048, operation: "attach") + travel 1.second + entry3 = Storage::Entry.record(account: @account, delta: 512, operation: "attach") + + pending = total.pending_entries + assert_equal 2, pending.count + assert_includes pending, entry2 + assert_includes pending, entry3 + assert_not_includes pending, entry1 + end + + test "current_usage returns snapshot value when no pending entries" do + total = @account.create_storage_total!(bytes_stored: 5000) + + # No entries exist, so nothing pending + assert_equal 5000, total.current_usage + end + + test "current_usage sums snapshot and pending entries" do + # Create first entry and set cursor + entry1 = Storage::Entry.record(account: @account, delta: 1024, operation: "attach") + total = @account.create_storage_total!(last_entry_id: entry1.id, bytes_stored: 1024) + + # Small delay to ensure UUIDv7 timestamp component advances + travel 1.second + + # Create more entries AFTER cursor is set + Storage::Entry.record(account: @account, delta: 2048, operation: "attach") + travel 1.second + Storage::Entry.record(account: @account, delta: -512, operation: "detach") + + # 1024 (snapshot) + 2048 - 512 (pending) = 2560 + assert_equal 2560, total.current_usage + end + + test "belongs to owner polymorphically" do + account_total = Storage::Total.create!(owner: @account) + assert_equal @account, account_total.owner + + board_total = Storage::Total.create!(owner: @board) + assert_equal @board, board_total.owner + end + + test "unique constraint on owner" do + Storage::Total.create!(owner: @account) + + assert_raises ActiveRecord::RecordNotUnique do + Storage::Total.create!(owner: @account) + end + end +end diff --git a/test/models/storage/totaled_test.rb b/test/models/storage/totaled_test.rb new file mode 100644 index 000000000..f53ce1a89 --- /dev/null +++ b/test/models/storage/totaled_test.rb @@ -0,0 +1,276 @@ +require "test_helper" + +class Storage::TotaledTest < ActiveSupport::TestCase + setup do + Current.session = sessions(:david) + @account = accounts("37s") + @board = boards(:writebook) + end + + + # bytes_used (fast snapshot) + + test "bytes_used returns 0 when no storage_total exists" do + assert_nil @account.storage_total + assert_equal 0, @account.bytes_used + end + + test "bytes_used returns snapshot value" do + @account.create_storage_total!(bytes_stored: 10_000) + assert_equal 10_000, @account.bytes_used + end + + test "bytes_used does not include pending entries (fast path)" do + @account.create_storage_total!(bytes_stored: 1000) + + # Create pending entry (not materialized) + Storage::Entry.record(account: @account, delta: 500, operation: "attach") + + # bytes_used is fast path - only reads snapshot + assert_equal 1000, @account.bytes_used + end + + + # bytes_used_exact (snapshot + pending) + + test "bytes_used_exact creates storage_total if missing" do + assert_nil @account.storage_total + + @account.bytes_used_exact + + assert_not_nil @account.reload.storage_total + end + + test "bytes_used_exact includes pending entries" do + # Create first entry and set cursor at that entry + entry = Storage::Entry.record(account: @account, delta: 500, operation: "attach") + @account.create_storage_total!(bytes_stored: 500, last_entry_id: entry.id) + + # Small delay to ensure UUIDv7 timestamp advances + travel 1.second + + # Create pending entry AFTER cursor + Storage::Entry.record(account: @account, delta: 256, operation: "attach") + + # 500 (snapshot) + 256 (pending) = 756 + assert_equal 756, @account.bytes_used_exact + end + + test "bytes_used_exact returns 0 when no entries and no snapshot" do + assert_equal 0, @account.bytes_used_exact + end + + + # materialize_storage + + test "materialize_storage creates storage_total if missing" do + assert_nil @account.storage_total + + Storage::Entry.record(account: @account, delta: 1024, operation: "attach") + @account.materialize_storage + + total = @account.reload.storage_total + assert_not_nil total + assert_equal 1024, total.bytes_stored + end + + test "materialize_storage processes all pending entries" do + Storage::Entry.record(account: @account, delta: 1000, operation: "attach") + Storage::Entry.record(account: @account, delta: 2000, operation: "attach") + Storage::Entry.record(account: @account, delta: -500, operation: "detach") + + @account.materialize_storage + + assert_equal 2500, @account.storage_total.bytes_stored + assert_equal 0, @account.storage_total.pending_entries.count + end + + test "materialize_storage updates cursor to latest entry" do + entry1 = Storage::Entry.record(account: @account, delta: 1000, operation: "attach") + entry2 = Storage::Entry.record(account: @account, delta: 500, operation: "attach") + + @account.materialize_storage + + assert_equal entry2.id, @account.storage_total.last_entry_id + end + + test "materialize_storage is idempotent when no new entries" do + Storage::Entry.record(account: @account, delta: 1000, operation: "attach") + @account.materialize_storage + + initial_bytes = @account.storage_total.bytes_stored + initial_cursor = @account.storage_total.last_entry_id + + @account.materialize_storage + + assert_equal initial_bytes, @account.storage_total.bytes_stored + assert_equal initial_cursor, @account.storage_total.last_entry_id + end + + test "materialize_storage processes only entries since cursor" do + entry1 = Storage::Entry.record(account: @account, delta: 1000, operation: "attach") + @account.materialize_storage + + assert_equal 1000, @account.storage_total.bytes_stored + + # Small delay to ensure UUIDv7 timestamp advances + travel 1.second + + # Add more entries + Storage::Entry.record(account: @account, delta: 500, operation: "attach") + @account.materialize_storage + + assert_equal 1500, @account.storage_total.bytes_stored + end + + test "materialize_storage does nothing when no entries" do + @account.materialize_storage + + total = @account.reload.storage_total + assert_not_nil total + assert_equal 0, total.bytes_stored + assert_nil total.last_entry_id + end + + test "materialize_storage handles concurrent calls safely" do + # Pre-create storage_total to avoid unique constraint race + @account.create_storage_total! + + Storage::Entry.record(account: @account, delta: 1000, operation: "attach") + + # Simulate concurrent materialization + threads = 3.times.map do + Thread.new do + ActiveRecord::Base.connection_pool.with_connection do + @account.materialize_storage + end + end + end + threads.each(&:join) + + # Should still have correct total + assert_equal 1000, @account.reload.storage_total.bytes_stored + end + + + # storage_entries association + + test "account has storage_entries association" do + entry = Storage::Entry.record(account: @account, delta: 1024, operation: "attach") + + assert_includes @account.storage_entries, entry + end + + test "board has storage_entries association" do + entry = Storage::Entry.record(account: @account, board: @board, delta: 1024, operation: "attach") + + assert_includes @board.storage_entries, entry + end + + + # storage_total association + + test "storage_total is destroyed when owner is destroyed" do + @account.create_storage_total!(bytes_stored: 1000) + total_id = @account.storage_total.id + + # Create a new account to destroy (don't destroy fixtures) + new_account = Account.create!(name: "Temp Account") + new_account.create_storage_total!(bytes_stored: 500) + storage_total_id = new_account.storage_total.id + + new_account.destroy! + + assert_not Storage::Total.exists?(storage_total_id) + end + + + # Board-specific tests + + test "board bytes_used works independently of account" do + # Create entries for both account and board + Storage::Entry.record(account: @account, board: nil, delta: 1000, operation: "attach") + Storage::Entry.record(account: @account, board: @board, delta: 500, operation: "attach") + + @account.materialize_storage + @board.materialize_storage + + # Account sees all its entries (1000 + 500 = 1500) + assert_equal 1500, @account.bytes_used + + # Board only sees entries with its board_id (500) + assert_equal 500, @board.bytes_used + end + + test "board and account have independent cursors" do + entry1 = Storage::Entry.record(account: @account, board: @board, delta: 1000, operation: "attach") + + @account.materialize_storage + # Board not yet materialized + + entry2 = Storage::Entry.record(account: @account, board: @board, delta: 500, operation: "attach") + + # Account cursor at entry1, board has no cursor yet + assert_equal entry1.id, @account.storage_total.last_entry_id + + @board.materialize_storage + + # Board cursor now at entry2 + assert_equal entry2.id, @board.storage_total.last_entry_id + assert_equal 1500, @board.bytes_used + end + + + # reconcile_storage + + test "reconcile_storage creates entry for drift" do + board = @account.boards.create!(name: "Test Board", creator: users(:david)) + card = board.cards.create!(title: "Test Card", creator: users(:david)) + card.image.attach io: StringIO.new("x" * 1000), filename: "test.png", content_type: "image/png" + + # Delete entry to simulate drift + Storage::Entry.where(board: board).delete_all + + assert_difference "Storage::Entry.count", +1 do + board.reconcile_storage + end + + entry = Storage::Entry.find_by(board: board, operation: "reconcile") + assert_equal 1000, entry.delta + end + + test "reconcile_storage no-op when ledger matches reality" do + board = @account.boards.create!(name: "Test Board", creator: users(:david)) + card = board.cards.create!(title: "Test Card", creator: users(:david)) + card.image.attach io: StringIO.new("x" * 1000), filename: "test.png", content_type: "image/png" + + assert_no_difference "Storage::Entry.where(operation: 'reconcile').count" do + board.reconcile_storage + end + end + + test "reconcile_storage handles empty board" do + board = @account.boards.create!(name: "Empty Board", creator: users(:david)) + + assert_no_difference "Storage::Entry.count" do + board.reconcile_storage + end + end + + test "reconcile_storage handles negative drift" do + board = @account.boards.create!(name: "Test Board", creator: users(:david)) + + # Create fake ledger entry with no real attachment + Storage::Entry.create! \ + account_id: @account.id, + board_id: board.id, + delta: 5000, + operation: "attach" + + board.reconcile_storage + + entry = Storage::Entry.find_by(board: board, operation: "reconcile") + assert_not_nil entry + assert_equal(-5000, entry.delta) + end +end diff --git a/test/models/storage/tracked_test.rb b/test/models/storage/tracked_test.rb new file mode 100644 index 000000000..029329792 --- /dev/null +++ b/test/models/storage/tracked_test.rb @@ -0,0 +1,119 @@ +require "test_helper" + +class Storage::TrackedTest < ActiveSupport::TestCase + setup do + Current.session = sessions(:david) + @account = accounts("37s") + @board1 = boards(:writebook) + @board2 = boards(:private) + @card = cards(:logo) + end + + test "storage_bytes returns 0 when no attachments" do + assert_equal 0, @card.storage_bytes + end + + test "storage_bytes sums all attachment blob sizes" do + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + assert_equal 1024, @card.storage_bytes + end + + test "board transfer creates transfer_out entry for old board" do + @card.image.attach io: StringIO.new("x" * 2048), filename: "test.png", content_type: "image/png" + old_board_id = @card.board_id + + assert_difference "Storage::Entry.count", +2 do + @card.update!(board: @board2) + end + + transfer_out = Storage::Entry.find_by(board_id: old_board_id, operation: "transfer_out") + + assert_not_nil transfer_out + assert_equal -2048, transfer_out.delta + assert_equal @account.id, transfer_out.account_id + assert_equal @card.class.name, transfer_out.recordable_type + assert_equal @card.id, transfer_out.recordable_id + end + + test "board transfer creates transfer_in entry for new board" do + @card.image.attach io: StringIO.new("x" * 2048), filename: "test.png", content_type: "image/png" + @card.update!(board: @board2) + + transfer_in = Storage::Entry.find_by(board_id: @board2.id, operation: "transfer_in") + + assert_not_nil transfer_in + assert_equal 2048, transfer_in.delta + assert_equal @account.id, transfer_in.account_id + end + + test "board transfer does not create entries when no attachments" do + # Ensure card has no attachments + @card.image.purge if @card.image.attached? + + # Count only transfer entries + initial_count = Storage::Entry.where(operation: [ "transfer_out", "transfer_in" ]).count + + @card.update!(board: @board2) + + final_count = Storage::Entry.where(operation: [ "transfer_out", "transfer_in" ]).count + assert_equal initial_count, final_count + end + + test "board transfer net effect on account is zero" do + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + + # Materialize account storage before transfer + @account.materialize_storage + initial_account_bytes = @account.bytes_used + + @card.update!(board: @board2) + + # Materialize again + @account.materialize_storage + + # Account total should be unchanged (transfer_out + transfer_in = 0 for account) + assert_equal initial_account_bytes, @account.bytes_used + end + + test "board transfer correctly moves storage between boards" do + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + + # Materialize both boards + @board1.materialize_storage + @board2.materialize_storage + + board1_initial = @board1.bytes_used + board2_initial = @board2.bytes_used + + # Small delay to ensure UUIDv7 timestamp advances for transfer entries + travel 1.second + + @card.update!(board: @board2) + + # Materialize again + @board1.materialize_storage + @board2.materialize_storage + + # Board1 loses 1024, Board2 gains 1024 + assert_equal board1_initial - 1024, @board1.bytes_used + assert_equal board2_initial + 1024, @board2.bytes_used + end + + test "non-board updates do not trigger transfer tracking" do + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + initial_count = Storage::Entry.where(operation: [ "transfer_out", "transfer_in" ]).count + + @card.update!(title: "New Title") + + final_count = Storage::Entry.where(operation: [ "transfer_out", "transfer_in" ]).count + assert_equal initial_count, final_count + end + + test "attachments_for_storage returns all direct attachments" do + @card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png" + attachments = @card.send(:attachments_for_storage) + + assert_equal 1, attachments.count + assert_equal @card.image.blob.byte_size, attachments.first.blob.byte_size + end +end