Speedy, auditable, deadlock-resistant storage tracking (#2026)
An append-only storage ledger replaces deadlock-prone synchronous counter updates and drift-prone async updates. All content storage (card images, card/comment/board description embeds) is tracked. Account exports (which expire) and avatars are not tracked. Storage is accounted for by Account and Board. Both consume the same event stream independently: no bubble-up storage bumps triggering deadlocks due to lock sequencing. Each calculates its own total from the underlying ledger with independent cursors and materialization. * Storage::Entry: Append-only ledger recording attach/detach/transfer events with delta bytes. Single event stream indexed for both Account and Board cursor queries. * Storage::Total: Polymorphic snapshot cache with cursor (last_entry_id) tracking which entries have been materialized. * Storage::Totaled concern: Provides bytes_used (fast snapshot) and bytes_used_exact (snapshot + pending) query modes, plus materialize_storage! to roll up pending entries. * Storage::Tracked concern: For models owning attachments (Card, Comment, Board). Provides board_for_storage_tracking for models where board is determined differently (Board returns self). Handles board transfers by recording transfer_out/transfer_in entries. * Storage::AttachmentTracking: Hooks ActiveStorage::Attachment lifecycle to record attach/detach entries. Handles ActionText::RichText embeds by traversing to the actual model. Snapshots context in before_destroy to handle cascading deletes where parent record may be gone by after_destroy_commit. * MaterializeJob: Rolls up pending entries into snapshot. Concurrency limited per owner to prevent duplicate work. * ReconcileJob: On-demand reconciliation against actual attachment storage for support/debugging. Compares ledger total to real bytes from card images, card embeds, comment embeds, and board embeds. Usage: * account.bytes_used / board.bytes_used: fast, slightly stale bytesize * account.bytes_used_exact / board.bytes_used_exact: real-time bytesize * Storage::Entry: audit trail for debugging and point-in-time queries
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
class Storage::MaterializeJob < ApplicationJob
|
||||
queue_as :backend
|
||||
limits_concurrency to: 1, key: ->(owner) { owner }
|
||||
|
||||
def perform(owner)
|
||||
owner.materialize_storage
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,7 @@
|
||||
class Storage::ReconcileJob < ApplicationJob
|
||||
queue_as :backend
|
||||
|
||||
def perform(owner)
|
||||
owner.reconcile_storage
|
||||
end
|
||||
end
|
||||
@@ -1,5 +1,5 @@
|
||||
class Account < ApplicationRecord
|
||||
include Entropic, Seedeable
|
||||
include Account::Storage, Entropic, Seedeable
|
||||
|
||||
has_one :join_code
|
||||
has_many :users, dependent: :destroy
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
module Account::Storage
|
||||
extend ActiveSupport::Concern
|
||||
include Storage::Totaled
|
||||
|
||||
private
|
||||
def calculate_real_storage_bytes
|
||||
boards.sum { |board| board.send(:calculate_real_storage_bytes) }
|
||||
end
|
||||
end
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
class Board < ApplicationRecord
|
||||
include Accessible, AutoPostponing, Broadcastable, Cards, Entropic, Filterable, Publishable, Triageable
|
||||
include Accessible, AutoPostponing, Board::Storage, Broadcastable, Cards, Entropic, Filterable, Publishable, ::Storage::Tracked, Triageable
|
||||
|
||||
belongs_to :creator, class_name: "User", default: -> { Current.user }
|
||||
belongs_to :account, default: -> { creator.account }
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
module Board::Storage
|
||||
extend ActiveSupport::Concern
|
||||
include Storage::Totaled
|
||||
|
||||
# Board's own embeds (public_description) count toward itself
|
||||
def board_for_storage_tracking
|
||||
self
|
||||
end
|
||||
|
||||
private
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
# Calculate actual storage by summing blob sizes.
|
||||
#
|
||||
# Uses batched pluck queries to avoid loading huge ID arrays, and avoids
|
||||
# ActiveRecord model queries on ActiveStorage tables to sidestep cross-pool
|
||||
# issues when ActiveStorage uses separate connection pools (e.g., with replicas).
|
||||
def calculate_real_storage_bytes
|
||||
card_image_bytes + card_embed_bytes + comment_embed_bytes + board_embed_bytes
|
||||
end
|
||||
|
||||
def card_image_bytes
|
||||
sum_blob_bytes_in_batches \
|
||||
ActiveStorage::Attachment.where(record_type: "Card", name: "image"),
|
||||
cards.pluck(:id)
|
||||
end
|
||||
|
||||
def card_embed_bytes
|
||||
sum_embed_bytes_for "Card", cards.pluck(:id)
|
||||
end
|
||||
|
||||
def comment_embed_bytes
|
||||
sum_embed_bytes_for "Comment", Comment.where(card_id: cards.pluck(:id)).pluck(:id)
|
||||
end
|
||||
|
||||
def board_embed_bytes
|
||||
sum_embed_bytes_for "Board", [ id ]
|
||||
end
|
||||
|
||||
def sum_embed_bytes_for(record_type, record_ids)
|
||||
rich_text_ids = ActionText::RichText \
|
||||
.where(record_type: record_type, record_id: record_ids)
|
||||
.pluck(:id)
|
||||
|
||||
sum_blob_bytes_in_batches \
|
||||
ActiveStorage::Attachment.where(record_type: "ActionText::RichText", name: "embeds"),
|
||||
rich_text_ids
|
||||
end
|
||||
|
||||
def sum_blob_bytes_in_batches(base_scope, record_ids)
|
||||
record_ids.each_slice(BATCH_SIZE).sum do |batch_ids|
|
||||
blob_ids = base_scope.where(record_id: batch_ids).pluck(:blob_id)
|
||||
ActiveStorage::Blob.where(id: blob_ids).sum(:byte_size)
|
||||
end
|
||||
end
|
||||
end
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
class Card < ApplicationRecord
|
||||
include Assignable, Attachments, Broadcastable, Closeable, Colored, Entropic, Eventable,
|
||||
Exportable, Golden, Mentions, Multistep, Pinnable, Postponable, Promptable,
|
||||
Readable, Searchable, Stallable, Statuses, Taggable, Triageable, Watchable
|
||||
Readable, Searchable, Stallable, Statuses, Storage::Tracked, Taggable, Triageable, Watchable
|
||||
|
||||
belongs_to :account, default: -> { board.account }
|
||||
belongs_to :board
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
class Comment < ApplicationRecord
|
||||
include Attachments, Eventable, Mentions, Promptable, Searchable
|
||||
include Attachments, Eventable, Mentions, Promptable, Searchable, Storage::Tracked
|
||||
|
||||
belongs_to :account, default: -> { card.account }
|
||||
belongs_to :card, touch: true
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
module Storage::Totaled
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
included do
|
||||
has_one :storage_total, as: :owner, class_name: "Storage::Total", dependent: :destroy
|
||||
has_many :storage_entries, class_name: "Storage::Entry", foreign_key: foreign_key_for_storage
|
||||
end
|
||||
|
||||
class_methods do
|
||||
def foreign_key_for_storage
|
||||
"#{model_name.singular}_id"
|
||||
end
|
||||
end
|
||||
|
||||
# Fast: materialized snapshot (may be slightly stale)
|
||||
def bytes_used
|
||||
storage_total&.bytes_stored || 0
|
||||
end
|
||||
|
||||
# Exact: snapshot + pending entries
|
||||
def bytes_used_exact
|
||||
(storage_total || create_storage_total!).current_usage
|
||||
end
|
||||
|
||||
def materialize_storage_later
|
||||
Storage::MaterializeJob.perform_later(self)
|
||||
end
|
||||
|
||||
# Materialize all pending entries into snapshot
|
||||
def materialize_storage
|
||||
total = storage_total || create_storage_total!
|
||||
|
||||
total.with_lock do
|
||||
latest_entry_id = storage_entries.maximum(:id)
|
||||
|
||||
if latest_entry_id && total.last_entry_id != latest_entry_id
|
||||
scope = storage_entries.where(id: ..latest_entry_id)
|
||||
scope = scope.where.not(id: ..total.last_entry_id) if total.last_entry_id
|
||||
delta_sum = scope.sum(:delta)
|
||||
|
||||
total.update! bytes_stored: total.bytes_stored + delta_sum, last_entry_id: latest_entry_id
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Reconcile ledger against actual attachment storage.
|
||||
# Uses cursor to ensure consistency: captures max entry ID first, then calculates
|
||||
# real bytes, then sums only entries up to that cursor. Concurrent uploads during
|
||||
# calculation will have entries with IDs beyond the cursor, avoiding double-count.
|
||||
def reconcile_storage
|
||||
max_entry_id = storage_entries.maximum(:id)
|
||||
real_bytes = calculate_real_storage_bytes
|
||||
ledger_bytes = max_entry_id ? storage_entries.where(id: ..max_entry_id).sum(:delta) : 0
|
||||
diff = real_bytes - ledger_bytes
|
||||
|
||||
if diff.nonzero?
|
||||
Storage::Entry.record \
|
||||
account: is_a?(Account) ? self : account,
|
||||
board: is_a?(Board) ? self : nil,
|
||||
recordable: nil,
|
||||
delta: diff,
|
||||
operation: "reconcile"
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def calculate_real_storage_bytes
|
||||
raise NotImplementedError, "Subclass must implement calculate_real_storage_bytes"
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,57 @@
|
||||
module Storage::Tracked
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
included do
|
||||
before_update :track_board_transfer, if: :board_transfer?
|
||||
end
|
||||
|
||||
# Return self as the trackable record for storage entries
|
||||
def storage_tracked_record
|
||||
self
|
||||
end
|
||||
|
||||
# Override in models where board is determined differently (e.g., Board itself)
|
||||
def board_for_storage_tracking
|
||||
board
|
||||
end
|
||||
|
||||
# Total bytes for all attachments on this record
|
||||
def storage_bytes
|
||||
attachments_for_storage.sum { |a| a.blob.byte_size }
|
||||
end
|
||||
|
||||
private
|
||||
def board_transfer?
|
||||
respond_to?(:board_id_changed?) && board_id_changed?
|
||||
end
|
||||
|
||||
def track_board_transfer
|
||||
old_board_id = board_id_was
|
||||
current_bytes = storage_bytes
|
||||
|
||||
if current_bytes.positive?
|
||||
# Debit old board
|
||||
if old_board_id
|
||||
Storage::Entry.record \
|
||||
account: account,
|
||||
board_id: old_board_id,
|
||||
recordable: self,
|
||||
delta: -current_bytes,
|
||||
operation: "transfer_out"
|
||||
end
|
||||
|
||||
# Credit new board
|
||||
Storage::Entry.record \
|
||||
account: account,
|
||||
board: board,
|
||||
recordable: self,
|
||||
delta: current_bytes,
|
||||
operation: "transfer_in"
|
||||
end
|
||||
end
|
||||
|
||||
# Override if needed. Default = all direct attachments
|
||||
def attachments_for_storage
|
||||
ActiveStorage::Attachment.where(record: self)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,5 @@
|
||||
module Storage
|
||||
def self.table_name_prefix
|
||||
"storage_"
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,53 @@
|
||||
module Storage::AttachmentTracking
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
included do
|
||||
# Snapshot IDs in before_destroy since parent record may be deleted
|
||||
# by the time after_destroy_commit runs
|
||||
before_destroy :snapshot_storage_context
|
||||
after_create_commit :record_storage_attach
|
||||
after_destroy_commit :record_storage_detach
|
||||
end
|
||||
|
||||
private
|
||||
def record_storage_attach
|
||||
return unless storage_tracked_record
|
||||
|
||||
Storage::Entry.record \
|
||||
account: storage_tracked_record.account,
|
||||
board: storage_tracked_record.board_for_storage_tracking,
|
||||
recordable: storage_tracked_record,
|
||||
blob: blob,
|
||||
delta: blob.byte_size,
|
||||
operation: "attach"
|
||||
end
|
||||
|
||||
def record_storage_detach
|
||||
return unless @storage_snapshot
|
||||
|
||||
Storage::Entry.record \
|
||||
account_id: @storage_snapshot[:account_id],
|
||||
board_id: @storage_snapshot[:board_id],
|
||||
recordable_type: @storage_snapshot[:recordable_type],
|
||||
recordable_id: @storage_snapshot[:recordable_id],
|
||||
blob_id: @storage_snapshot[:blob_id],
|
||||
delta: -blob.byte_size,
|
||||
operation: "detach"
|
||||
end
|
||||
|
||||
def snapshot_storage_context
|
||||
return unless storage_tracked_record
|
||||
|
||||
@storage_snapshot = {
|
||||
account_id: storage_tracked_record.account.id,
|
||||
board_id: storage_tracked_record.board_for_storage_tracking&.id,
|
||||
recordable_type: storage_tracked_record.class.name,
|
||||
recordable_id: storage_tracked_record.id,
|
||||
blob_id: blob.id
|
||||
}
|
||||
end
|
||||
|
||||
def storage_tracked_record
|
||||
record.try(:storage_tracked_record)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,35 @@
|
||||
class Storage::Entry < ApplicationRecord
|
||||
belongs_to :account
|
||||
belongs_to :board, optional: true
|
||||
belongs_to :recordable, polymorphic: true, optional: true
|
||||
|
||||
scope :pending, ->(last_entry_id) { where.not(id: ..last_entry_id) if last_entry_id }
|
||||
|
||||
# Accepts either objects or _id params (for after_destroy_commit snapshots)
|
||||
def self.record(delta:, operation:, account: nil, account_id: nil, board: nil, board_id: nil,
|
||||
recordable: nil, recordable_type: nil, recordable_id: nil, blob: nil, blob_id: nil)
|
||||
return if delta.zero?
|
||||
|
||||
account_id ||= account&.id
|
||||
board_id ||= board&.id
|
||||
blob_id ||= blob&.id
|
||||
|
||||
entry = create! \
|
||||
account_id: account_id,
|
||||
board_id: board_id,
|
||||
recordable_type: recordable_type || recordable&.class&.name,
|
||||
recordable_id: recordable_id || recordable&.id,
|
||||
blob_id: blob_id,
|
||||
delta: delta,
|
||||
operation: operation,
|
||||
user_id: Current.user&.id,
|
||||
request_id: Current.request_id
|
||||
|
||||
# Enqueue materialization - use find_by to handle cascading deletes
|
||||
# (Account/Board may be destroyed while attachments are still being cleaned up)
|
||||
Account.find_by(id: account_id)&.materialize_storage_later
|
||||
Board.find_by(id: board_id)&.materialize_storage_later if board_id
|
||||
|
||||
entry
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,12 @@
|
||||
class Storage::Total < ApplicationRecord
|
||||
belongs_to :owner, polymorphic: true
|
||||
|
||||
def pending_entries
|
||||
owner.storage_entries.pending(last_entry_id)
|
||||
end
|
||||
|
||||
# Exact current usage (snapshot + pending)
|
||||
def current_usage
|
||||
bytes_stored + pending_entries.sum(:delta)
|
||||
end
|
||||
end
|
||||
@@ -11,6 +11,11 @@ module ActionText
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Delegate storage tracking to the parent record (Card, Comment, Board, etc.)
|
||||
def storage_tracked_record
|
||||
record.try(:storage_tracked_record)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
ActiveSupport.on_load(:active_storage_attachment) do
|
||||
include Storage::AttachmentTracking
|
||||
end
|
||||
|
||||
ActiveSupport.on_load(:active_storage_blob) do
|
||||
ActiveStorage::DiskController.after_action only: :show do
|
||||
expires_in 5.minutes, public: true
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
class CreateStorageTables < ActiveRecord::Migration[8.0]
|
||||
def change
|
||||
# Storage ledger: debit/credit event stream
|
||||
create_table :storage_entries, id: :uuid do |t|
|
||||
t.references :account, type: :uuid, null: false
|
||||
t.references :board, type: :uuid, null: true
|
||||
|
||||
t.references :recordable, type: :uuid, polymorphic: true, null: true
|
||||
|
||||
t.bigint :delta, null: false
|
||||
t.string :operation, null: false
|
||||
|
||||
t.datetime :created_at, null: false
|
||||
end
|
||||
|
||||
# Storage totals: cached snapshots
|
||||
create_table :storage_totals, id: :uuid do |t|
|
||||
t.references :owner, type: :uuid, polymorphic: true, null: false, index: false
|
||||
|
||||
t.bigint :bytes_stored, null: false, default: 0
|
||||
t.uuid :last_entry_id # Cursor: includes all entries <= this ID
|
||||
|
||||
t.timestamps
|
||||
t.index %i[ owner_type owner_id ], unique: true
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,9 @@
|
||||
class AddBlobIdAndAuditContextToStorageEntries < ActiveRecord::Migration[8.2]
|
||||
def change
|
||||
change_table :storage_entries do |t|
|
||||
t.references :blob, type: :uuid, foreign_key: false, index: true
|
||||
t.references :user, type: :uuid, foreign_key: false, index: true
|
||||
t.string :request_id, index: true
|
||||
end
|
||||
end
|
||||
end
|
||||
Generated
+30
-1
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do
|
||||
ActiveRecord::Schema[8.2].define(version: 2025_12_10_054934) do
|
||||
create_table "accesses", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||
t.datetime "accessed_at"
|
||||
t.uuid "account_id", null: false
|
||||
@@ -697,6 +697,35 @@ ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do
|
||||
t.index ["card_id"], name: "index_steps_on_card_id"
|
||||
end
|
||||
|
||||
create_table "storage_entries", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||
t.uuid "account_id", null: false
|
||||
t.uuid "blob_id"
|
||||
t.uuid "board_id"
|
||||
t.datetime "created_at", null: false
|
||||
t.bigint "delta", null: false
|
||||
t.string "operation", null: false
|
||||
t.uuid "recordable_id"
|
||||
t.string "recordable_type"
|
||||
t.string "request_id"
|
||||
t.uuid "user_id"
|
||||
t.index ["account_id"], name: "index_storage_entries_on_account_id"
|
||||
t.index ["blob_id"], name: "index_storage_entries_on_blob_id"
|
||||
t.index ["board_id"], name: "index_storage_entries_on_board_id"
|
||||
t.index ["recordable_type", "recordable_id"], name: "index_storage_entries_on_recordable"
|
||||
t.index ["request_id"], name: "index_storage_entries_on_request_id"
|
||||
t.index ["user_id"], name: "index_storage_entries_on_user_id"
|
||||
end
|
||||
|
||||
create_table "storage_totals", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||
t.bigint "bytes_stored", default: 0, null: false
|
||||
t.datetime "created_at", null: false
|
||||
t.uuid "last_entry_id"
|
||||
t.uuid "owner_id", null: false
|
||||
t.string "owner_type", null: false
|
||||
t.datetime "updated_at", null: false
|
||||
t.index ["owner_type", "owner_id"], name: "index_storage_totals_on_owner_type_and_owner_id", unique: true
|
||||
end
|
||||
|
||||
create_table "taggings", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||
t.uuid "account_id", null: false
|
||||
t.uuid "card_id", null: false
|
||||
|
||||
+30
-1
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do
|
||||
ActiveRecord::Schema[8.2].define(version: 2025_12_10_054934) do
|
||||
create_table "accesses", id: :uuid, force: :cascade do |t|
|
||||
t.datetime "accessed_at"
|
||||
t.uuid "account_id", null: false
|
||||
@@ -470,6 +470,35 @@ ActiveRecord::Schema[8.2].define(version: 2025_12_05_010536) do
|
||||
t.index ["card_id"], name: "index_steps_on_card_id"
|
||||
end
|
||||
|
||||
create_table "storage_entries", id: :uuid, force: :cascade do |t|
|
||||
t.uuid "account_id", null: false
|
||||
t.uuid "blob_id"
|
||||
t.uuid "board_id"
|
||||
t.datetime "created_at", null: false
|
||||
t.bigint "delta", null: false
|
||||
t.string "operation", limit: 255, null: false
|
||||
t.uuid "recordable_id"
|
||||
t.string "recordable_type", limit: 255
|
||||
t.string "request_id"
|
||||
t.uuid "user_id"
|
||||
t.index ["account_id"], name: "index_storage_entries_on_account_id"
|
||||
t.index ["blob_id"], name: "index_storage_entries_on_blob_id"
|
||||
t.index ["board_id"], name: "index_storage_entries_on_board_id"
|
||||
t.index ["recordable_type", "recordable_id"], name: "index_storage_entries_on_recordable"
|
||||
t.index ["request_id"], name: "index_storage_entries_on_request_id"
|
||||
t.index ["user_id"], name: "index_storage_entries_on_user_id"
|
||||
end
|
||||
|
||||
create_table "storage_totals", id: :uuid, force: :cascade do |t|
|
||||
t.bigint "bytes_stored", default: 0, null: false
|
||||
t.datetime "created_at", null: false
|
||||
t.uuid "last_entry_id"
|
||||
t.uuid "owner_id", null: false
|
||||
t.string "owner_type", limit: 255, null: false
|
||||
t.datetime "updated_at", null: false
|
||||
t.index ["owner_type", "owner_id"], name: "index_storage_totals_on_owner_type_and_owner_id", unique: true
|
||||
end
|
||||
|
||||
create_table "taggings", id: :uuid, force: :cascade do |t|
|
||||
t.uuid "account_id", null: false
|
||||
t.uuid "card_id", null: false
|
||||
|
||||
@@ -34,6 +34,18 @@ module ActiveRecordReplicaSupport
|
||||
def replica_configured?
|
||||
configurations.find_db_config("replica").present?
|
||||
end
|
||||
|
||||
# Execute block using read replica if available, otherwise use primary.
|
||||
#
|
||||
# Example:
|
||||
# ApplicationRecord.with_reading_role { User.count }
|
||||
def with_reading_role(&block)
|
||||
if replica_configured?
|
||||
connected_to(role: :reading, &block)
|
||||
else
|
||||
yield
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env ruby
|
||||
|
||||
# Backfill storage ledger with attach entries for all existing attachments.
|
||||
#
|
||||
# Run locally:
|
||||
# bin/rails runner script/migrations/backfill-storage-ledger.rb
|
||||
#
|
||||
# Run via Kamal:
|
||||
# kamal app exec -d <stage> -p --reuse "bin/rails runner script/migrations/backfill-storage-ledger.rb"
|
||||
#
|
||||
# Safe to re-run: skips attachments that already have entries (by blob_id).
|
||||
class BackfillStorageLedger
|
||||
def run
|
||||
puts "Backfilling storage entries…"
|
||||
backfill_entries
|
||||
|
||||
puts "\nMaterializing totals…"
|
||||
materialize_totals
|
||||
end
|
||||
|
||||
private
|
||||
def backfill_entries
|
||||
created = 0
|
||||
skipped = 0
|
||||
|
||||
ActiveStorage::Attachment.includes(:blob).find_each do |attachment|
|
||||
record = attachment.record.try(:storage_tracked_record)
|
||||
|
||||
if record.nil? || Storage::Entry.exists?(blob_id: attachment.blob_id)
|
||||
skipped += 1
|
||||
next
|
||||
end
|
||||
|
||||
Storage::Entry.create! \
|
||||
account_id: record.account.id,
|
||||
board_id: record.board_for_storage_tracking&.id,
|
||||
recordable_type: record.class.name,
|
||||
recordable_id: record.id,
|
||||
blob_id: attachment.blob_id,
|
||||
delta: attachment.blob.byte_size,
|
||||
operation: "attach"
|
||||
created += 1
|
||||
|
||||
print "." if created % 100 == 0
|
||||
end
|
||||
|
||||
puts "\n\nBackfill complete!"
|
||||
puts " Entries created: #{created}"
|
||||
puts " Attachments skipped: #{skipped}"
|
||||
end
|
||||
|
||||
def materialize_totals
|
||||
boards_materialized = 0
|
||||
accounts_materialized = 0
|
||||
|
||||
Board.find_each do |board|
|
||||
board.materialize_storage
|
||||
boards_materialized += 1
|
||||
print "." if boards_materialized % 100 == 0
|
||||
end
|
||||
|
||||
Account.find_each do |account|
|
||||
account.materialize_storage
|
||||
accounts_materialized += 1
|
||||
end
|
||||
|
||||
puts "\n\nMaterialization complete!"
|
||||
puts " Boards: #{boards_materialized}"
|
||||
puts " Accounts: #{accounts_materialized}"
|
||||
end
|
||||
end
|
||||
|
||||
BackfillStorageLedger.new.run
|
||||
@@ -0,0 +1,58 @@
|
||||
require "test_helper"
|
||||
|
||||
class Storage::MaterializeJobTest < ActiveJob::TestCase
|
||||
setup do
|
||||
@account = accounts("37s")
|
||||
@board = boards(:writebook)
|
||||
end
|
||||
|
||||
test "calls materialize_storage on account" do
|
||||
Storage::Entry.record(account: @account, delta: 1024, operation: "attach")
|
||||
|
||||
Storage::MaterializeJob.perform_now(@account)
|
||||
|
||||
assert_not_nil @account.storage_total
|
||||
assert_equal 1024, @account.bytes_used
|
||||
end
|
||||
|
||||
test "calls materialize_storage on board" do
|
||||
Storage::Entry.record(account: @account, board: @board, delta: 2048, operation: "attach")
|
||||
|
||||
Storage::MaterializeJob.perform_now(@board)
|
||||
|
||||
assert_not_nil @board.storage_total
|
||||
assert_equal 2048, @board.bytes_used
|
||||
end
|
||||
|
||||
test "job is idempotent" do
|
||||
Storage::Entry.record(account: @account, delta: 1024, operation: "attach")
|
||||
|
||||
3.times { Storage::MaterializeJob.perform_now(@account) }
|
||||
|
||||
assert_equal 1024, @account.bytes_used
|
||||
end
|
||||
|
||||
test "job processes entries added between runs" do
|
||||
Storage::Entry.record(account: @account, delta: 1000, operation: "attach")
|
||||
Storage::MaterializeJob.perform_now(@account)
|
||||
|
||||
# Small delay to ensure UUIDv7 timestamp advances
|
||||
travel 1.second
|
||||
|
||||
Storage::Entry.record(account: @account, delta: 500, operation: "attach")
|
||||
Storage::MaterializeJob.perform_now(@account)
|
||||
|
||||
assert_equal 1500, @account.bytes_used
|
||||
end
|
||||
|
||||
test "job queued to backend queue" do
|
||||
assert_equal "backend", Storage::MaterializeJob.new.queue_name
|
||||
end
|
||||
|
||||
test "job has concurrency limit by owner" do
|
||||
job = Storage::MaterializeJob.new(@account)
|
||||
# limits_concurrency is a Solid Queue feature
|
||||
# Just verify the job can be instantiated and has the correct queue
|
||||
assert_equal "backend", job.queue_name
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,48 @@
|
||||
require "test_helper"
|
||||
|
||||
class Storage::ReconcileJobTest < ActiveJob::TestCase
|
||||
setup do
|
||||
Current.session = sessions(:david)
|
||||
@account = accounts("37s")
|
||||
@board = @account.boards.create!(name: "Test Board", creator: users(:david))
|
||||
@card = @board.cards.create!(title: "Test Card", creator: users(:david))
|
||||
end
|
||||
|
||||
test "reconcile_storage corrects drift when ledger undercounts" do
|
||||
@card.image.attach io: StringIO.new("x" * 1000), filename: "test.png", content_type: "image/png"
|
||||
Storage::Entry.where(board: @board).delete_all
|
||||
|
||||
Storage::ReconcileJob.perform_now(@board)
|
||||
|
||||
entry = Storage::Entry.find_by(board: @board, operation: "reconcile")
|
||||
assert_not_nil entry
|
||||
assert_equal 1000, entry.delta
|
||||
end
|
||||
|
||||
test "reconcile_storage corrects drift when ledger overcounts" do
|
||||
Storage::Entry.create! \
|
||||
account_id: @account.id,
|
||||
board_id: @board.id,
|
||||
delta: 5000,
|
||||
operation: "attach"
|
||||
|
||||
Storage::ReconcileJob.perform_now(@board)
|
||||
|
||||
entry = Storage::Entry.find_by(board: @board, operation: "reconcile")
|
||||
assert_not_nil entry
|
||||
assert_equal(-5000, entry.delta)
|
||||
end
|
||||
|
||||
test "reconcile_storage creates no entry when ledger matches reality" do
|
||||
@card.image.attach io: StringIO.new("x" * 1000), filename: "test.png", content_type: "image/png"
|
||||
initial_count = Storage::Entry.where(board: @board).count
|
||||
|
||||
Storage::ReconcileJob.perform_now(@board)
|
||||
|
||||
assert_equal initial_count, Storage::Entry.where(board: @board).count
|
||||
end
|
||||
|
||||
test "job queued to backend queue" do
|
||||
assert_equal "backend", Storage::ReconcileJob.new.queue_name
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,263 @@
|
||||
require "test_helper"
|
||||
|
||||
class Storage::AttachmentTrackingTest < ActiveSupport::TestCase
|
||||
setup do
|
||||
Current.session = sessions(:david)
|
||||
Current.request_id = "test-request-123"
|
||||
@account = accounts("37s")
|
||||
@board = boards(:writebook)
|
||||
@card = cards(:logo)
|
||||
end
|
||||
|
||||
|
||||
# Attachment Creation
|
||||
|
||||
test "attaching file creates storage entry with positive delta" do
|
||||
assert_difference "Storage::Entry.count", +1 do
|
||||
@card.image.attach io: StringIO.new("x" * 2048), filename: "test.png", content_type: "image/png"
|
||||
end
|
||||
|
||||
entry = Storage::Entry.last
|
||||
assert_equal 2048, entry.delta
|
||||
assert_equal "attach", entry.operation
|
||||
assert_equal @account.id, entry.account_id
|
||||
assert_equal @board.id, entry.board_id
|
||||
assert_equal @card.class.name, entry.recordable_type
|
||||
assert_equal @card.id, entry.recordable_id
|
||||
assert_equal @card.image.blob.id, entry.blob_id
|
||||
assert_equal Current.user.id, entry.user_id
|
||||
assert_equal Current.request_id, entry.request_id
|
||||
end
|
||||
|
||||
test "attaching file enqueues MaterializeJob for account" do
|
||||
assert_enqueued_with job: Storage::MaterializeJob, args: [ @account ] do
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
end
|
||||
end
|
||||
|
||||
test "attaching file enqueues MaterializeJob for board" do
|
||||
assert_enqueued_with job: Storage::MaterializeJob, args: [ @board ] do
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# Attachment Deletion
|
||||
|
||||
test "destroying attachment creates storage entry with negative delta" do
|
||||
@card.image.attach io: StringIO.new("x" * 2048), filename: "test.png", content_type: "image/png"
|
||||
attachment = @card.image.attachment
|
||||
blob_id = attachment.blob_id
|
||||
|
||||
# Destroy the attachment directly to trigger callbacks
|
||||
attachment.destroy!
|
||||
|
||||
entry = Storage::Entry.find_by(operation: "detach", recordable: @card)
|
||||
assert_not_nil entry, "Expected detach entry to be created"
|
||||
assert_equal -2048, entry.delta
|
||||
assert_equal "detach", entry.operation
|
||||
assert_equal blob_id, entry.blob_id
|
||||
end
|
||||
|
||||
test "destroying attachment uses snapshotted IDs from before_destroy" do
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
|
||||
# Capture expected values before destroy
|
||||
expected_account_id = @account.id
|
||||
expected_board_id = @board.id
|
||||
expected_recordable_type = @card.class.name
|
||||
expected_recordable_id = @card.id
|
||||
|
||||
attachment = @card.image.attachment
|
||||
attachment.destroy!
|
||||
|
||||
entry = Storage::Entry.find_by(operation: "detach", recordable_id: expected_recordable_id)
|
||||
assert_not_nil entry, "Expected detach entry to be created"
|
||||
assert_equal expected_account_id, entry.account_id
|
||||
assert_equal expected_board_id, entry.board_id
|
||||
assert_equal expected_recordable_type, entry.recordable_type
|
||||
assert_equal expected_recordable_id, entry.recordable_id
|
||||
end
|
||||
|
||||
|
||||
# Non-Trackable Records
|
||||
|
||||
test "does not track attachments on records without account method" do
|
||||
# Account uploads are not trackable (Account.account returns self, but
|
||||
# uploads on Account are not board-scoped in the same way)
|
||||
# This test verifies the guard clause works
|
||||
|
||||
# Create a model that doesn't respond to :board
|
||||
identity = identities(:david)
|
||||
|
||||
# Identity doesn't have :account or :board, so attachments shouldn't be tracked
|
||||
# (Though in practice, Identity may not have attachments in this codebase)
|
||||
# We test the guard by checking that the tracking module handles non-trackable records
|
||||
assert_respond_to @card, :account
|
||||
assert_respond_to @card, :board
|
||||
end
|
||||
|
||||
|
||||
# Edge Cases
|
||||
|
||||
test "attachment tracking handles nil board gracefully" do
|
||||
# Create a card with nil board association won't happen in practice
|
||||
# but test that entry creation handles nil board_id
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
entry = Storage::Entry.last
|
||||
assert_not_nil entry.account_id
|
||||
# board_id should be present for cards
|
||||
assert_not_nil entry.board_id
|
||||
end
|
||||
|
||||
test "replacing attachment creates detach and attach entries" do
|
||||
# First attachment
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "first.png", content_type: "image/png"
|
||||
initial_count = Storage::Entry.count
|
||||
|
||||
# Replace with new attachment
|
||||
@card.image.attach io: StringIO.new("x" * 2048), filename: "second.png", content_type: "image/png"
|
||||
|
||||
# Should have detach (-1024) and attach (+2048) entries
|
||||
# Note: depending on purge_later vs purge, the detach might be async
|
||||
entries = Storage::Entry.where(recordable: @card).order(:id).last(2)
|
||||
|
||||
# At minimum, we should have the new attach entry
|
||||
attach_entry = entries.find { |e| e.operation == "attach" && e.delta == 2048 }
|
||||
assert_not_nil attach_entry
|
||||
end
|
||||
|
||||
|
||||
# Rich Text Embeds
|
||||
#
|
||||
# ActionText embeds are automatically extracted from body content that contains
|
||||
# <action-text-attachment> tags referencing ActiveStorage::Blob objects.
|
||||
# The embeds association is populated during before_validation callback.
|
||||
|
||||
test "card description embed creates storage entry" do
|
||||
blob = ActiveStorage::Blob.create_and_upload! \
|
||||
io: file_fixture("moon.jpg").open,
|
||||
filename: "card_embed.jpg",
|
||||
content_type: "image/jpeg"
|
||||
|
||||
# Create rich text content with embedded blob attachment
|
||||
attachment_html = ActionText::Attachment.from_attachable(blob).to_html
|
||||
|
||||
assert_difference "Storage::Entry.count", +1 do
|
||||
@card.update!(description: "<p>Description with image: #{attachment_html}</p>")
|
||||
end
|
||||
|
||||
entry = Storage::Entry.last
|
||||
assert_equal blob.byte_size, entry.delta
|
||||
assert_equal "attach", entry.operation
|
||||
assert_equal "Card", entry.recordable_type
|
||||
assert_equal @card.id, entry.recordable_id
|
||||
end
|
||||
|
||||
test "comment embed creates storage entry via rich text body" do
|
||||
blob = ActiveStorage::Blob.create_and_upload! \
|
||||
io: file_fixture("moon.jpg").open,
|
||||
filename: "comment_image.jpg",
|
||||
content_type: "image/jpeg"
|
||||
|
||||
attachment_html = ActionText::Attachment.from_attachable(blob).to_html
|
||||
|
||||
assert_difference "Storage::Entry.count", +1 do
|
||||
@card.comments.create!(body: "<p>Comment with image: #{attachment_html}</p>")
|
||||
end
|
||||
|
||||
entry = Storage::Entry.last
|
||||
assert_equal blob.byte_size, entry.delta
|
||||
assert_equal "attach", entry.operation
|
||||
assert_equal @account.id, entry.account_id
|
||||
assert_equal @board.id, entry.board_id
|
||||
assert_equal "Comment", entry.recordable_type
|
||||
end
|
||||
|
||||
test "comment embed uses card's board for tracking" do
|
||||
blob = ActiveStorage::Blob.create_and_upload! \
|
||||
io: file_fixture("moon.jpg").open,
|
||||
filename: "test.jpg",
|
||||
content_type: "image/jpeg"
|
||||
|
||||
attachment_html = ActionText::Attachment.from_attachable(blob).to_html
|
||||
comment = @card.comments.create!(body: "<p>Comment: #{attachment_html}</p>")
|
||||
|
||||
entry = Storage::Entry.last
|
||||
assert_equal @card.board_id, entry.board_id
|
||||
assert_equal comment.id, entry.recordable_id
|
||||
end
|
||||
|
||||
test "board public_description embed creates storage entry" do
|
||||
blob = ActiveStorage::Blob.create_and_upload! \
|
||||
io: file_fixture("moon.jpg").open,
|
||||
filename: "board_image.jpg",
|
||||
content_type: "image/jpeg"
|
||||
|
||||
attachment_html = ActionText::Attachment.from_attachable(blob).to_html
|
||||
|
||||
assert_difference "Storage::Entry.count", +1 do
|
||||
@board.update!(public_description: "<p>Board description: #{attachment_html}</p>")
|
||||
end
|
||||
|
||||
entry = Storage::Entry.last
|
||||
assert_equal blob.byte_size, entry.delta
|
||||
assert_equal "attach", entry.operation
|
||||
assert_equal @account.id, entry.account_id
|
||||
assert_equal @board.id, entry.board_id
|
||||
assert_equal "Board", entry.recordable_type
|
||||
assert_equal @board.id, entry.recordable_id
|
||||
end
|
||||
|
||||
|
||||
# Reconciliation includes all attachment types
|
||||
|
||||
test "board calculate_real_storage_bytes includes comment embeds" do
|
||||
blob = ActiveStorage::Blob.create_and_upload! \
|
||||
io: file_fixture("moon.jpg").open,
|
||||
filename: "comment_embed.jpg",
|
||||
content_type: "image/jpeg"
|
||||
|
||||
attachment_html = ActionText::Attachment.from_attachable(blob).to_html
|
||||
@card.comments.create!(body: "<p>Comment: #{attachment_html}</p>")
|
||||
|
||||
board_bytes = @board.send(:calculate_real_storage_bytes)
|
||||
|
||||
assert board_bytes >= blob.byte_size, "board bytes should include comment embed bytes"
|
||||
end
|
||||
|
||||
test "account calculate_real_storage_bytes includes comment embeds via boards" do
|
||||
blob = ActiveStorage::Blob.create_and_upload! \
|
||||
io: file_fixture("moon.jpg").open,
|
||||
filename: "comment_embed.jpg",
|
||||
content_type: "image/jpeg"
|
||||
|
||||
attachment_html = ActionText::Attachment.from_attachable(blob).to_html
|
||||
@card.comments.create!(body: "<p>Comment: #{attachment_html}</p>")
|
||||
|
||||
account_bytes = @account.send(:calculate_real_storage_bytes)
|
||||
|
||||
assert account_bytes >= blob.byte_size, "account bytes should include comment embed bytes"
|
||||
end
|
||||
|
||||
|
||||
# Cascading Deletes
|
||||
|
||||
test "attachment tracking handles card deletion gracefully" do
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
card_id = @card.id
|
||||
|
||||
# Delete the card - this should trigger attachment purge
|
||||
# The before_destroy snapshot should capture IDs before card is gone
|
||||
perform_enqueued_jobs do
|
||||
assert_nothing_raised do
|
||||
@card.destroy!
|
||||
end
|
||||
end
|
||||
|
||||
# Should have detach entry with snapshotted IDs
|
||||
detach_entry = Storage::Entry.find_by(recordable_id: card_id, operation: "detach")
|
||||
assert_not_nil detach_entry, "Expected detach entry for destroyed card"
|
||||
assert_equal -1024, detach_entry.delta
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,168 @@
|
||||
require "test_helper"
|
||||
|
||||
class Storage::EntryTest < ActiveSupport::TestCase
|
||||
setup do
|
||||
@account = accounts("37s")
|
||||
@board = boards(:writebook)
|
||||
@card = cards(:logo)
|
||||
end
|
||||
|
||||
test "record! creates entry with positive delta" do
|
||||
assert_difference "Storage::Entry.count", +1 do
|
||||
entry = Storage::Entry.record \
|
||||
account: @account,
|
||||
board: @board,
|
||||
recordable: @card,
|
||||
delta: 1024,
|
||||
operation: "attach"
|
||||
|
||||
assert_equal @account.id, entry.account_id
|
||||
assert_equal @board.id, entry.board_id
|
||||
assert_equal @card.class.name, entry.recordable_type
|
||||
assert_equal @card.id, entry.recordable_id
|
||||
assert_equal 1024, entry.delta
|
||||
assert_equal "attach", entry.operation
|
||||
end
|
||||
end
|
||||
|
||||
test "record! creates entry with negative delta" do
|
||||
entry = Storage::Entry.record \
|
||||
account: @account,
|
||||
board: @board,
|
||||
recordable: @card,
|
||||
delta: -512,
|
||||
operation: "detach"
|
||||
|
||||
assert_equal -512, entry.delta
|
||||
assert_equal "detach", entry.operation
|
||||
end
|
||||
|
||||
test "record! returns nil and creates no entry when delta is zero" do
|
||||
assert_no_difference "Storage::Entry.count" do
|
||||
result = Storage::Entry.record \
|
||||
account: @account,
|
||||
board: @board,
|
||||
recordable: @card,
|
||||
delta: 0,
|
||||
operation: "attach"
|
||||
|
||||
assert_nil result
|
||||
end
|
||||
end
|
||||
|
||||
test "record! accepts _id params for after_destroy_commit snapshots" do
|
||||
entry = Storage::Entry.record \
|
||||
account_id: @account.id,
|
||||
board_id: @board.id,
|
||||
recordable_type: "Card",
|
||||
recordable_id: @card.id,
|
||||
delta: 2048,
|
||||
operation: "detach"
|
||||
|
||||
assert_equal @account.id, entry.account_id
|
||||
assert_equal @board.id, entry.board_id
|
||||
assert_equal "Card", entry.recordable_type
|
||||
assert_equal @card.id, entry.recordable_id
|
||||
end
|
||||
|
||||
test "record! creates entry without board" do
|
||||
entry = Storage::Entry.record \
|
||||
account: @account,
|
||||
board: nil,
|
||||
recordable: @card,
|
||||
delta: 1024,
|
||||
operation: "attach"
|
||||
|
||||
assert_nil entry.board_id
|
||||
end
|
||||
|
||||
test "record! creates entry without recordable" do
|
||||
entry = Storage::Entry.record \
|
||||
account: @account,
|
||||
board: @board,
|
||||
recordable: nil,
|
||||
delta: 1024,
|
||||
operation: "reconcile"
|
||||
|
||||
assert_nil entry.recordable_type
|
||||
assert_nil entry.recordable_id
|
||||
end
|
||||
|
||||
test "record! enqueues MaterializeJob for account" do
|
||||
assert_enqueued_with job: Storage::MaterializeJob, args: [ @account ] do
|
||||
Storage::Entry.record \
|
||||
account: @account,
|
||||
board: nil,
|
||||
recordable: nil,
|
||||
delta: 1024,
|
||||
operation: "attach"
|
||||
end
|
||||
end
|
||||
|
||||
test "record! enqueues MaterializeJob for board when board_id present" do
|
||||
assert_enqueued_with job: Storage::MaterializeJob, args: [ @board ] do
|
||||
Storage::Entry.record \
|
||||
account: @account,
|
||||
board: @board,
|
||||
recordable: nil,
|
||||
delta: 1024,
|
||||
operation: "attach"
|
||||
end
|
||||
end
|
||||
|
||||
test "record! does not enqueue job when account is deleted" do
|
||||
# The graceful handling is that find_by returns nil, so no job is enqueued
|
||||
# for a non-existent account. We can't test with a fake ID due to FK constraints,
|
||||
# but we can verify the find_by behavior by stubbing.
|
||||
Account.stubs(:find_by).returns(nil)
|
||||
|
||||
assert_no_enqueued_jobs only: Storage::MaterializeJob do
|
||||
Storage::Entry.record \
|
||||
account: @account,
|
||||
delta: 1024,
|
||||
operation: "attach"
|
||||
end
|
||||
end
|
||||
|
||||
test "record! does not enqueue board job when board is deleted" do
|
||||
Board.stubs(:find_by).returns(nil)
|
||||
|
||||
# Account job still enqueued, but board job is not
|
||||
entry = Storage::Entry.record \
|
||||
account: @account,
|
||||
board: @board,
|
||||
delta: 1024,
|
||||
operation: "attach"
|
||||
|
||||
assert_not_nil entry
|
||||
end
|
||||
|
||||
test "entries belong to account" do
|
||||
entry = Storage::Entry.record \
|
||||
account: @account,
|
||||
delta: 1024,
|
||||
operation: "attach"
|
||||
|
||||
assert_equal @account, entry.account
|
||||
end
|
||||
|
||||
test "entries belong to board (optional)" do
|
||||
entry = Storage::Entry.record \
|
||||
account: @account,
|
||||
board: @board,
|
||||
delta: 1024,
|
||||
operation: "attach"
|
||||
|
||||
assert_equal @board, entry.board
|
||||
end
|
||||
|
||||
test "entries belong to recordable (polymorphic, optional)" do
|
||||
entry = Storage::Entry.record \
|
||||
account: @account,
|
||||
recordable: @card,
|
||||
delta: 1024,
|
||||
operation: "attach"
|
||||
|
||||
assert_equal @card, entry.recordable
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,83 @@
|
||||
require "test_helper"
|
||||
|
||||
class Storage::TotalTest < ActiveSupport::TestCase
|
||||
setup do
|
||||
@account = accounts("37s")
|
||||
@board = boards(:writebook)
|
||||
end
|
||||
|
||||
test "pending_entries returns all entries when no cursor" do
|
||||
# Create some entries
|
||||
3.times do |i|
|
||||
Storage::Entry.record \
|
||||
account: @account,
|
||||
delta: 1024 * (i + 1),
|
||||
operation: "attach"
|
||||
end
|
||||
|
||||
total = @account.create_storage_total!
|
||||
assert_nil total.last_entry_id
|
||||
|
||||
assert_equal 3, total.pending_entries.count
|
||||
end
|
||||
|
||||
test "pending_entries returns only entries after cursor" do
|
||||
# Create first entry and set cursor
|
||||
entry1 = Storage::Entry.record(account: @account, delta: 1024, operation: "attach")
|
||||
total = @account.create_storage_total!(last_entry_id: entry1.id, bytes_stored: 1024)
|
||||
|
||||
# Advance time to ensure UUIDv7 timestamps sort correctly
|
||||
travel 1.second
|
||||
|
||||
# Create more entries AFTER cursor is set
|
||||
entry2 = Storage::Entry.record(account: @account, delta: 2048, operation: "attach")
|
||||
travel 1.second
|
||||
entry3 = Storage::Entry.record(account: @account, delta: 512, operation: "attach")
|
||||
|
||||
pending = total.pending_entries
|
||||
assert_equal 2, pending.count
|
||||
assert_includes pending, entry2
|
||||
assert_includes pending, entry3
|
||||
assert_not_includes pending, entry1
|
||||
end
|
||||
|
||||
test "current_usage returns snapshot value when no pending entries" do
|
||||
total = @account.create_storage_total!(bytes_stored: 5000)
|
||||
|
||||
# No entries exist, so nothing pending
|
||||
assert_equal 5000, total.current_usage
|
||||
end
|
||||
|
||||
test "current_usage sums snapshot and pending entries" do
|
||||
# Create first entry and set cursor
|
||||
entry1 = Storage::Entry.record(account: @account, delta: 1024, operation: "attach")
|
||||
total = @account.create_storage_total!(last_entry_id: entry1.id, bytes_stored: 1024)
|
||||
|
||||
# Small delay to ensure UUIDv7 timestamp component advances
|
||||
travel 1.second
|
||||
|
||||
# Create more entries AFTER cursor is set
|
||||
Storage::Entry.record(account: @account, delta: 2048, operation: "attach")
|
||||
travel 1.second
|
||||
Storage::Entry.record(account: @account, delta: -512, operation: "detach")
|
||||
|
||||
# 1024 (snapshot) + 2048 - 512 (pending) = 2560
|
||||
assert_equal 2560, total.current_usage
|
||||
end
|
||||
|
||||
test "belongs to owner polymorphically" do
|
||||
account_total = Storage::Total.create!(owner: @account)
|
||||
assert_equal @account, account_total.owner
|
||||
|
||||
board_total = Storage::Total.create!(owner: @board)
|
||||
assert_equal @board, board_total.owner
|
||||
end
|
||||
|
||||
test "unique constraint on owner" do
|
||||
Storage::Total.create!(owner: @account)
|
||||
|
||||
assert_raises ActiveRecord::RecordNotUnique do
|
||||
Storage::Total.create!(owner: @account)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,276 @@
|
||||
require "test_helper"
|
||||
|
||||
class Storage::TotaledTest < ActiveSupport::TestCase
|
||||
setup do
|
||||
Current.session = sessions(:david)
|
||||
@account = accounts("37s")
|
||||
@board = boards(:writebook)
|
||||
end
|
||||
|
||||
|
||||
# bytes_used (fast snapshot)
|
||||
|
||||
test "bytes_used returns 0 when no storage_total exists" do
|
||||
assert_nil @account.storage_total
|
||||
assert_equal 0, @account.bytes_used
|
||||
end
|
||||
|
||||
test "bytes_used returns snapshot value" do
|
||||
@account.create_storage_total!(bytes_stored: 10_000)
|
||||
assert_equal 10_000, @account.bytes_used
|
||||
end
|
||||
|
||||
test "bytes_used does not include pending entries (fast path)" do
|
||||
@account.create_storage_total!(bytes_stored: 1000)
|
||||
|
||||
# Create pending entry (not materialized)
|
||||
Storage::Entry.record(account: @account, delta: 500, operation: "attach")
|
||||
|
||||
# bytes_used is fast path - only reads snapshot
|
||||
assert_equal 1000, @account.bytes_used
|
||||
end
|
||||
|
||||
|
||||
# bytes_used_exact (snapshot + pending)
|
||||
|
||||
test "bytes_used_exact creates storage_total if missing" do
|
||||
assert_nil @account.storage_total
|
||||
|
||||
@account.bytes_used_exact
|
||||
|
||||
assert_not_nil @account.reload.storage_total
|
||||
end
|
||||
|
||||
test "bytes_used_exact includes pending entries" do
|
||||
# Create first entry and set cursor at that entry
|
||||
entry = Storage::Entry.record(account: @account, delta: 500, operation: "attach")
|
||||
@account.create_storage_total!(bytes_stored: 500, last_entry_id: entry.id)
|
||||
|
||||
# Small delay to ensure UUIDv7 timestamp advances
|
||||
travel 1.second
|
||||
|
||||
# Create pending entry AFTER cursor
|
||||
Storage::Entry.record(account: @account, delta: 256, operation: "attach")
|
||||
|
||||
# 500 (snapshot) + 256 (pending) = 756
|
||||
assert_equal 756, @account.bytes_used_exact
|
||||
end
|
||||
|
||||
test "bytes_used_exact returns 0 when no entries and no snapshot" do
|
||||
assert_equal 0, @account.bytes_used_exact
|
||||
end
|
||||
|
||||
|
||||
# materialize_storage
|
||||
|
||||
test "materialize_storage creates storage_total if missing" do
|
||||
assert_nil @account.storage_total
|
||||
|
||||
Storage::Entry.record(account: @account, delta: 1024, operation: "attach")
|
||||
@account.materialize_storage
|
||||
|
||||
total = @account.reload.storage_total
|
||||
assert_not_nil total
|
||||
assert_equal 1024, total.bytes_stored
|
||||
end
|
||||
|
||||
test "materialize_storage processes all pending entries" do
|
||||
Storage::Entry.record(account: @account, delta: 1000, operation: "attach")
|
||||
Storage::Entry.record(account: @account, delta: 2000, operation: "attach")
|
||||
Storage::Entry.record(account: @account, delta: -500, operation: "detach")
|
||||
|
||||
@account.materialize_storage
|
||||
|
||||
assert_equal 2500, @account.storage_total.bytes_stored
|
||||
assert_equal 0, @account.storage_total.pending_entries.count
|
||||
end
|
||||
|
||||
test "materialize_storage updates cursor to latest entry" do
|
||||
entry1 = Storage::Entry.record(account: @account, delta: 1000, operation: "attach")
|
||||
entry2 = Storage::Entry.record(account: @account, delta: 500, operation: "attach")
|
||||
|
||||
@account.materialize_storage
|
||||
|
||||
assert_equal entry2.id, @account.storage_total.last_entry_id
|
||||
end
|
||||
|
||||
test "materialize_storage is idempotent when no new entries" do
|
||||
Storage::Entry.record(account: @account, delta: 1000, operation: "attach")
|
||||
@account.materialize_storage
|
||||
|
||||
initial_bytes = @account.storage_total.bytes_stored
|
||||
initial_cursor = @account.storage_total.last_entry_id
|
||||
|
||||
@account.materialize_storage
|
||||
|
||||
assert_equal initial_bytes, @account.storage_total.bytes_stored
|
||||
assert_equal initial_cursor, @account.storage_total.last_entry_id
|
||||
end
|
||||
|
||||
test "materialize_storage processes only entries since cursor" do
|
||||
entry1 = Storage::Entry.record(account: @account, delta: 1000, operation: "attach")
|
||||
@account.materialize_storage
|
||||
|
||||
assert_equal 1000, @account.storage_total.bytes_stored
|
||||
|
||||
# Small delay to ensure UUIDv7 timestamp advances
|
||||
travel 1.second
|
||||
|
||||
# Add more entries
|
||||
Storage::Entry.record(account: @account, delta: 500, operation: "attach")
|
||||
@account.materialize_storage
|
||||
|
||||
assert_equal 1500, @account.storage_total.bytes_stored
|
||||
end
|
||||
|
||||
test "materialize_storage does nothing when no entries" do
|
||||
@account.materialize_storage
|
||||
|
||||
total = @account.reload.storage_total
|
||||
assert_not_nil total
|
||||
assert_equal 0, total.bytes_stored
|
||||
assert_nil total.last_entry_id
|
||||
end
|
||||
|
||||
test "materialize_storage handles concurrent calls safely" do
|
||||
# Pre-create storage_total to avoid unique constraint race
|
||||
@account.create_storage_total!
|
||||
|
||||
Storage::Entry.record(account: @account, delta: 1000, operation: "attach")
|
||||
|
||||
# Simulate concurrent materialization
|
||||
threads = 3.times.map do
|
||||
Thread.new do
|
||||
ActiveRecord::Base.connection_pool.with_connection do
|
||||
@account.materialize_storage
|
||||
end
|
||||
end
|
||||
end
|
||||
threads.each(&:join)
|
||||
|
||||
# Should still have correct total
|
||||
assert_equal 1000, @account.reload.storage_total.bytes_stored
|
||||
end
|
||||
|
||||
|
||||
# storage_entries association
|
||||
|
||||
test "account has storage_entries association" do
|
||||
entry = Storage::Entry.record(account: @account, delta: 1024, operation: "attach")
|
||||
|
||||
assert_includes @account.storage_entries, entry
|
||||
end
|
||||
|
||||
test "board has storage_entries association" do
|
||||
entry = Storage::Entry.record(account: @account, board: @board, delta: 1024, operation: "attach")
|
||||
|
||||
assert_includes @board.storage_entries, entry
|
||||
end
|
||||
|
||||
|
||||
# storage_total association
|
||||
|
||||
test "storage_total is destroyed when owner is destroyed" do
|
||||
@account.create_storage_total!(bytes_stored: 1000)
|
||||
total_id = @account.storage_total.id
|
||||
|
||||
# Create a new account to destroy (don't destroy fixtures)
|
||||
new_account = Account.create!(name: "Temp Account")
|
||||
new_account.create_storage_total!(bytes_stored: 500)
|
||||
storage_total_id = new_account.storage_total.id
|
||||
|
||||
new_account.destroy!
|
||||
|
||||
assert_not Storage::Total.exists?(storage_total_id)
|
||||
end
|
||||
|
||||
|
||||
# Board-specific tests
|
||||
|
||||
test "board bytes_used works independently of account" do
|
||||
# Create entries for both account and board
|
||||
Storage::Entry.record(account: @account, board: nil, delta: 1000, operation: "attach")
|
||||
Storage::Entry.record(account: @account, board: @board, delta: 500, operation: "attach")
|
||||
|
||||
@account.materialize_storage
|
||||
@board.materialize_storage
|
||||
|
||||
# Account sees all its entries (1000 + 500 = 1500)
|
||||
assert_equal 1500, @account.bytes_used
|
||||
|
||||
# Board only sees entries with its board_id (500)
|
||||
assert_equal 500, @board.bytes_used
|
||||
end
|
||||
|
||||
test "board and account have independent cursors" do
|
||||
entry1 = Storage::Entry.record(account: @account, board: @board, delta: 1000, operation: "attach")
|
||||
|
||||
@account.materialize_storage
|
||||
# Board not yet materialized
|
||||
|
||||
entry2 = Storage::Entry.record(account: @account, board: @board, delta: 500, operation: "attach")
|
||||
|
||||
# Account cursor at entry1, board has no cursor yet
|
||||
assert_equal entry1.id, @account.storage_total.last_entry_id
|
||||
|
||||
@board.materialize_storage
|
||||
|
||||
# Board cursor now at entry2
|
||||
assert_equal entry2.id, @board.storage_total.last_entry_id
|
||||
assert_equal 1500, @board.bytes_used
|
||||
end
|
||||
|
||||
|
||||
# reconcile_storage
|
||||
|
||||
test "reconcile_storage creates entry for drift" do
|
||||
board = @account.boards.create!(name: "Test Board", creator: users(:david))
|
||||
card = board.cards.create!(title: "Test Card", creator: users(:david))
|
||||
card.image.attach io: StringIO.new("x" * 1000), filename: "test.png", content_type: "image/png"
|
||||
|
||||
# Delete entry to simulate drift
|
||||
Storage::Entry.where(board: board).delete_all
|
||||
|
||||
assert_difference "Storage::Entry.count", +1 do
|
||||
board.reconcile_storage
|
||||
end
|
||||
|
||||
entry = Storage::Entry.find_by(board: board, operation: "reconcile")
|
||||
assert_equal 1000, entry.delta
|
||||
end
|
||||
|
||||
test "reconcile_storage no-op when ledger matches reality" do
|
||||
board = @account.boards.create!(name: "Test Board", creator: users(:david))
|
||||
card = board.cards.create!(title: "Test Card", creator: users(:david))
|
||||
card.image.attach io: StringIO.new("x" * 1000), filename: "test.png", content_type: "image/png"
|
||||
|
||||
assert_no_difference "Storage::Entry.where(operation: 'reconcile').count" do
|
||||
board.reconcile_storage
|
||||
end
|
||||
end
|
||||
|
||||
test "reconcile_storage handles empty board" do
|
||||
board = @account.boards.create!(name: "Empty Board", creator: users(:david))
|
||||
|
||||
assert_no_difference "Storage::Entry.count" do
|
||||
board.reconcile_storage
|
||||
end
|
||||
end
|
||||
|
||||
test "reconcile_storage handles negative drift" do
|
||||
board = @account.boards.create!(name: "Test Board", creator: users(:david))
|
||||
|
||||
# Create fake ledger entry with no real attachment
|
||||
Storage::Entry.create! \
|
||||
account_id: @account.id,
|
||||
board_id: board.id,
|
||||
delta: 5000,
|
||||
operation: "attach"
|
||||
|
||||
board.reconcile_storage
|
||||
|
||||
entry = Storage::Entry.find_by(board: board, operation: "reconcile")
|
||||
assert_not_nil entry
|
||||
assert_equal(-5000, entry.delta)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,119 @@
|
||||
require "test_helper"
|
||||
|
||||
class Storage::TrackedTest < ActiveSupport::TestCase
|
||||
setup do
|
||||
Current.session = sessions(:david)
|
||||
@account = accounts("37s")
|
||||
@board1 = boards(:writebook)
|
||||
@board2 = boards(:private)
|
||||
@card = cards(:logo)
|
||||
end
|
||||
|
||||
test "storage_bytes returns 0 when no attachments" do
|
||||
assert_equal 0, @card.storage_bytes
|
||||
end
|
||||
|
||||
test "storage_bytes sums all attachment blob sizes" do
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
assert_equal 1024, @card.storage_bytes
|
||||
end
|
||||
|
||||
test "board transfer creates transfer_out entry for old board" do
|
||||
@card.image.attach io: StringIO.new("x" * 2048), filename: "test.png", content_type: "image/png"
|
||||
old_board_id = @card.board_id
|
||||
|
||||
assert_difference "Storage::Entry.count", +2 do
|
||||
@card.update!(board: @board2)
|
||||
end
|
||||
|
||||
transfer_out = Storage::Entry.find_by(board_id: old_board_id, operation: "transfer_out")
|
||||
|
||||
assert_not_nil transfer_out
|
||||
assert_equal -2048, transfer_out.delta
|
||||
assert_equal @account.id, transfer_out.account_id
|
||||
assert_equal @card.class.name, transfer_out.recordable_type
|
||||
assert_equal @card.id, transfer_out.recordable_id
|
||||
end
|
||||
|
||||
test "board transfer creates transfer_in entry for new board" do
|
||||
@card.image.attach io: StringIO.new("x" * 2048), filename: "test.png", content_type: "image/png"
|
||||
@card.update!(board: @board2)
|
||||
|
||||
transfer_in = Storage::Entry.find_by(board_id: @board2.id, operation: "transfer_in")
|
||||
|
||||
assert_not_nil transfer_in
|
||||
assert_equal 2048, transfer_in.delta
|
||||
assert_equal @account.id, transfer_in.account_id
|
||||
end
|
||||
|
||||
test "board transfer does not create entries when no attachments" do
|
||||
# Ensure card has no attachments
|
||||
@card.image.purge if @card.image.attached?
|
||||
|
||||
# Count only transfer entries
|
||||
initial_count = Storage::Entry.where(operation: [ "transfer_out", "transfer_in" ]).count
|
||||
|
||||
@card.update!(board: @board2)
|
||||
|
||||
final_count = Storage::Entry.where(operation: [ "transfer_out", "transfer_in" ]).count
|
||||
assert_equal initial_count, final_count
|
||||
end
|
||||
|
||||
test "board transfer net effect on account is zero" do
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
|
||||
# Materialize account storage before transfer
|
||||
@account.materialize_storage
|
||||
initial_account_bytes = @account.bytes_used
|
||||
|
||||
@card.update!(board: @board2)
|
||||
|
||||
# Materialize again
|
||||
@account.materialize_storage
|
||||
|
||||
# Account total should be unchanged (transfer_out + transfer_in = 0 for account)
|
||||
assert_equal initial_account_bytes, @account.bytes_used
|
||||
end
|
||||
|
||||
test "board transfer correctly moves storage between boards" do
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
|
||||
# Materialize both boards
|
||||
@board1.materialize_storage
|
||||
@board2.materialize_storage
|
||||
|
||||
board1_initial = @board1.bytes_used
|
||||
board2_initial = @board2.bytes_used
|
||||
|
||||
# Small delay to ensure UUIDv7 timestamp advances for transfer entries
|
||||
travel 1.second
|
||||
|
||||
@card.update!(board: @board2)
|
||||
|
||||
# Materialize again
|
||||
@board1.materialize_storage
|
||||
@board2.materialize_storage
|
||||
|
||||
# Board1 loses 1024, Board2 gains 1024
|
||||
assert_equal board1_initial - 1024, @board1.bytes_used
|
||||
assert_equal board2_initial + 1024, @board2.bytes_used
|
||||
end
|
||||
|
||||
test "non-board updates do not trigger transfer tracking" do
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
initial_count = Storage::Entry.where(operation: [ "transfer_out", "transfer_in" ]).count
|
||||
|
||||
@card.update!(title: "New Title")
|
||||
|
||||
final_count = Storage::Entry.where(operation: [ "transfer_out", "transfer_in" ]).count
|
||||
assert_equal initial_count, final_count
|
||||
end
|
||||
|
||||
test "attachments_for_storage returns all direct attachments" do
|
||||
@card.image.attach io: StringIO.new("x" * 1024), filename: "test.png", content_type: "image/png"
|
||||
attachments = @card.send(:attachments_for_storage)
|
||||
|
||||
assert_equal 1, attachments.count
|
||||
assert_equal @card.image.blob.byte_size, attachments.first.blob.byte_size
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user