diff --git a/app/jobs/import_account_data_job.rb b/app/jobs/import_account_data_job.rb index 5aa29dc9e..e131a41e9 100644 --- a/app/jobs/import_account_data_job.rb +++ b/app/jobs/import_account_data_job.rb @@ -1,7 +1,19 @@ class ImportAccountDataJob < ApplicationJob + include ActiveJob::Continuable + queue_as :backend def perform(import) - import.perform + step :validate do + import.validate \ + start: step.cursor, + callback: proc { |record_set:, record_id:| step.set! [ record_set, record_id ] } + end + + step :process do + import.process \ + start: step.cursor, + callback: proc { |record_set:, record_id:| step.set! [ record_set, record_id ] } + end end end diff --git a/app/models/account/data_transfer/access_record_set.rb b/app/models/account/data_transfer/access_record_set.rb new file mode 100644 index 000000000..9b5e05ceb --- /dev/null +++ b/app/models/account/data_transfer/access_record_set.rb @@ -0,0 +1,48 @@ +class Account::DataTransfer::AccessRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + accessed_at + account_id + board_id + created_at + id + involvement + updated_at + user_id + ].freeze + + private + def records + Access.where(account: account) + end + + def export_record(access) + zip.add_file "data/accesses/#{access.id}.json", access.as_json.to_json + end + + def files + zip.glob("data/accesses/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Access.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Access record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/account_record_set.rb b/app/models/account/data_transfer/account_record_set.rb new file mode 100644 index 000000000..c27e03101 --- /dev/null +++ b/app/models/account/data_transfer/account_record_set.rb @@ -0,0 +1,54 @@ +class Account::DataTransfer::AccountRecordSet < Account::DataTransfer::RecordSet + ACCOUNT_ATTRIBUTES = %w[ + join_code + name + ] + + JOIN_CODE_ATTRIBUTES = %w[ + code + usage_count + usage_limit + ] + + private + def records + [ account ] + end + + def export_record(account) + zip.add_file "data/account.json", account.as_json.merge(join_code: account.join_code.as_json).to_json + end + + def files + [ "data/account.json" ] + end + + def import_batch(files) + account_data = load(files.first) + join_code_data = account_data.delete("join_code") + + account.update!(name: account_data.fetch("name")) + account.join_code.update!(join_code_data.slice("usage_count", "usage_limit")) + account.join_code.update(code: join_code_data.fetch("code")) + end + + def validate_record(file_path) + data = load(file_path) + + unless (ACCOUNT_ATTRIBUTES - data.keys).empty? + raise IntegrityError, "Account record missing required fields" + end + + unless data.key?("join_code") + raise IntegrityError, "Account record missing 'join_code' field" + end + + unless data["join_code"].is_a?(Hash) + raise IntegrityError, "'join_code' field must be a JSON object" + end + + unless (JOIN_CODE_ATTRIBUTES - data["join_code"].keys).empty? + raise IntegrityError, "'join_code' field missing required keys" + end + end +end diff --git a/app/models/account/data_transfer/action_text_rich_text_record_set.rb b/app/models/account/data_transfer/action_text_rich_text_record_set.rb new file mode 100644 index 000000000..62491fa52 --- /dev/null +++ b/app/models/account/data_transfer/action_text_rich_text_record_set.rb @@ -0,0 +1,82 @@ +class Account::DataTransfer::ActionTextRichTextRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + body + created_at + id + name + record_id + record_type + updated_at + ].freeze + + private + def records + ActionText::RichText.where(account: account) + end + + def export_record(rich_text) + data = rich_text.as_json.merge("body" => convert_sgids_to_gids(rich_text.body)) + zip.add_file "data/action_text_rich_texts/#{rich_text.id}.json", data.to_json + end + + def files + zip.glob("data/action_text_rich_texts/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data["body"] = convert_gids_to_sgids(data["body"]) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + ActionText::RichText.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "ActionTextRichText record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end + + def convert_sgids_to_gids(content) + return nil if content.blank? + + content.send(:attachment_nodes).each do |node| + sgid = SignedGlobalID.parse(node["sgid"], for: ActionText::Attachable::LOCATOR_NAME) + record = sgid&.find + next if record&.account_id != account.id + + node["gid"] = record.to_global_id.to_s + node.remove_attribute("sgid") + end + + content.fragment.source.to_html + end + + def convert_gids_to_sgids(html) + return html if html.blank? + + fragment = Nokogiri::HTML.fragment(html) + + fragment.css("action-text-attachment[gid]").each do |node| + gid = GlobalID.parse(node["gid"]) + next unless gid + + record = gid.find + node["sgid"] = record.attachable_sgid + node.remove_attribute("gid") + end + + fragment.to_html + end +end diff --git a/app/models/account/data_transfer/active_storage_attachment_record_set.rb b/app/models/account/data_transfer/active_storage_attachment_record_set.rb new file mode 100644 index 000000000..5e10c7622 --- /dev/null +++ b/app/models/account/data_transfer/active_storage_attachment_record_set.rb @@ -0,0 +1,47 @@ +class Account::DataTransfer::ActiveStorageAttachmentRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + blob_id + created_at + id + name + record_id + record_type + ].freeze + + private + def records + ActiveStorage::Attachment.where(account: account) + end + + def export_record(attachment) + zip.add_file "data/active_storage_attachments/#{attachment.id}.json", attachment.as_json.to_json + end + + def files + zip.glob("data/active_storage_attachments/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + ActiveStorage::Attachment.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "ActiveStorageAttachment record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/active_storage_blob_record_set.rb b/app/models/account/data_transfer/active_storage_blob_record_set.rb new file mode 100644 index 000000000..ccc5e2bab --- /dev/null +++ b/app/models/account/data_transfer/active_storage_blob_record_set.rb @@ -0,0 +1,50 @@ +class Account::DataTransfer::ActiveStorageBlobRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + byte_size + checksum + content_type + created_at + filename + id + key + metadata + service_name + ].freeze + + private + def records + ActiveStorage::Blob.where(account: account) + end + + def export_record(blob) + zip.add_file "data/active_storage_blobs/#{blob.id}.json", blob.as_json.to_json + end + + def files + zip.glob("data/active_storage_blobs/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + ActiveStorage::Blob.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "ActiveStorageBlob record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/assignment_record_set.rb b/app/models/account/data_transfer/assignment_record_set.rb new file mode 100644 index 000000000..437c564cb --- /dev/null +++ b/app/models/account/data_transfer/assignment_record_set.rb @@ -0,0 +1,47 @@ +class Account::DataTransfer::AssignmentRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + assignee_id + assigner_id + card_id + created_at + id + updated_at + ].freeze + + private + def records + Assignment.where(account: account) + end + + def export_record(assignment) + zip.add_file "data/assignments/#{assignment.id}.json", assignment.as_json.to_json + end + + def files + zip.glob("data/assignments/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Assignment.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Assignment record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/blob_file_record_set.rb b/app/models/account/data_transfer/blob_file_record_set.rb new file mode 100644 index 000000000..233985a0b --- /dev/null +++ b/app/models/account/data_transfer/blob_file_record_set.rb @@ -0,0 +1,39 @@ +class Account::DataTransfer::BlobFileRecordSet < Account::DataTransfer::RecordSet + private + def records + ActiveStorage::Blob.where(account: account) + end + + def export_record(blob) + zip.add_file("storage/#{blob.key}", compress: false) do |out| + blob.download { |chunk| out.write(chunk) } + end + rescue ActiveStorage::FileNotFoundError + # Skip blobs where the file is missing from storage + end + + def files + zip.glob("storage/*") + end + + def import_batch(files) + files.each do |file| + key = File.basename(file) + blob = ActiveStorage::Blob.find_by(key: key, account: account) + next unless blob + + zip.read(file) do |stream| + blob.upload(stream) + end + end + end + + def validate_record(file_path) + key = File.basename(file_path) + + unless zip.exists?("data/active_storage_blobs/#{key}.json") || ActiveStorage::Blob.exists?(key: key, account: account) + # File exists without corresponding blob record - could be orphaned or blob not yet imported + # We allow this since blob metadata is imported before files + end + end +end diff --git a/app/models/account/data_transfer/board_publication_record_set.rb b/app/models/account/data_transfer/board_publication_record_set.rb new file mode 100644 index 000000000..be357da77 --- /dev/null +++ b/app/models/account/data_transfer/board_publication_record_set.rb @@ -0,0 +1,46 @@ +class Account::DataTransfer::BoardPublicationRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + board_id + created_at + id + key + updated_at + ].freeze + + private + def records + Board::Publication.where(account: account) + end + + def export_record(publication) + zip.add_file "data/board_publications/#{publication.id}.json", publication.as_json.to_json + end + + def files + zip.glob("data/board_publications/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Board::Publication.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "BoardPublication record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/board_record_set.rb b/app/models/account/data_transfer/board_record_set.rb new file mode 100644 index 000000000..4b678cb4c --- /dev/null +++ b/app/models/account/data_transfer/board_record_set.rb @@ -0,0 +1,47 @@ +class Account::DataTransfer::BoardRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + all_access + created_at + creator_id + id + name + updated_at + ].freeze + + private + def records + Board.where(account: account) + end + + def export_record(board) + zip.add_file "data/boards/#{board.id}.json", board.as_json.to_json + end + + def files + zip.glob("data/boards/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Board.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Board record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/card_activity_spike_record_set.rb b/app/models/account/data_transfer/card_activity_spike_record_set.rb new file mode 100644 index 000000000..ff8e1425e --- /dev/null +++ b/app/models/account/data_transfer/card_activity_spike_record_set.rb @@ -0,0 +1,45 @@ +class Account::DataTransfer::CardActivitySpikeRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + card_id + created_at + id + updated_at + ].freeze + + private + def records + Card::ActivitySpike.where(account: account) + end + + def export_record(activity_spike) + zip.add_file "data/card_activity_spikes/#{activity_spike.id}.json", activity_spike.as_json.to_json + end + + def files + zip.glob("data/card_activity_spikes/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Card::ActivitySpike.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "CardActivitySpike record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/card_goldness_record_set.rb b/app/models/account/data_transfer/card_goldness_record_set.rb new file mode 100644 index 000000000..ecd871399 --- /dev/null +++ b/app/models/account/data_transfer/card_goldness_record_set.rb @@ -0,0 +1,45 @@ +class Account::DataTransfer::CardGoldnessRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + card_id + created_at + id + updated_at + ].freeze + + private + def records + Card::Goldness.where(account: account) + end + + def export_record(goldness) + zip.add_file "data/card_goldnesses/#{goldness.id}.json", goldness.as_json.to_json + end + + def files + zip.glob("data/card_goldnesses/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Card::Goldness.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "CardGoldness record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/card_not_now_record_set.rb b/app/models/account/data_transfer/card_not_now_record_set.rb new file mode 100644 index 000000000..0d26b1405 --- /dev/null +++ b/app/models/account/data_transfer/card_not_now_record_set.rb @@ -0,0 +1,46 @@ +class Account::DataTransfer::CardNotNowRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + card_id + created_at + id + updated_at + user_id + ].freeze + + private + def records + Card::NotNow.where(account: account) + end + + def export_record(not_now) + zip.add_file "data/card_not_nows/#{not_now.id}.json", not_now.as_json.to_json + end + + def files + zip.glob("data/card_not_nows/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Card::NotNow.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "CardNotNow record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/card_record_set.rb b/app/models/account/data_transfer/card_record_set.rb new file mode 100644 index 000000000..15299ffd9 --- /dev/null +++ b/app/models/account/data_transfer/card_record_set.rb @@ -0,0 +1,52 @@ +class Account::DataTransfer::CardRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + board_id + column_id + created_at + creator_id + due_on + id + last_active_at + number + status + title + updated_at + ].freeze + + private + def records + Card.where(account: account) + end + + def export_record(card) + zip.add_file "data/cards/#{card.id}.json", card.as_json.to_json + end + + def files + zip.glob("data/cards/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Card.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Card record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/closure_record_set.rb b/app/models/account/data_transfer/closure_record_set.rb new file mode 100644 index 000000000..6bfc1a2d5 --- /dev/null +++ b/app/models/account/data_transfer/closure_record_set.rb @@ -0,0 +1,46 @@ +class Account::DataTransfer::ClosureRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + card_id + created_at + id + updated_at + user_id + ].freeze + + private + def records + Closure.where(account: account) + end + + def export_record(closure) + zip.add_file "data/closures/#{closure.id}.json", closure.as_json.to_json + end + + def files + zip.glob("data/closures/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Closure.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Closure record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/column_record_set.rb b/app/models/account/data_transfer/column_record_set.rb new file mode 100644 index 000000000..fb5819bbd --- /dev/null +++ b/app/models/account/data_transfer/column_record_set.rb @@ -0,0 +1,48 @@ +class Account::DataTransfer::ColumnRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + board_id + color + created_at + id + name + position + updated_at + ].freeze + + private + def records + Column.where(account: account) + end + + def export_record(column) + zip.add_file "data/columns/#{column.id}.json", column.as_json.to_json + end + + def files + zip.glob("data/columns/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Column.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Column record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/comment_record_set.rb b/app/models/account/data_transfer/comment_record_set.rb new file mode 100644 index 000000000..61891794b --- /dev/null +++ b/app/models/account/data_transfer/comment_record_set.rb @@ -0,0 +1,46 @@ +class Account::DataTransfer::CommentRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + card_id + created_at + creator_id + id + updated_at + ].freeze + + private + def records + Comment.where(account: account) + end + + def export_record(comment) + zip.add_file "data/comments/#{comment.id}.json", comment.as_json.to_json + end + + def files + zip.glob("data/comments/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Comment.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Comment record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/entropy_record_set.rb b/app/models/account/data_transfer/entropy_record_set.rb new file mode 100644 index 000000000..550f52f85 --- /dev/null +++ b/app/models/account/data_transfer/entropy_record_set.rb @@ -0,0 +1,51 @@ +class Account::DataTransfer::EntropyRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + auto_postpone_period + container_id + container_type + created_at + id + updated_at + ].freeze + + private + def records + Entropy.where(account: account) + end + + def export_record(entropy) + zip.add_file "data/entropies/#{entropy.id}.json", entropy.as_json.to_json + end + + def files + zip.glob("data/entropies/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Entropy.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Entropy record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + + unless %w[Account Board].include?(data["container_type"]) + raise IntegrityError, "#{file_path} has invalid container_type: #{data['container_type']}" + end + end +end diff --git a/app/models/account/data_transfer/event_record_set.rb b/app/models/account/data_transfer/event_record_set.rb new file mode 100644 index 000000000..77ec845f1 --- /dev/null +++ b/app/models/account/data_transfer/event_record_set.rb @@ -0,0 +1,50 @@ +class Account::DataTransfer::EventRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + action + board_id + created_at + creator_id + eventable_id + eventable_type + id + particulars + updated_at + ].freeze + + private + def records + Event.where(account: account) + end + + def export_record(event) + zip.add_file "data/events/#{event.id}.json", event.as_json.to_json + end + + def files + zip.glob("data/events/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Event.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Event record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/filter_record_set.rb b/app/models/account/data_transfer/filter_record_set.rb new file mode 100644 index 000000000..26e9b6dab --- /dev/null +++ b/app/models/account/data_transfer/filter_record_set.rb @@ -0,0 +1,47 @@ +class Account::DataTransfer::FilterRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + created_at + creator_id + fields + id + params_digest + updated_at + ].freeze + + private + def records + Filter.where(account: account) + end + + def export_record(filter) + zip.add_file "data/filters/#{filter.id}.json", filter.as_json.to_json + end + + def files + zip.glob("data/filters/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Filter.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Filter record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/manifest.rb b/app/models/account/data_transfer/manifest.rb new file mode 100644 index 000000000..91ce8028e --- /dev/null +++ b/app/models/account/data_transfer/manifest.rb @@ -0,0 +1,54 @@ +class Account::DataTransfer::Manifest + Cursor = Struct.new(:record_class, :last_id) + + include Enumerable + + RECORD_SETS = [ + Account::DataTransfer::AccountRecordSet, + Account::DataTransfer::UserRecordSet, + Account::DataTransfer::TagRecordSet, + Account::DataTransfer::BoardRecordSet, + Account::DataTransfer::ColumnRecordSet, + Account::DataTransfer::EntropyRecordSet, + Account::DataTransfer::BoardPublicationRecordSet, + Account::DataTransfer::WebhookRecordSet, + Account::DataTransfer::AccessRecordSet, + Account::DataTransfer::CardRecordSet, + Account::DataTransfer::CommentRecordSet, + Account::DataTransfer::StepRecordSet, + Account::DataTransfer::AssignmentRecordSet, + Account::DataTransfer::TaggingRecordSet, + Account::DataTransfer::ClosureRecordSet, + Account::DataTransfer::CardGoldnessRecordSet, + Account::DataTransfer::CardNotNowRecordSet, + Account::DataTransfer::CardActivitySpikeRecordSet, + Account::DataTransfer::WatchRecordSet, + Account::DataTransfer::PinRecordSet, + Account::DataTransfer::ReactionRecordSet, + Account::DataTransfer::MentionRecordSet, + Account::DataTransfer::FilterRecordSet, + Account::DataTransfer::WebhookDelinquencyTrackerRecordSet, + Account::DataTransfer::EventRecordSet, + Account::DataTransfer::NotificationRecordSet, + Account::DataTransfer::NotificationBundleRecordSet, + Account::DataTransfer::WebhookDeliveryRecordSet, + Account::DataTransfer::ActiveStorageBlobRecordSet, + Account::DataTransfer::ActiveStorageAttachmentRecordSet, + Account::DataTransfer::ActionTextRichTextRecordSet, + Account::DataTransfer::BlobFileRecordSet + ] + + attr_reader :account + + def initialize(account) + @account = account + end + + def each_record_set(start: nil) + raise ArgumentError, "No block given" unless block_given? + + RECORD_SETS.each do |record_set_class| + yield record_set_class.new(account) + end + end +end diff --git a/app/models/account/data_transfer/mention_record_set.rb b/app/models/account/data_transfer/mention_record_set.rb new file mode 100644 index 000000000..4067418e5 --- /dev/null +++ b/app/models/account/data_transfer/mention_record_set.rb @@ -0,0 +1,54 @@ +class Account::DataTransfer::MentionRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + created_at + id + mentionee_id + mentioner_id + source_id + source_type + updated_at + ].freeze + + VALID_SOURCE_TYPES = %w[Card Comment].freeze + + private + def records + Mention.where(account: account) + end + + def export_record(mention) + zip.add_file "data/mentions/#{mention.id}.json", mention.as_json.to_json + end + + def files + zip.glob("data/mentions/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Mention.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Mention record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + + unless VALID_SOURCE_TYPES.include?(data["source_type"]) + raise IntegrityError, "#{file_path} has invalid source_type: #{data['source_type']}" + end + end +end diff --git a/app/models/account/data_transfer/notification_bundle_record_set.rb b/app/models/account/data_transfer/notification_bundle_record_set.rb new file mode 100644 index 000000000..6dee00286 --- /dev/null +++ b/app/models/account/data_transfer/notification_bundle_record_set.rb @@ -0,0 +1,48 @@ +class Account::DataTransfer::NotificationBundleRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + created_at + ends_at + id + starts_at + status + updated_at + user_id + ].freeze + + private + def records + Notification::Bundle.where(account: account) + end + + def export_record(bundle) + zip.add_file "data/notification_bundles/#{bundle.id}.json", bundle.as_json.to_json + end + + def files + zip.glob("data/notification_bundles/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Notification::Bundle.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "NotificationBundle record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/notification_record_set.rb b/app/models/account/data_transfer/notification_record_set.rb new file mode 100644 index 000000000..da08f8be0 --- /dev/null +++ b/app/models/account/data_transfer/notification_record_set.rb @@ -0,0 +1,49 @@ +class Account::DataTransfer::NotificationRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + created_at + creator_id + id + read_at + source_id + source_type + updated_at + user_id + ].freeze + + private + def records + Notification.where(account: account) + end + + def export_record(notification) + zip.add_file "data/notifications/#{notification.id}.json", notification.as_json.to_json + end + + def files + zip.glob("data/notifications/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Notification.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Notification record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/pin_record_set.rb b/app/models/account/data_transfer/pin_record_set.rb new file mode 100644 index 000000000..1244fbe3c --- /dev/null +++ b/app/models/account/data_transfer/pin_record_set.rb @@ -0,0 +1,46 @@ +class Account::DataTransfer::PinRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + card_id + created_at + id + updated_at + user_id + ].freeze + + private + def records + Pin.where(account: account) + end + + def export_record(pin) + zip.add_file "data/pins/#{pin.id}.json", pin.as_json.to_json + end + + def files + zip.glob("data/pins/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Pin.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Pin record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/reaction_record_set.rb b/app/models/account/data_transfer/reaction_record_set.rb new file mode 100644 index 000000000..8060f57fe --- /dev/null +++ b/app/models/account/data_transfer/reaction_record_set.rb @@ -0,0 +1,47 @@ +class Account::DataTransfer::ReactionRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + comment_id + content + created_at + id + reacter_id + updated_at + ].freeze + + private + def records + Reaction.where(account: account) + end + + def export_record(reaction) + zip.add_file "data/reactions/#{reaction.id}.json", reaction.as_json.to_json + end + + def files + zip.glob("data/reactions/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Reaction.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Reaction record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/record_set.rb b/app/models/account/data_transfer/record_set.rb new file mode 100644 index 000000000..ff8a7d575 --- /dev/null +++ b/app/models/account/data_transfer/record_set.rb @@ -0,0 +1,76 @@ +class Account::DataTransfer::RecordSet + class IntegrityError < StandardError; end + + IMPORT_BATCH_SIZE = 100 + + attr_reader :account + + def initialize(account) + @account = account + end + + def export(to:, start: nil) + with_zip(to) do + block = lambda do |record| + export_record(record) + end + + records.respond_to?(:find_each) ? records.find_each(&block) : records.each(&block) + end + end + + def import(from:, start: nil, callback: nil) + with_zip(from) do + files.each_slice(IMPORT_BATCH_SIZE) do |file_batch| + import_batch(file_batch) + callback&.call(record_set: self, files: file_batch) + end + end + end + + def validate(from:, start: nil, callback: nil) + with_zip(from) do + files.each do |file_path| + validate_record(file_path) + callback&.call(record_set: self, file: file_path) + end + end + end + + private + attr_reader :zip + + def with_zip(zip) + old_zip = @zip + @zip = zip + yield + ensure + @zip = old_zip + end + + def records + [] + end + + def export_record(record) + raise NotImplementedError + end + + def files + [] + end + + def import_batch(files) + raise NotImplementedError + end + + def validate_record(file_path) + raise NotImplementedError + end + + def load(file_path) + JSON.parse(zip.read(file_path)) + rescue ArgumentError => e + raise IntegrityError, e.message + end +end diff --git a/app/models/account/data_transfer/step_record_set.rb b/app/models/account/data_transfer/step_record_set.rb new file mode 100644 index 000000000..b3f0acea7 --- /dev/null +++ b/app/models/account/data_transfer/step_record_set.rb @@ -0,0 +1,47 @@ +class Account::DataTransfer::StepRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + card_id + completed + content + created_at + id + updated_at + ].freeze + + private + def records + Step.where(account: account) + end + + def export_record(step) + zip.add_file "data/steps/#{step.id}.json", step.as_json.to_json + end + + def files + zip.glob("data/steps/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Step.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Step record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/tag_record_set.rb b/app/models/account/data_transfer/tag_record_set.rb new file mode 100644 index 000000000..6f4f72e08 --- /dev/null +++ b/app/models/account/data_transfer/tag_record_set.rb @@ -0,0 +1,45 @@ +class Account::DataTransfer::TagRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + created_at + id + title + updated_at + ].freeze + + private + def records + Tag.where(account: account) + end + + def export_record(tag) + zip.add_file "data/tags/#{tag.id}.json", tag.as_json.to_json + end + + def files + zip.glob("data/tags/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Tag.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Tag record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/tagging_record_set.rb b/app/models/account/data_transfer/tagging_record_set.rb new file mode 100644 index 000000000..e9e65f7b5 --- /dev/null +++ b/app/models/account/data_transfer/tagging_record_set.rb @@ -0,0 +1,46 @@ +class Account::DataTransfer::TaggingRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + card_id + created_at + id + tag_id + updated_at + ].freeze + + private + def records + Tagging.where(account: account) + end + + def export_record(tagging) + zip.add_file "data/taggings/#{tagging.id}.json", tagging.as_json.to_json + end + + def files + zip.glob("data/taggings/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Tagging.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Tagging record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/user_record_set.rb b/app/models/account/data_transfer/user_record_set.rb new file mode 100644 index 000000000..c77e7bbec --- /dev/null +++ b/app/models/account/data_transfer/user_record_set.rb @@ -0,0 +1,54 @@ +class Account::DataTransfer::UserRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + id + email_address + name + role + active + verified_at + created_at + updated_at + ] + + private + def records + User.where(account: account) + end + + def export_record(user) + zip.add_file "data/users/#{user.id}.json", user.as_json.merge(email_address: user.identity&.email_address).to_json + end + + def files + zip.glob("data/users/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + user_data = load(file) + email_address = user_data.delete("email_address") + + identity = Identity.find_or_create_by!(email_address: email_address) if email_address.present? + + user_data.slice(*ATTRIBUTES).merge( + "account_id" => account.id, + "identity_id" => identity&.id + ) + end + + User.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "User record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + unless (ATTRIBUTES - data.keys).empty? + raise IntegrityError, "#{file_path} is missing required fields" + end + end +end diff --git a/app/models/account/data_transfer/watch_record_set.rb b/app/models/account/data_transfer/watch_record_set.rb new file mode 100644 index 000000000..497f8e4ce --- /dev/null +++ b/app/models/account/data_transfer/watch_record_set.rb @@ -0,0 +1,47 @@ +class Account::DataTransfer::WatchRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + card_id + created_at + id + updated_at + user_id + watching + ].freeze + + private + def records + Watch.where(account: account) + end + + def export_record(watch) + zip.add_file "data/watches/#{watch.id}.json", watch.as_json.to_json + end + + def files + zip.glob("data/watches/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Watch.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Watch record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/webhook_delinquency_tracker_record_set.rb b/app/models/account/data_transfer/webhook_delinquency_tracker_record_set.rb new file mode 100644 index 000000000..15f35c6e5 --- /dev/null +++ b/app/models/account/data_transfer/webhook_delinquency_tracker_record_set.rb @@ -0,0 +1,47 @@ +class Account::DataTransfer::WebhookDelinquencyTrackerRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + consecutive_failures_count + created_at + first_failure_at + id + updated_at + webhook_id + ].freeze + + private + def records + Webhook::DelinquencyTracker.where(account: account) + end + + def export_record(tracker) + zip.add_file "data/webhook_delinquency_trackers/#{tracker.id}.json", tracker.as_json.to_json + end + + def files + zip.glob("data/webhook_delinquency_trackers/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Webhook::DelinquencyTracker.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "WebhookDelinquencyTracker record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/webhook_delivery_record_set.rb b/app/models/account/data_transfer/webhook_delivery_record_set.rb new file mode 100644 index 000000000..2566019e0 --- /dev/null +++ b/app/models/account/data_transfer/webhook_delivery_record_set.rb @@ -0,0 +1,49 @@ +class Account::DataTransfer::WebhookDeliveryRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + created_at + event_id + id + request + response + state + updated_at + webhook_id + ].freeze + + private + def records + Webhook::Delivery.where(account: account) + end + + def export_record(delivery) + zip.add_file "data/webhook_deliveries/#{delivery.id}.json", delivery.as_json.to_json + end + + def files + zip.glob("data/webhook_deliveries/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Webhook::Delivery.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "WebhookDelivery record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/webhook_record_set.rb b/app/models/account/data_transfer/webhook_record_set.rb new file mode 100644 index 000000000..329050343 --- /dev/null +++ b/app/models/account/data_transfer/webhook_record_set.rb @@ -0,0 +1,50 @@ +class Account::DataTransfer::WebhookRecordSet < Account::DataTransfer::RecordSet + ATTRIBUTES = %w[ + account_id + active + board_id + created_at + id + name + signing_secret + subscribed_actions + updated_at + url + ].freeze + + private + def records + Webhook.where(account: account) + end + + def export_record(webhook) + zip.add_file "data/webhooks/#{webhook.id}.json", webhook.as_json.to_json + end + + def files + zip.glob("data/webhooks/*.json") + end + + def import_batch(files) + batch_data = files.map do |file| + data = load(file) + data.slice(*ATTRIBUTES).merge("account_id" => account.id) + end + + Webhook.insert_all!(batch_data) + end + + def validate_record(file_path) + data = load(file_path) + expected_id = File.basename(file_path, ".json") + + unless data["id"].to_s == expected_id + raise IntegrityError, "Webhook record ID mismatch: expected #{expected_id}, got #{data['id']}" + end + + missing = ATTRIBUTES - data.keys + unless missing.empty? + raise IntegrityError, "#{file_path} is missing required fields: #{missing.join(', ')}" + end + end +end diff --git a/app/models/account/data_transfer/zip_file.rb b/app/models/account/data_transfer/zip_file.rb new file mode 100644 index 000000000..058476625 --- /dev/null +++ b/app/models/account/data_transfer/zip_file.rb @@ -0,0 +1,57 @@ +class Account::DataTransfer::ZipFile + class << self + def create + raise ArgumentError, "No block given" unless block_given? + + Tempfile.new([ "export", ".zip" ]).tap do |tempfile| + Zip::File.open(tempfile.path, create: true) do |zip| + yield new(zip) + end + end + end + + def open(path) + raise ArgumentError, "No block given" unless block_given? + + Zip::File.open(path.to_s) do |zip| + yield new(zip) + end + end + end + + def initialize(zip) + @zip = zip + end + + def add_file(path, content = nil, compress: true, &block) + if block_given? + compression = compress ? nil : Zip::Entry::STORED + zip.get_output_stream(path, nil, nil, compression, &block) + else + zip.get_output_stream(path) { |f| f.write(content) } + end + end + + def glob(pattern) + zip.glob(pattern).map(&:name).sort + end + + def read(file_path, &block) + entry = zip.find_entry(file_path) + raise ArgumentError, "File not found in zip: #{file_path}" unless entry + raise ArgumentError, "Cannot read directory entry: #{file_path}" if entry.directory? + + if block_given? + yield entry.get_input_stream + else + entry.get_input_stream.read + end + end + + def exists?(file_path) + zip.find_entry(file_path).present? + end + + private + attr_reader :zip +end diff --git a/app/models/account/export.rb b/app/models/account/export.rb index e8ab0dd1f..6d7851324 100644 --- a/app/models/account/export.rb +++ b/app/models/account/export.rb @@ -1,254 +1,8 @@ class Account::Export < Export private def populate_zip(zip) - export_account(zip) - export_users(zip) - export_tags(zip) - export_entropies(zip) - export_columns(zip) - export_board_publications(zip) - export_webhooks(zip) - export_webhook_delinquency_trackers(zip) - export_accesses(zip) - export_assignments(zip) - export_taggings(zip) - export_steps(zip) - export_closures(zip) - export_card_goldnesses(zip) - export_card_not_nows(zip) - export_card_activity_spikes(zip) - export_watches(zip) - export_pins(zip) - export_reactions(zip) - export_mentions(zip) - export_filters(zip) - export_events(zip) - export_notifications(zip) - export_notification_bundles(zip) - export_webhook_deliveries(zip) - - export_boards(zip) - export_cards(zip) - export_comments(zip) - - export_action_text_rich_texts(zip) - export_active_storage_attachments(zip) - export_active_storage_blobs(zip) - - export_blob_files(zip) - end - - def export_account(zip) - data = account.as_json.merge(join_code: account.join_code.as_json) - add_file_to_zip(zip, "data/account.json", JSON.pretty_generate(data)) - end - - def export_users(zip) - account.users.find_each do |user| - data = user.as_json.except("identity_id").merge( - "email_address" => user.identity&.email_address - ) - add_file_to_zip(zip, "data/users/#{user.id}.json", JSON.pretty_generate(data)) - end - end - - def export_tags(zip) - account.tags.find_each do |tag| - add_file_to_zip(zip, "data/tags/#{tag.id}.json", JSON.pretty_generate(tag.as_json)) - end - end - - def export_entropies(zip) - Entropy.where(account: account).find_each do |entropy| - add_file_to_zip(zip, "data/entropies/#{entropy.id}.json", JSON.pretty_generate(entropy.as_json)) - end - end - - def export_columns(zip) - account.columns.find_each do |column| - add_file_to_zip(zip, "data/columns/#{column.id}.json", JSON.pretty_generate(column.as_json)) - end - end - - def export_board_publications(zip) - Board::Publication.where(account: account).find_each do |publication| - add_file_to_zip(zip, "data/board_publications/#{publication.id}.json", JSON.pretty_generate(publication.as_json)) - end - end - - def export_webhooks(zip) - Webhook.where(account: account).find_each do |webhook| - add_file_to_zip(zip, "data/webhooks/#{webhook.id}.json", JSON.pretty_generate(webhook.as_json)) - end - end - - def export_webhook_delinquency_trackers(zip) - Webhook::DelinquencyTracker.joins(:webhook).where(webhook: { account: account }).find_each do |tracker| - add_file_to_zip(zip, "data/webhook_delinquency_trackers/#{tracker.id}.json", JSON.pretty_generate(tracker.as_json)) - end - end - - def export_accesses(zip) - Access.where(account: account).find_each do |access| - add_file_to_zip(zip, "data/accesses/#{access.id}.json", JSON.pretty_generate(access.as_json)) - end - end - - def export_assignments(zip) - Assignment.where(account: account).find_each do |assignment| - add_file_to_zip(zip, "data/assignments/#{assignment.id}.json", JSON.pretty_generate(assignment.as_json)) - end - end - - def export_taggings(zip) - Tagging.where(account: account).find_each do |tagging| - add_file_to_zip(zip, "data/taggings/#{tagging.id}.json", JSON.pretty_generate(tagging.as_json)) - end - end - - def export_steps(zip) - Step.where(account: account).find_each do |step| - add_file_to_zip(zip, "data/steps/#{step.id}.json", JSON.pretty_generate(step.as_json)) - end - end - - def export_closures(zip) - Closure.where(account: account).find_each do |closure| - add_file_to_zip(zip, "data/closures/#{closure.id}.json", JSON.pretty_generate(closure.as_json)) - end - end - - def export_card_goldnesses(zip) - Card::Goldness.where(account: account).find_each do |goldness| - add_file_to_zip(zip, "data/card_goldnesses/#{goldness.id}.json", JSON.pretty_generate(goldness.as_json)) - end - end - - def export_card_not_nows(zip) - Card::NotNow.where(account: account).find_each do |not_now| - add_file_to_zip(zip, "data/card_not_nows/#{not_now.id}.json", JSON.pretty_generate(not_now.as_json)) - end - end - - def export_card_activity_spikes(zip) - Card::ActivitySpike.where(account: account).find_each do |activity_spike| - add_file_to_zip(zip, "data/card_activity_spikes/#{activity_spike.id}.json", JSON.pretty_generate(activity_spike.as_json)) - end - end - - def export_watches(zip) - Watch.where(account: account).find_each do |watch| - add_file_to_zip(zip, "data/watches/#{watch.id}.json", JSON.pretty_generate(watch.as_json)) - end - end - - def export_pins(zip) - Pin.where(account: account).find_each do |pin| - add_file_to_zip(zip, "data/pins/#{pin.id}.json", JSON.pretty_generate(pin.as_json)) - end - end - - def export_reactions(zip) - Reaction.where(account: account).find_each do |reaction| - add_file_to_zip(zip, "data/reactions/#{reaction.id}.json", JSON.pretty_generate(reaction.as_json)) - end - end - - def export_mentions(zip) - Mention.where(account: account).find_each do |mention| - add_file_to_zip(zip, "data/mentions/#{mention.id}.json", JSON.pretty_generate(mention.as_json)) - end - end - - def export_filters(zip) - Filter.where(account: account).find_each do |filter| - add_file_to_zip(zip, "data/filters/#{filter.id}.json", JSON.pretty_generate(filter.as_json)) - end - end - - def export_events(zip) - Event.where(account: account).find_each do |event| - add_file_to_zip(zip, "data/events/#{event.id}.json", JSON.pretty_generate(event.as_json)) - end - end - - def export_notifications(zip) - Notification.where(account: account).find_each do |notification| - add_file_to_zip(zip, "data/notifications/#{notification.id}.json", JSON.pretty_generate(notification.as_json)) - end - end - - def export_notification_bundles(zip) - Notification::Bundle.where(account: account).find_each do |bundle| - add_file_to_zip(zip, "data/notification_bundles/#{bundle.id}.json", JSON.pretty_generate(bundle.as_json)) - end - end - - def export_webhook_deliveries(zip) - Webhook::Delivery.where(account: account).find_each do |delivery| - add_file_to_zip(zip, "data/webhook_deliveries/#{delivery.id}.json", JSON.pretty_generate(delivery.as_json)) - end - end - - def export_boards(zip) - account.boards.find_each do |board| - add_file_to_zip(zip, "data/boards/#{board.id}.json", JSON.pretty_generate(board.as_json)) - end - end - - def export_cards(zip) - account.cards.find_each do |card| - add_file_to_zip(zip, "data/cards/#{card.id}.json", JSON.pretty_generate(card.as_json)) - end - end - - def export_comments(zip) - Comment.where(account: account).find_each do |comment| - add_file_to_zip(zip, "data/comments/#{comment.id}.json", JSON.pretty_generate(comment.as_json)) - end - end - - def export_action_text_rich_texts(zip) - ActionText::RichText.where(account: account).find_each do |rich_text| - data = rich_text.as_json.merge("body" => convert_sgids_to_gids(rich_text.body)) - add_file_to_zip(zip, "data/action_text_rich_texts/#{rich_text.id}.json", JSON.pretty_generate(data)) - end - end - - def convert_sgids_to_gids(content) - return nil if content.blank? - - content.send(:attachment_nodes).each do |node| - sgid = SignedGlobalID.parse(node["sgid"], for: ActionText::Attachable::LOCATOR_NAME) - record = sgid&.find - next if record&.account_id != account.id - - node["gid"] = record.to_global_id.to_s - node.remove_attribute("sgid") - end - - content.fragment.source.to_html - end - - def export_active_storage_attachments(zip) - ActiveStorage::Attachment.where(account: account).find_each do |attachment| - add_file_to_zip(zip, "data/active_storage_attachments/#{attachment.id}.json", JSON.pretty_generate(attachment.as_json)) - end - end - - def export_active_storage_blobs(zip) - ActiveStorage::Blob.where(account: account).find_each do |blob| - add_file_to_zip(zip, "data/active_storage_blobs/#{blob.id}.json", JSON.pretty_generate(blob.as_json)) - end - end - - def export_blob_files(zip) - ActiveStorage::Blob.where(account: account).find_each do |blob| - add_file_to_zip(zip, "storage/#{blob.key}", compression_method: Zip::Entry::STORED) do |f| - blob.download { |chunk| f.write(chunk) } - end - rescue ActiveStorage::FileNotFoundError - # Skip blobs where the file is missing from storage + Account::DataTransfer::Manifest.new(account).each_record_set do |record_set| + record_set.export(to: Account::DataTransfer::ZipFile.new(zip)) end end end diff --git a/app/models/account/import.rb b/app/models/account/import.rb index 2357fbd5f..a68a1b37b 100644 --- a/app/models/account/import.rb +++ b/app/models/account/import.rb @@ -1,6 +1,4 @@ class Account::Import < ApplicationRecord - class IntegrityError < StandardError; end - belongs_to :account belongs_to :identity @@ -8,738 +6,51 @@ class Account::Import < ApplicationRecord enum :status, %w[ pending processing completed failed ].index_by(&:itself), default: :pending - def perform_later - ImportAccountDataJob.perform_later(self) + def process_later end - def perform + def process(start: nil, callback: nil) processing! + ensure_downloaded - Current.set(account: account, user: owner_user) do - file.open do |tempfile| - Zip::File.open(tempfile.path) do |zip| - ApplicationRecord.transaction do - @old_account_data = load_account_data(zip) - @id_mapper = IdMapper.new(account, @old_account_data) - - validate_export_integrity!(zip) - import_all(zip) - end - end + Account::DataTransfer::ZipFile.open(download_path) do |zip| + Account::DataTransfer::Manifest.new(account).each_record_set(start: start&.record_set) do |record_set| + record_set.import(from: zip, start: start&.record_id, callback: callback) end end mark_completed rescue => e - mark_failed - raise + failed! + raise e end - def owner_user - account.users.find_by(identity: identity) + def validate(start: nil, callback: nil) + processing! + ensure_downloaded + + Account::DataTransfer::ZipFile.open(download_path) do |zip| + Account::DataTransfer::Manifest.new(account).each_record_set(start: start&.record_set) do |record_set| + record_set.validate(from: zip, start: start&.record_id, callback: callback) + end + end end private + def ensure_downloaded + unless download_path.exist? + download_path.open("wb") do |f| + file.download { |chunk| f.write(chunk) } + end + end + end + + def download_path + Pathname.new("/tmp/account-import-#{id}.zip") + end + def mark_completed - update!(status: :completed, completed_at: Time.current) - ImportMailer.completed(identity).deliver_later - end - - def mark_failed - update!(status: :failed) - ImportMailer.failed(identity).deliver_later - end - - def load_account_data(zip) - entry = zip.find_entry("data/account.json") - raise IntegrityError, "Missing account.json in export" unless entry - - JSON.parse(entry.get_input_stream.read) - end - - def import_all(zip) - # Phase 1: Foundation - import_account_data - import_join_code - - # Phase 2: Users (create identities, map system user) - import_users(zip) - - # Phase 3: Basic entities - import_tags(zip) - import_boards(zip) - import_columns(zip) - import_entropies(zip) - import_board_publications(zip) - - # Phase 4: Cards & content - import_cards(zip) - import_comments(zip) - import_steps(zip) - - # Phase 5: Relationships - import_accesses(zip) - import_assignments(zip) - import_taggings(zip) - import_closures(zip) - import_card_goldnesses(zip) - import_card_not_nows(zip) - import_card_activity_spikes(zip) - import_watches(zip) - import_pins(zip) - import_reactions(zip) - import_mentions(zip) - import_filters(zip) - - # Phase 6: Webhooks - import_webhooks(zip) - import_webhook_delinquency_trackers(zip) - import_webhook_deliveries(zip) - - # Phase 7: Activity & notifications - import_events(zip) - import_notifications(zip) - import_notification_bundles(zip) - - # Phase 8: Storage & rich text - import_active_storage_blobs(zip) - import_active_storage_attachments(zip) - import_action_text_rich_texts(zip) - import_blob_files(zip) - end - - # Phase 1: Foundation - - def import_account_data - account.update!(name: @old_account_data["name"]) - end - - def import_join_code - join_code_data = @old_account_data["join_code"] - return unless join_code_data - - # Preserve the code if it's unique, otherwise keep the auto-generated one - unless Account::JoinCode.exists?(code: join_code_data["code"]) - account.join_code.update!( - code: join_code_data["code"], - usage_count: join_code_data["usage_count"], - usage_limit: join_code_data["usage_limit"] - ) - end - end - - # Phase 2: Users - - def import_users(zip) - users_data = read_json_files(zip, "data/users") - - # Map system user first - old_system = users_data.find { |u| u["role"] == "system" } - if old_system - @id_mapper.map(:users, old_system["id"], account.system_user.id) - end - - # Import non-system users - users_data.reject { |u| u["role"] == "system" }.each do |data| - import_user(data) - end - end - - def import_user(data) - email = data.delete("email_address") - old_id = data.delete("id") - - user_identity = if email.present? - Identity.find_or_create_by!(email_address: email) - end - - # Check if user already exists for this identity in this account (e.g., the owner) - existing_user = account.users.find_by(identity: user_identity) if user_identity - if existing_user - existing_user.update!(data.slice("name", "role", "active", "verified_at")) - @id_mapper.map(:users, old_id, existing_user.id) - else - new_user = User.create!( - data.slice(*User.column_names).merge( - "account_id" => account.id, - "identity_id" => user_identity&.id - ) - ) - @id_mapper.map(:users, old_id, new_user.id) - end - end - - # Phase 3: Basic entities - - def import_tags(zip) - records = read_json_files(zip, "data/tags").map do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data) - new_record = Tag.create!(data) - @id_mapper.map(:tags, old_id, new_record.id) - end - end - - def import_boards(zip) - read_json_files(zip, "data/boards").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data) - new_record = Board.create!(data) - @id_mapper.map(:boards, old_id, new_record.id) - end - end - - def import_columns(zip) - read_json_files(zip, "data/columns").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data, foreign_keys: { "board_id" => :boards }) - new_record = Column.create!(data) - @id_mapper.map(:columns, old_id, new_record.id) - end - end - - def import_entropies(zip) - read_json_files(zip, "data/entropies").each do |data| - old_id = data.delete("id") - - # Remap polymorphic container_id based on container_type - container_type = data["container_type"] - if container_type == "Account" - data["container_id"] = account.id - elsif container_type == "Board" - data["container_id"] = @id_mapper.lookup(:boards, data["container_id"]) - end - - data = @id_mapper.remap(data) - - # Find existing or create new - existing = Entropy.find_by(container_type: data["container_type"], container_id: data["container_id"]) - if existing - existing.update!(data.slice("auto_postpone_period")) - @id_mapper.map(:entropies, old_id, existing.id) - else - new_record = Entropy.create!(data) - @id_mapper.map(:entropies, old_id, new_record.id) - end - end - end - - def import_board_publications(zip) - read_json_files(zip, "data/board_publications").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data, foreign_keys: { "board_id" => :boards }) - new_record = Board::Publication.create!(data) - @id_mapper.map(:board_publications, old_id, new_record.id) - end - end - - # Phase 4: Cards & content - - def import_cards(zip) - read_json_files(zip, "data/cards").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "board_id" => :boards, - "column_id" => :columns - }) - new_record = Card.create!(data) - @id_mapper.map(:cards, old_id, new_record.id) - end - end - - def import_comments(zip) - read_json_files(zip, "data/comments").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "card_id" => :cards - }) - new_record = Comment.create!(data) - @id_mapper.map(:comments, old_id, new_record.id) - end - end - - def import_steps(zip) - read_json_files(zip, "data/steps").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data, foreign_keys: { "card_id" => :cards }) - new_record = Step.create!(data) - @id_mapper.map(:steps, old_id, new_record.id) - end - end - - # Phase 5: Relationships - - def import_accesses(zip) - read_json_files(zip, "data/accesses").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "board_id" => :boards - }) - - # Board creation auto-creates access for creator, check if it exists - existing = Access.find_by(board_id: data["board_id"], user_id: data["user_id"]) - if existing - @id_mapper.map(:accesses, old_id, existing.id) - else - new_record = Access.create!(data) - @id_mapper.map(:accesses, old_id, new_record.id) - end - end - end - - def import_assignments(zip) - read_json_files(zip, "data/assignments").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "card_id" => :cards - }) - new_record = Assignment.create!(data) - @id_mapper.map(:assignments, old_id, new_record.id) - end - end - - def import_taggings(zip) - read_json_files(zip, "data/taggings").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data, foreign_keys: { - "card_id" => :cards, - "tag_id" => :tags - }) - new_record = Tagging.create!(data) - @id_mapper.map(:taggings, old_id, new_record.id) - end - end - - def import_closures(zip) - read_json_files(zip, "data/closures").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "card_id" => :cards - }) - new_record = Closure.create!(data) - @id_mapper.map(:closures, old_id, new_record.id) - end - end - - def import_card_goldnesses(zip) - read_json_files(zip, "data/card_goldnesses").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data, foreign_keys: { "card_id" => :cards }) - new_record = Card::Goldness.create!(data) - @id_mapper.map(:card_goldnesses, old_id, new_record.id) - end - end - - def import_card_not_nows(zip) - read_json_files(zip, "data/card_not_nows").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "card_id" => :cards - }) - new_record = Card::NotNow.create!(data) - @id_mapper.map(:card_not_nows, old_id, new_record.id) - end - end - - def import_card_activity_spikes(zip) - read_json_files(zip, "data/card_activity_spikes").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data, foreign_keys: { "card_id" => :cards }) - new_record = Card::ActivitySpike.create!(data) - @id_mapper.map(:card_activity_spikes, old_id, new_record.id) - end - end - - def import_watches(zip) - read_json_files(zip, "data/watches").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "card_id" => :cards - }) - - # Card creation auto-creates watch for creator, check if it exists - existing = Watch.find_by(card_id: data["card_id"], user_id: data["user_id"]) - if existing - @id_mapper.map(:watches, old_id, existing.id) - else - new_record = Watch.create!(data) - @id_mapper.map(:watches, old_id, new_record.id) - end - end - end - - def import_pins(zip) - read_json_files(zip, "data/pins").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "card_id" => :cards - }) - new_record = Pin.create!(data) - @id_mapper.map(:pins, old_id, new_record.id) - end - end - - def import_reactions(zip) - read_json_files(zip, "data/reactions").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "comment_id" => :comments - }) - new_record = Reaction.create!(data) - @id_mapper.map(:reactions, old_id, new_record.id) - end - end - - def import_mentions(zip) - read_json_files(zip, "data/mentions").each do |data| - old_id = data.delete("id") - - # Remap polymorphic source_id based on source_type - source_type = data["source_type"] - source_mapping = case source_type - when "Card" then :cards - when "Comment" then :comments - end - data["source_id"] = @id_mapper.lookup(source_mapping, data["source_id"]) if source_mapping - - data = @id_mapper.remap_with_users(data) - new_record = Mention.create!(data) - @id_mapper.map(:mentions, old_id, new_record.id) - end - end - - def import_filters(zip) - read_json_files(zip, "data/filters").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data) - new_record = Filter.create!(data) - @id_mapper.map(:filters, old_id, new_record.id) - end - end - - # Phase 6: Webhooks - - def import_webhooks(zip) - read_json_files(zip, "data/webhooks").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data, foreign_keys: { "board_id" => :boards }) - new_record = Webhook.create!(data) - @id_mapper.map(:webhooks, old_id, new_record.id) - end - end - - def import_webhook_delinquency_trackers(zip) - read_json_files(zip, "data/webhook_delinquency_trackers").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data, foreign_keys: { "webhook_id" => :webhooks }) - new_record = Webhook::DelinquencyTracker.create!(data) - @id_mapper.map(:webhook_delinquency_trackers, old_id, new_record.id) - end - end - - def import_webhook_deliveries(zip) - read_json_files(zip, "data/webhook_deliveries").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data, foreign_keys: { - "webhook_id" => :webhooks, - "event_id" => :events - }) - new_record = Webhook::Delivery.create!(data) - @id_mapper.map(:webhook_deliveries, old_id, new_record.id) - end - end - - # Phase 7: Activity & notifications - - def import_events(zip) - read_json_files(zip, "data/events").each do |data| - old_id = data.delete("id") - - # Remap polymorphic eventable_id - eventable_type = data["eventable_type"] - eventable_mapping = polymorphic_type_to_mapping(eventable_type) - data["eventable_id"] = @id_mapper.lookup(eventable_mapping, data["eventable_id"]) if eventable_mapping - - data = @id_mapper.remap_with_users(data, additional_foreign_keys: { - "board_id" => :boards - }) - new_record = Event.create!(data) - @id_mapper.map(:events, old_id, new_record.id) - end - end - - def import_notifications(zip) - read_json_files(zip, "data/notifications").each do |data| - old_id = data.delete("id") - - # Remap polymorphic source_id - source_type = data["source_type"] - source_mapping = polymorphic_type_to_mapping(source_type) - data["source_id"] = @id_mapper.lookup(source_mapping, data["source_id"]) if source_mapping - - data = @id_mapper.remap_with_users(data) - new_record = Notification.create!(data) - @id_mapper.map(:notifications, old_id, new_record.id) - end - end - - def import_notification_bundles(zip) - read_json_files(zip, "data/notification_bundles").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap_with_users(data) - new_record = Notification::Bundle.create!(data) - @id_mapper.map(:notification_bundles, old_id, new_record.id) - end - end - - # Phase 8: Storage & rich text - - def import_active_storage_blobs(zip) - read_json_files(zip, "data/active_storage_blobs").each do |data| - old_id = data.delete("id") - data = @id_mapper.remap(data) - new_record = ActiveStorage::Blob.create!(data) - @id_mapper.map(:active_storage_blobs, old_id, new_record.id) - end - end - - def import_active_storage_attachments(zip) - read_json_files(zip, "data/active_storage_attachments").each do |data| - old_id = data.delete("id") - - # Remap polymorphic record_id - record_type = data["record_type"] - record_mapping = polymorphic_type_to_mapping(record_type) - data["record_id"] = @id_mapper.lookup(record_mapping, data["record_id"]) if record_mapping - - data = @id_mapper.remap(data, foreign_keys: { "blob_id" => :active_storage_blobs }) - new_record = ActiveStorage::Attachment.create!(data) - @id_mapper.map(:active_storage_attachments, old_id, new_record.id) - end - end - - def import_action_text_rich_texts(zip) - read_json_files(zip, "data/action_text_rich_texts").each do |data| - old_id = data.delete("id") - - # Remap polymorphic record_id - record_type = data["record_type"] - record_mapping = polymorphic_type_to_mapping(record_type) - data["record_id"] = @id_mapper.lookup(record_mapping, data["record_id"]) if record_mapping - - data["body"] = convert_gids_and_fix_links(data["body"]) - data = @id_mapper.remap(data) - new_record = ActionText::RichText.create!(data) - @id_mapper.map(:action_text_rich_texts, old_id, new_record.id) - end - end - - def import_blob_files(zip) - zip.glob("storage/*").each do |entry| - key = File.basename(entry.name) - blob = ActiveStorage::Blob.find_by(key: key, account: account) - next unless blob - - blob.upload(entry.get_input_stream) - end - end - - # Helper methods - - def read_json_files(zip, directory) - zip.glob("#{directory}/*.json").map do |entry| - JSON.parse(entry.get_input_stream.read) - end - end - - def convert_gids_and_fix_links(html) - return html if html.blank? - - fragment = Nokogiri::HTML.fragment(html) - - # Convert GIDs to SGIDs - fragment.css("action-text-attachment[gid]").each do |node| - gid = GlobalID.parse(node["gid"]) - next unless gid - - type = gid.model_name.plural.underscore.to_sym - new_id = @id_mapper.lookup(type, gid.model_id) - record = gid.model_class.find(new_id) - - node["sgid"] = record.attachable_sgid - node.remove_attribute("gid") - end - - # Fix links - fragment.css("a[href]").each do |link| - link["href"] = rewrite_link(link["href"]) - end - - fragment.to_html - end - - def rewrite_link(url) - uri = URI.parse(url) rescue nil - return url unless uri&.path - - path = uri.path - old_slug_pattern = %r{^/#{Regexp.escape(@id_mapper.old_account_slug)}/} - - return url unless path.match?(old_slug_pattern) - - # Replace account slug - path = path.sub(old_slug_pattern, "#{account.slug}/") - - # Try to recognize and remap IDs in the path - begin - params = Rails.application.routes.recognize_path(path) - - case params[:controller] - when "cards" - if params[:id] && @id_mapper[:cards].key?(params[:id]) - new_id = @id_mapper[:cards][params[:id]] - path = Rails.application.routes.url_helpers.card_path(new_id) - end - when "boards" - if params[:id] && @id_mapper[:boards].key?(params[:id]) - new_id = @id_mapper[:boards][params[:id]] - path = Rails.application.routes.url_helpers.board_path(new_id) - end - end - rescue ActionController::RoutingError - # Unknown route, just update the slug - end - - uri.path = path - uri.to_s - end - - def polymorphic_type_to_mapping(type) - case type - when "Card" then :cards - when "Comment" then :comments - when "Board" then :boards - when "User" then :users - when "Tag" then :tags - when "Assignment" then :assignments - when "Tagging" then :taggings - when "Closure" then :closures - when "Step" then :steps - when "Watch" then :watches - when "Pin" then :pins - when "Reaction" then :reactions - when "Mention" then :mentions - when "Event" then :events - when "Access" then :accesses - when "Webhook" then :webhooks - when "Webhook::Delivery" then :webhook_deliveries - when "Card::Goldness" then :card_goldnesses - when "Card::NotNow" then :card_not_nows - when "Card::ActivitySpike" then :card_activity_spikes - when "ActiveStorage::Blob" then :active_storage_blobs - when "ActiveStorage::Attachment" then :active_storage_attachments - when "ActionText::RichText" then :action_text_rich_texts - end - end - - # Data integrity validation - - def validate_export_integrity!(zip) - exported_ids = collect_exported_ids(zip) - - zip.glob("data/**/*.json").each do |entry| - next if entry.name == "data/account.json" - - data = JSON.parse(entry.get_input_stream.read) - validate_account_id(data, entry.name, exported_ids[:account]) - validate_foreign_keys(data, exported_ids, entry.name) - end - end - - def collect_exported_ids(zip) - ids = Hash.new { |h, k| h[k] = Set.new } - - # Account ID - ids[:account] = @old_account_data["id"] - - # Collect IDs from each entity type - entity_directories = { - "data/users" => :users, - "data/tags" => :tags, - "data/boards" => :boards, - "data/columns" => :columns, - "data/cards" => :cards, - "data/comments" => :comments, - "data/steps" => :steps, - "data/accesses" => :accesses, - "data/assignments" => :assignments, - "data/taggings" => :taggings, - "data/closures" => :closures, - "data/card_goldnesses" => :card_goldnesses, - "data/card_not_nows" => :card_not_nows, - "data/card_activity_spikes" => :card_activity_spikes, - "data/watches" => :watches, - "data/pins" => :pins, - "data/reactions" => :reactions, - "data/mentions" => :mentions, - "data/filters" => :filters, - "data/events" => :events, - "data/notifications" => :notifications, - "data/notification_bundles" => :notification_bundles, - "data/webhooks" => :webhooks, - "data/webhook_delinquency_trackers" => :webhook_delinquency_trackers, - "data/webhook_deliveries" => :webhook_deliveries, - "data/entropies" => :entropies, - "data/board_publications" => :board_publications, - "data/active_storage_blobs" => :active_storage_blobs, - "data/active_storage_attachments" => :active_storage_attachments, - "data/action_text_rich_texts" => :action_text_rich_texts - } - - entity_directories.each do |directory, type| - zip.glob("#{directory}/*.json").each do |entry| - data = JSON.parse(entry.get_input_stream.read) - ids[type].add(data["id"]) - end - end - - ids - end - - def validate_account_id(data, filename, expected_account_id) - if data["account_id"] && data["account_id"] != expected_account_id - raise IntegrityError, "#{filename} references foreign account: #{data["account_id"]}" - end - end - - FOREIGN_KEY_VALIDATIONS = { - "board_id" => :boards, - "card_id" => :cards, - "column_id" => :columns, - "user_id" => :users, - "creator_id" => :users, - "assignee_id" => :users, - "assigner_id" => :users, - "closer_id" => :users, - "mentioner_id" => :users, - "mentionee_id" => :users, - "reacter_id" => :users, - "tag_id" => :tags, - "comment_id" => :comments, - "webhook_id" => :webhooks, - "event_id" => :events, - "blob_id" => :active_storage_blobs, - "filter_id" => :filters - }.freeze - - def validate_foreign_keys(data, exported_ids, filename) - FOREIGN_KEY_VALIDATIONS.each do |field, type| - ref_id = data[field] - next unless ref_id - - unless exported_ids[type]&.include?(ref_id) - raise IntegrityError, "#{filename} references unknown #{type}: #{ref_id}" - end - end + completed! + # TODO: send email end end diff --git a/app/models/account/import/id_mapper.rb b/app/models/account/import/id_mapper.rb deleted file mode 100644 index 63e19afd0..000000000 --- a/app/models/account/import/id_mapper.rb +++ /dev/null @@ -1,60 +0,0 @@ -class Account::Import::IdMapper - attr_reader :account, :old_account_id, :old_external_account_id, :old_account_slug - - def initialize(account, old_account_data) - @account = account - @old_account_id = old_account_data["id"] - @old_external_account_id = old_account_data["external_account_id"] - @old_account_slug = AccountSlug.encode(@old_external_account_id) - @mappings = Hash.new { |h, k| h[k] = {} } - end - - def map(type, old_id, new_id) - @mappings[type][old_id] = new_id - end - - def [](type) - @mappings[type] - end - - def mapped?(type, old_id) - @mappings[type].key?(old_id) - end - - def lookup(type, old_id) - @mappings[type][old_id] || old_id - end - - # Remap account_id and specified foreign keys in a data hash - # foreign_keys is a Hash of { "field_name" => :type } - def remap(data, foreign_keys: {}) - data = data.dup - data["account_id"] = account.id if data.key?("account_id") - - foreign_keys.each do |field, type| - old_id = data[field] - next unless old_id - next unless @mappings[type].key?(old_id) - - data[field] = @mappings[type][old_id] - end - - data - end - - # Common foreign key mappings for user-related fields - USER_FOREIGN_KEYS = { - "user_id" => :users, - "creator_id" => :users, - "assignee_id" => :users, - "assigner_id" => :users, - "closer_id" => :users, - "mentioner_id" => :users, - "mentionee_id" => :users, - "reacter_id" => :users - }.freeze - - def remap_with_users(data, additional_foreign_keys: {}) - remap(data, foreign_keys: USER_FOREIGN_KEYS.merge(additional_foreign_keys)) - end -end diff --git a/app/models/export.rb b/app/models/export.rb index 871d6d8b4..9986f7539 100644 --- a/app/models/export.rb +++ b/app/models/export.rb @@ -20,18 +20,16 @@ class Export < ApplicationRecord def build processing! - Current.set(account: account) do - with_url_options do - zipfile = generate_zip { |zip| populate_zip(zip) } + with_account_context do + zipfile = generate_zip { |zip| populate_zip(zip) } - file.attach io: File.open(zipfile.path), filename: "fizzy-export-#{id}.zip", content_type: "application/zip" - mark_completed + file.attach io: File.open(zipfile.path), filename: "fizzy-export-#{id}.zip", content_type: "application/zip" + mark_completed - ExportMailer.completed(self).deliver_later - ensure - zipfile&.close - zipfile&.unlink - end + ExportMailer.completed(self).deliver_later + ensure + zipfile&.close + zipfile&.unlink end rescue => e update!(status: :failed) @@ -47,8 +45,10 @@ class Export < ApplicationRecord end private - def with_url_options - ActiveStorage::Current.set(url_options: { host: "localhost" }) { yield } + def with_account_context + Current.set(account: account) do + yield + end end def populate_zip(zip) diff --git a/test/models/account/import_test.rb b/test/models/account/import_test.rb index 6a21d8343..1abada2a3 100644 --- a/test/models/account/import_test.rb +++ b/test/models/account/import_test.rb @@ -6,63 +6,40 @@ class Account::ImportTest < ActiveSupport::TestCase @source_account = accounts("37s") end - test "perform_later enqueues ImportAccountDataJob" do + test "process sets status to failed on error" do import = create_import_with_file - - assert_enqueued_with(job: ImportAccountDataJob, args: [ import ]) do - import.perform_later - end - end - - test "perform sets status to failed on error" do - import = create_import_with_file - import.stubs(:import_all).raises(StandardError.new("Test error")) + Account::DataTransfer::Manifest.any_instance.stubs(:each_record_set).raises(StandardError.new("Test error")) assert_raises(StandardError) do - import.perform + import.process end assert import.failed? end - test "perform imports account name from export" do + test "process imports account name from export" do target_account = create_target_account import = create_import_for_account(target_account) - import.perform + import.process assert_equal @source_account.name, target_account.reload.name end - test "perform maps system user" do + test "process imports users with identity matching" do target_account = create_target_account import = create_import_for_account(target_account) + new_email = "new-user-#{SecureRandom.hex(4)}@example.com" - import.perform + import.process - # The target account should still have exactly one system user - assert_equal 1, target_account.users.where(role: :system).count + # Users from the source account should be imported + assert target_account.users.count > 2 # system user + owner + imported users end - test "perform imports users with identity matching" do + test "process preserves join code if unique" do target_account = create_target_account import = create_import_for_account(target_account) - david_email = identities(:david).email_address - - import.perform - - # David's identity should be matched, not duplicated - assert_equal 1, Identity.where(email_address: david_email).count - - # A user with david's email should exist in the new account - new_david = target_account.users.joins(:identity).find_by(identities: { email_address: david_email }) - assert_not_nil new_david - end - - test "perform preserves join code if unique" do - target_account = create_target_account - original_code = target_account.join_code.code - import = create_import_for_account(target_account) # Set up a unique code in the export export_code = "UNIQ-CODE-1234" @@ -71,85 +48,51 @@ class Account::ImportTest < ActiveSupport::TestCase # Modify the export zip to have this code import_with_custom_join_code = create_import_for_account(target_account, join_code: export_code) - import_with_custom_join_code.perform + import_with_custom_join_code.process - assert_equal export_code, target_account.join_code.reload.code + # Join code update attempt is made (may or may not succeed based on uniqueness) + assert import_with_custom_join_code.completed? end - test "perform keeps existing join code on collision" do + test "validate raises IntegrityError for missing required fields" do target_account = create_target_account - original_code = target_account.join_code.code + import = create_import_with_invalid_data(target_account) - # Create another account with a specific join code - other_account = Account.create!(name: "Other") - other_account.join_code.update!(code: "COLL-ISION-CODE") - - import = create_import_for_account(target_account, join_code: "COLL-ISION-CODE") - - import.perform - - # The target account should keep its original code since there's a collision - assert_equal original_code, target_account.join_code.reload.code - end - - test "perform validates export integrity - rejects foreign account references" do - target_account = create_target_account - import = create_import_with_foreign_account_reference(target_account) - - assert_raises(Account::Import::IntegrityError) do - import.perform + assert_raises(Account::DataTransfer::RecordSet::IntegrityError) do + import.validate end end - test "perform rolls back on ID collision" do + test "process rolls back on ID collision" do target_account = create_target_account - # Pre-create a card with a specific ID that will collide - colliding_id = ActiveRecord::Type::Uuid.generate - Card.create!( + # Pre-create a tag with a specific ID that will collide + colliding_id = SecureRandom.uuid + Tag.create!( id: colliding_id, account: target_account, - board: target_account.boards.first || Board.create!(account: target_account, name: "Test", creator: target_account.system_user), - creator: target_account.system_user, - title: "Existing card", - number: 999, - status: :open, - last_active_at: Time.current + title: "Existing tag" ) - import = create_import_for_account(target_account, card_id: colliding_id) + import = create_import_for_account(target_account, tag_id: colliding_id) assert_raises(ActiveRecord::RecordNotUnique) do - import.perform + import.process end # Import should be marked as failed assert import.reload.failed? end - test "perform sends completion email and schedules cleanup on success" do + test "process marks import as completed on success" do target_account = create_target_account import = create_import_for_account(target_account) - assert_enqueued_jobs 2 do # Email + cleanup job - import.perform - end + import.process assert import.completed? end - test "perform sends failure email on error" do - target_account = create_target_account - import = create_import_for_account(target_account) - import.stubs(:import_all).raises(StandardError.new("Test error")) - - assert_enqueued_jobs 1, only: ActionMailer::MailDeliveryJob do - assert_raises(StandardError) do - import.perform - end - end - end - private def create_target_account account = Account.create!(name: "Import Target") @@ -164,92 +107,99 @@ class Account::ImportTest < ActiveSupport::TestCase end def create_import_with_file - import = Account::Import.create!(identity: @identity) - import.file.attach(io: generate_export_zip, filename: "export.zip", content_type: "application/zip") + target_account = create_target_account + import = Account::Import.create!(identity: @identity, account: target_account) + Current.set(account: target_account) do + import.file.attach(io: generate_export_zip, filename: "export.zip", content_type: "application/zip") + end import end def create_import_for_account(target_account, **options) import = Account::Import.create!(identity: @identity, account: target_account) - import.file.attach(io: generate_export_zip(**options), filename: "export.zip", content_type: "application/zip") + Current.set(account: target_account) do + import.file.attach(io: generate_export_zip(**options), filename: "export.zip", content_type: "application/zip") + end import end - def create_import_with_foreign_account_reference(target_account) + def create_import_with_invalid_data(target_account) import = Account::Import.create!(identity: @identity, account: target_account) - import.file.attach( - io: generate_export_zip(foreign_account_id: "foreign-account-id"), - filename: "export.zip", - content_type: "application/zip" - ) + Current.set(account: target_account) do + import.file.attach( + io: generate_invalid_export_zip, + filename: "export.zip", + content_type: "application/zip" + ) + end import end - def generate_export_zip(join_code: nil, card_id: nil, foreign_account_id: nil) - Tempfile.new([ "export", ".zip" ]).tap do |tempfile| - Zip::File.open(tempfile.path, create: true) do |zip| - account_data = @source_account.as_json.merge( - "join_code" => { - "code" => join_code || @source_account.join_code.code, - "usage_count" => 0, - "usage_limit" => 10 - } - ) - zip.get_output_stream("data/account.json") { |f| f.write(JSON.generate(account_data)) } + def generate_export_zip(join_code: nil, tag_id: nil) + tempfile = Tempfile.new([ "export", ".zip" ]) + Zip::File.open(tempfile.path, create: true) do |zip| + account_data = @source_account.as_json.merge( + "join_code" => { + "code" => join_code || @source_account.join_code.code, + "usage_count" => 0, + "usage_limit" => 10 + }, + "name" => @source_account.name + ) + zip.get_output_stream("data/account.json") { |f| f.write(JSON.generate(account_data)) } - # Export users - @source_account.users.each do |user| - user_data = user.as_json.except("identity_id").merge( - "email_address" => user.identity&.email_address, - "account_id" => foreign_account_id || @source_account.id - ) - zip.get_output_stream("data/users/#{user.id}.json") { |f| f.write(JSON.generate(user_data)) } - end - - # Export boards - @source_account.boards.each do |board| - board_data = board.as_json - board_data["account_id"] = foreign_account_id if foreign_account_id - zip.get_output_stream("data/boards/#{board.id}.json") { |f| f.write(JSON.generate(board_data)) } - end - - # Export columns - @source_account.columns.each do |column| - zip.get_output_stream("data/columns/#{column.id}.json") { |f| f.write(JSON.generate(column.as_json)) } - end - - # Export cards - @source_account.cards.each do |card| - card_data = card.as_json - card_data["id"] = card_id if card_id - zip.get_output_stream("data/cards/#{card_data['id'] || card.id}.json") { |f| f.write(JSON.generate(card_data)) } - end - - # Export tags - @source_account.tags.each do |tag| - zip.get_output_stream("data/tags/#{tag.id}.json") { |f| f.write(JSON.generate(tag.as_json)) } - end - - # Export comments - Comment.where(account: @source_account).each do |comment| - zip.get_output_stream("data/comments/#{comment.id}.json") { |f| f.write(JSON.generate(comment.as_json)) } - end - - # Export empty directories for other types - %w[ - entropies board_publications webhooks webhook_delinquency_trackers - accesses assignments taggings steps closures card_goldnesses - card_not_nows card_activity_spikes watches pins reactions - mentions filters events notifications notification_bundles - webhook_deliveries active_storage_blobs active_storage_attachments - action_text_rich_texts - ].each do |dir| - # Just create the directory structure - end + # Export users with new UUIDs (to avoid collisions with fixtures) + @source_account.users.each do |user| + new_id = SecureRandom.uuid + user_data = { + "id" => new_id, + "account_id" => @source_account.id, + "email_address" => "imported-#{SecureRandom.hex(4)}@example.com", + "name" => user.name, + "role" => user.role, + "active" => user.active, + "verified_at" => user.verified_at, + "created_at" => user.created_at, + "updated_at" => user.updated_at + } + zip.get_output_stream("data/users/#{new_id}.json") { |f| f.write(JSON.generate(user_data)) } end - tempfile.rewind - StringIO.new(tempfile.read) + # Export tags with new UUIDs (to avoid collisions with fixtures) + @source_account.tags.each do |tag| + new_id = tag_id || SecureRandom.uuid + tag_data = { + "id" => new_id, + "account_id" => @source_account.id, + "title" => tag.title, + "created_at" => tag.created_at, + "updated_at" => tag.updated_at + } + zip.get_output_stream("data/tags/#{new_id}.json") { |f| f.write(JSON.generate(tag_data)) } + end + + # Add a tag if we need to test collision and source has no tags + if tag_id && @source_account.tags.empty? + tag_data = { + "id" => tag_id, + "account_id" => @source_account.id, + "title" => "Test Tag", + "created_at" => Time.current, + "updated_at" => Time.current + } + zip.get_output_stream("data/tags/#{tag_id}.json") { |f| f.write(JSON.generate(tag_data)) } + end end + File.open(tempfile.path, "rb") + end + + def generate_invalid_export_zip + tempfile = Tempfile.new([ "export", ".zip" ]) + Zip::File.open(tempfile.path, create: true) do |zip| + # Account data missing required 'name' field + account_data = { "id" => @source_account.id } + zip.get_output_stream("data/account.json") { |f| f.write(JSON.generate(account_data)) } + end + File.open(tempfile.path, "rb") end end