diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb new file mode 100644 index 000000000..76cb41c83 --- /dev/null +++ b/app/controllers/imports_controller.rb @@ -0,0 +1,38 @@ +class ImportsController < ApplicationController + disallow_account_scope + + layout "public" + + def new + end + + def create + account = create_account_for_import + + Current.set(account: account) do + @import = account.imports.create!(identity: Current.identity, file: params[:file]) + end + + @import.perform_later + redirect_to import_path(@import) + end + + def show + @import = Current.identity.imports.find(params[:id]) + end + + private + def create_account_for_import + Account.create_with_owner( + account: { name: account_name_from_zip }, + owner: { name: Current.identity.email_address.split("@").first, identity: Current.identity } + ) + end + + def account_name_from_zip + Zip::File.open(params[:file].tempfile.path) do |zip| + entry = zip.find_entry("data/account.json") + JSON.parse(entry.get_input_stream.read)["name"] + end + end +end diff --git a/app/jobs/import_account_data_job.rb b/app/jobs/import_account_data_job.rb index 8caf731f9..5aa29dc9e 100644 --- a/app/jobs/import_account_data_job.rb +++ b/app/jobs/import_account_data_job.rb @@ -2,6 +2,6 @@ class ImportAccountDataJob < ApplicationJob queue_as :backend def perform(import) - import.build + import.perform end end diff --git a/app/mailers/import_mailer.rb b/app/mailers/import_mailer.rb index bb1c0c4c2..5040a224e 100644 --- a/app/mailers/import_mailer.rb +++ b/app/mailers/import_mailer.rb @@ -1,8 +1,9 @@ class ImportMailer < ApplicationMailer - def completed(import) - @import = import - @identity = import.identity + def completed(identity) + mail to: identity.email_address, subject: "Your Fizzy account import is complete" + end - mail to: @identity.email_address, subject: "Your Fizzy account import is complete" + def failed(identity) + mail to: identity.email_address, subject: "Your Fizzy account import failed" end end diff --git a/app/models/account.rb b/app/models/account.rb index 34b63b688..cd19eba5e 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -9,6 +9,7 @@ class Account < ApplicationRecord has_many :tags, dependent: :destroy has_many :columns, dependent: :destroy has_many :exports, class_name: "Account::Export", dependent: :destroy + has_many :imports, class_name: "Account::Import", dependent: :destroy before_create :assign_external_account_id after_create :create_join_code diff --git a/app/models/account/import.rb b/app/models/account/import.rb index f51e984fc..2357fbd5f 100644 --- a/app/models/account/import.rb +++ b/app/models/account/import.rb @@ -1,271 +1,745 @@ class Account::Import < ApplicationRecord - belongs_to :account, required: false + class IntegrityError < StandardError; end + + belongs_to :account belongs_to :identity has_one_attached :file enum :status, %w[ pending processing completed failed ].index_by(&:itself), default: :pending - def build_later + def perform_later ImportAccountDataJob.perform_later(self) end - def build + def perform processing! - ApplicationRecord.transaction do - populate_from_zip - end + Current.set(account: account, user: owner_user) do + file.open do |tempfile| + Zip::File.open(tempfile.path) do |zip| + ApplicationRecord.transaction do + @old_account_data = load_account_data(zip) + @id_mapper = IdMapper.new(account, @old_account_data) - mark_completed - ImportMailer.completed(self).deliver_later - rescue => e - update!(status: :failed) - raise - end - - def mark_completed - update!(status: :completed, completed_at: Time.current) - end - - private - def populate_from_zip - ApplicationRecord.transaction do - Zip::File.open_buffer(file.download) do |zip| - import_account(zip) - import_users(zip) - import_tags(zip) - import_entropies(zip) - import_columns(zip) - import_board_publications(zip) - import_webhooks(zip) - import_webhook_delinquency_trackers(zip) - import_accesses(zip) - import_assignments(zip) - import_taggings(zip) - import_steps(zip) - import_closures(zip) - import_card_goldnesses(zip) - import_card_not_nows(zip) - import_card_activity_spikes(zip) - import_watches(zip) - import_pins(zip) - import_reactions(zip) - import_mentions(zip) - import_filters(zip) - import_events(zip) - import_notifications(zip) - import_notification_bundles(zip) - import_webhook_deliveries(zip) - - import_boards(zip) - import_cards(zip) - import_comments(zip) - - import_active_storage_blobs(zip) - import_active_storage_attachments(zip) - import_action_text_rich_texts(zip) - - import_blob_files(zip) + validate_export_integrity!(zip) + import_all(zip) + end end end end - def import_account(zip) - entry = zip.find_entry("data/account.json") - raise "Missing account.json in export" unless entry + mark_completed + rescue => e + mark_failed + raise + end - data = JSON.parse(entry.get_input_stream.read) - join_code_data = data.delete("join_code") + def owner_user + account.users.find_by(identity: identity) + end - Account.insert(data) - update!(account_id: data["id"]) - - Account::JoinCode.insert(join_code_data) if join_code_data + private + def mark_completed + update!(status: :completed, completed_at: Time.current) + ImportMailer.completed(identity).deliver_later end + def mark_failed + update!(status: :failed) + ImportMailer.failed(identity).deliver_later + end + + def load_account_data(zip) + entry = zip.find_entry("data/account.json") + raise IntegrityError, "Missing account.json in export" unless entry + + JSON.parse(entry.get_input_stream.read) + end + + def import_all(zip) + # Phase 1: Foundation + import_account_data + import_join_code + + # Phase 2: Users (create identities, map system user) + import_users(zip) + + # Phase 3: Basic entities + import_tags(zip) + import_boards(zip) + import_columns(zip) + import_entropies(zip) + import_board_publications(zip) + + # Phase 4: Cards & content + import_cards(zip) + import_comments(zip) + import_steps(zip) + + # Phase 5: Relationships + import_accesses(zip) + import_assignments(zip) + import_taggings(zip) + import_closures(zip) + import_card_goldnesses(zip) + import_card_not_nows(zip) + import_card_activity_spikes(zip) + import_watches(zip) + import_pins(zip) + import_reactions(zip) + import_mentions(zip) + import_filters(zip) + + # Phase 6: Webhooks + import_webhooks(zip) + import_webhook_delinquency_trackers(zip) + import_webhook_deliveries(zip) + + # Phase 7: Activity & notifications + import_events(zip) + import_notifications(zip) + import_notification_bundles(zip) + + # Phase 8: Storage & rich text + import_active_storage_blobs(zip) + import_active_storage_attachments(zip) + import_action_text_rich_texts(zip) + import_blob_files(zip) + end + + # Phase 1: Foundation + + def import_account_data + account.update!(name: @old_account_data["name"]) + end + + def import_join_code + join_code_data = @old_account_data["join_code"] + return unless join_code_data + + # Preserve the code if it's unique, otherwise keep the auto-generated one + unless Account::JoinCode.exists?(code: join_code_data["code"]) + account.join_code.update!( + code: join_code_data["code"], + usage_count: join_code_data["usage_count"], + usage_limit: join_code_data["usage_limit"] + ) + end + end + + # Phase 2: Users + def import_users(zip) - records = read_json_files(zip, "data/users") - User.insert_all(records) if records.any? + users_data = read_json_files(zip, "data/users") + + # Map system user first + old_system = users_data.find { |u| u["role"] == "system" } + if old_system + @id_mapper.map(:users, old_system["id"], account.system_user.id) + end + + # Import non-system users + users_data.reject { |u| u["role"] == "system" }.each do |data| + import_user(data) + end end + def import_user(data) + email = data.delete("email_address") + old_id = data.delete("id") + + user_identity = if email.present? + Identity.find_or_create_by!(email_address: email) + end + + # Check if user already exists for this identity in this account (e.g., the owner) + existing_user = account.users.find_by(identity: user_identity) if user_identity + if existing_user + existing_user.update!(data.slice("name", "role", "active", "verified_at")) + @id_mapper.map(:users, old_id, existing_user.id) + else + new_user = User.create!( + data.slice(*User.column_names).merge( + "account_id" => account.id, + "identity_id" => user_identity&.id + ) + ) + @id_mapper.map(:users, old_id, new_user.id) + end + end + + # Phase 3: Basic entities + def import_tags(zip) - records = read_json_files(zip, "data/tags") - Tag.insert_all(records) if records.any? - end - - def import_entropies(zip) - records = read_json_files(zip, "data/entropies") - Entropy.insert_all(records) if records.any? - end - - def import_columns(zip) - records = read_json_files(zip, "data/columns") - Column.insert_all(records) if records.any? - end - - def import_board_publications(zip) - records = read_json_files(zip, "data/board_publications") - Board::Publication.insert_all(records) if records.any? - end - - def import_webhooks(zip) - records = read_json_files(zip, "data/webhooks") - Webhook.insert_all(records) if records.any? - end - - def import_webhook_delinquency_trackers(zip) - records = read_json_files(zip, "data/webhook_delinquency_trackers") - Webhook::DelinquencyTracker.insert_all(records) if records.any? - end - - def import_accesses(zip) - records = read_json_files(zip, "data/accesses") - Access.insert_all(records) if records.any? - end - - def import_assignments(zip) - records = read_json_files(zip, "data/assignments") - Assignment.insert_all(records) if records.any? - end - - def import_taggings(zip) - records = read_json_files(zip, "data/taggings") - Tagging.insert_all(records) if records.any? - end - - def import_steps(zip) - records = read_json_files(zip, "data/steps") - Step.insert_all(records) if records.any? - end - - def import_closures(zip) - records = read_json_files(zip, "data/closures") - Closure.insert_all(records) if records.any? - end - - def import_card_goldnesses(zip) - records = read_json_files(zip, "data/card_goldnesses") - Card::Goldness.insert_all(records) if records.any? - end - - def import_card_not_nows(zip) - records = read_json_files(zip, "data/card_not_nows") - Card::NotNow.insert_all(records) if records.any? - end - - def import_card_activity_spikes(zip) - records = read_json_files(zip, "data/card_activity_spikes") - Card::ActivitySpike.insert_all(records) if records.any? - end - - def import_watches(zip) - records = read_json_files(zip, "data/watches") - Watch.insert_all(records) if records.any? - end - - def import_pins(zip) - records = read_json_files(zip, "data/pins") - Pin.insert_all(records) if records.any? - end - - def import_reactions(zip) - records = read_json_files(zip, "data/reactions") - Reaction.insert_all(records) if records.any? - end - - def import_mentions(zip) - records = read_json_files(zip, "data/mentions") - Mention.insert_all(records) if records.any? - end - - def import_filters(zip) - records = read_json_files(zip, "data/filters") - Filter.insert_all(records) if records.any? - end - - def import_events(zip) - records = read_json_files(zip, "data/events") - Event.insert_all(records) if records.any? - end - - def import_notifications(zip) - records = read_json_files(zip, "data/notifications") - Notification.insert_all(records) if records.any? - end - - def import_notification_bundles(zip) - records = read_json_files(zip, "data/notification_bundles") - Notification::Bundle.insert_all(records) if records.any? - end - - def import_webhook_deliveries(zip) - records = read_json_files(zip, "data/webhook_deliveries") - Webhook::Delivery.insert_all(records) if records.any? + records = read_json_files(zip, "data/tags").map do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data) + new_record = Tag.create!(data) + @id_mapper.map(:tags, old_id, new_record.id) + end end def import_boards(zip) - records = read_json_files(zip, "data/boards") - Board.insert_all(records) if records.any? + read_json_files(zip, "data/boards").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data) + new_record = Board.create!(data) + @id_mapper.map(:boards, old_id, new_record.id) + end end + def import_columns(zip) + read_json_files(zip, "data/columns").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data, foreign_keys: { "board_id" => :boards }) + new_record = Column.create!(data) + @id_mapper.map(:columns, old_id, new_record.id) + end + end + + def import_entropies(zip) + read_json_files(zip, "data/entropies").each do |data| + old_id = data.delete("id") + + # Remap polymorphic container_id based on container_type + container_type = data["container_type"] + if container_type == "Account" + data["container_id"] = account.id + elsif container_type == "Board" + data["container_id"] = @id_mapper.lookup(:boards, data["container_id"]) + end + + data = @id_mapper.remap(data) + + # Find existing or create new + existing = Entropy.find_by(container_type: data["container_type"], container_id: data["container_id"]) + if existing + existing.update!(data.slice("auto_postpone_period")) + @id_mapper.map(:entropies, old_id, existing.id) + else + new_record = Entropy.create!(data) + @id_mapper.map(:entropies, old_id, new_record.id) + end + end + end + + def import_board_publications(zip) + read_json_files(zip, "data/board_publications").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data, foreign_keys: { "board_id" => :boards }) + new_record = Board::Publication.create!(data) + @id_mapper.map(:board_publications, old_id, new_record.id) + end + end + + # Phase 4: Cards & content + def import_cards(zip) - records = read_json_files(zip, "data/cards") - Card.insert_all(records) if records.any? + read_json_files(zip, "data/cards").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "board_id" => :boards, + "column_id" => :columns + }) + new_record = Card.create!(data) + @id_mapper.map(:cards, old_id, new_record.id) + end end def import_comments(zip) - records = read_json_files(zip, "data/comments") - Comment.insert_all(records) if records.any? + read_json_files(zip, "data/comments").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "card_id" => :cards + }) + new_record = Comment.create!(data) + @id_mapper.map(:comments, old_id, new_record.id) + end end + def import_steps(zip) + read_json_files(zip, "data/steps").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data, foreign_keys: { "card_id" => :cards }) + new_record = Step.create!(data) + @id_mapper.map(:steps, old_id, new_record.id) + end + end + + # Phase 5: Relationships + + def import_accesses(zip) + read_json_files(zip, "data/accesses").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "board_id" => :boards + }) + + # Board creation auto-creates access for creator, check if it exists + existing = Access.find_by(board_id: data["board_id"], user_id: data["user_id"]) + if existing + @id_mapper.map(:accesses, old_id, existing.id) + else + new_record = Access.create!(data) + @id_mapper.map(:accesses, old_id, new_record.id) + end + end + end + + def import_assignments(zip) + read_json_files(zip, "data/assignments").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "card_id" => :cards + }) + new_record = Assignment.create!(data) + @id_mapper.map(:assignments, old_id, new_record.id) + end + end + + def import_taggings(zip) + read_json_files(zip, "data/taggings").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data, foreign_keys: { + "card_id" => :cards, + "tag_id" => :tags + }) + new_record = Tagging.create!(data) + @id_mapper.map(:taggings, old_id, new_record.id) + end + end + + def import_closures(zip) + read_json_files(zip, "data/closures").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "card_id" => :cards + }) + new_record = Closure.create!(data) + @id_mapper.map(:closures, old_id, new_record.id) + end + end + + def import_card_goldnesses(zip) + read_json_files(zip, "data/card_goldnesses").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data, foreign_keys: { "card_id" => :cards }) + new_record = Card::Goldness.create!(data) + @id_mapper.map(:card_goldnesses, old_id, new_record.id) + end + end + + def import_card_not_nows(zip) + read_json_files(zip, "data/card_not_nows").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "card_id" => :cards + }) + new_record = Card::NotNow.create!(data) + @id_mapper.map(:card_not_nows, old_id, new_record.id) + end + end + + def import_card_activity_spikes(zip) + read_json_files(zip, "data/card_activity_spikes").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data, foreign_keys: { "card_id" => :cards }) + new_record = Card::ActivitySpike.create!(data) + @id_mapper.map(:card_activity_spikes, old_id, new_record.id) + end + end + + def import_watches(zip) + read_json_files(zip, "data/watches").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "card_id" => :cards + }) + + # Card creation auto-creates watch for creator, check if it exists + existing = Watch.find_by(card_id: data["card_id"], user_id: data["user_id"]) + if existing + @id_mapper.map(:watches, old_id, existing.id) + else + new_record = Watch.create!(data) + @id_mapper.map(:watches, old_id, new_record.id) + end + end + end + + def import_pins(zip) + read_json_files(zip, "data/pins").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "card_id" => :cards + }) + new_record = Pin.create!(data) + @id_mapper.map(:pins, old_id, new_record.id) + end + end + + def import_reactions(zip) + read_json_files(zip, "data/reactions").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "comment_id" => :comments + }) + new_record = Reaction.create!(data) + @id_mapper.map(:reactions, old_id, new_record.id) + end + end + + def import_mentions(zip) + read_json_files(zip, "data/mentions").each do |data| + old_id = data.delete("id") + + # Remap polymorphic source_id based on source_type + source_type = data["source_type"] + source_mapping = case source_type + when "Card" then :cards + when "Comment" then :comments + end + data["source_id"] = @id_mapper.lookup(source_mapping, data["source_id"]) if source_mapping + + data = @id_mapper.remap_with_users(data) + new_record = Mention.create!(data) + @id_mapper.map(:mentions, old_id, new_record.id) + end + end + + def import_filters(zip) + read_json_files(zip, "data/filters").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data) + new_record = Filter.create!(data) + @id_mapper.map(:filters, old_id, new_record.id) + end + end + + # Phase 6: Webhooks + + def import_webhooks(zip) + read_json_files(zip, "data/webhooks").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data, foreign_keys: { "board_id" => :boards }) + new_record = Webhook.create!(data) + @id_mapper.map(:webhooks, old_id, new_record.id) + end + end + + def import_webhook_delinquency_trackers(zip) + read_json_files(zip, "data/webhook_delinquency_trackers").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data, foreign_keys: { "webhook_id" => :webhooks }) + new_record = Webhook::DelinquencyTracker.create!(data) + @id_mapper.map(:webhook_delinquency_trackers, old_id, new_record.id) + end + end + + def import_webhook_deliveries(zip) + read_json_files(zip, "data/webhook_deliveries").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data, foreign_keys: { + "webhook_id" => :webhooks, + "event_id" => :events + }) + new_record = Webhook::Delivery.create!(data) + @id_mapper.map(:webhook_deliveries, old_id, new_record.id) + end + end + + # Phase 7: Activity & notifications + + def import_events(zip) + read_json_files(zip, "data/events").each do |data| + old_id = data.delete("id") + + # Remap polymorphic eventable_id + eventable_type = data["eventable_type"] + eventable_mapping = polymorphic_type_to_mapping(eventable_type) + data["eventable_id"] = @id_mapper.lookup(eventable_mapping, data["eventable_id"]) if eventable_mapping + + data = @id_mapper.remap_with_users(data, additional_foreign_keys: { + "board_id" => :boards + }) + new_record = Event.create!(data) + @id_mapper.map(:events, old_id, new_record.id) + end + end + + def import_notifications(zip) + read_json_files(zip, "data/notifications").each do |data| + old_id = data.delete("id") + + # Remap polymorphic source_id + source_type = data["source_type"] + source_mapping = polymorphic_type_to_mapping(source_type) + data["source_id"] = @id_mapper.lookup(source_mapping, data["source_id"]) if source_mapping + + data = @id_mapper.remap_with_users(data) + new_record = Notification.create!(data) + @id_mapper.map(:notifications, old_id, new_record.id) + end + end + + def import_notification_bundles(zip) + read_json_files(zip, "data/notification_bundles").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap_with_users(data) + new_record = Notification::Bundle.create!(data) + @id_mapper.map(:notification_bundles, old_id, new_record.id) + end + end + + # Phase 8: Storage & rich text + def import_active_storage_blobs(zip) - records = read_json_files(zip, "data/active_storage_blobs") - ActiveStorage::Blob.insert_all(records) if records.any? + read_json_files(zip, "data/active_storage_blobs").each do |data| + old_id = data.delete("id") + data = @id_mapper.remap(data) + new_record = ActiveStorage::Blob.create!(data) + @id_mapper.map(:active_storage_blobs, old_id, new_record.id) + end end def import_active_storage_attachments(zip) - records = read_json_files(zip, "data/active_storage_attachments") - ActiveStorage::Attachment.insert_all(records) if records.any? + read_json_files(zip, "data/active_storage_attachments").each do |data| + old_id = data.delete("id") + + # Remap polymorphic record_id + record_type = data["record_type"] + record_mapping = polymorphic_type_to_mapping(record_type) + data["record_id"] = @id_mapper.lookup(record_mapping, data["record_id"]) if record_mapping + + data = @id_mapper.remap(data, foreign_keys: { "blob_id" => :active_storage_blobs }) + new_record = ActiveStorage::Attachment.create!(data) + @id_mapper.map(:active_storage_attachments, old_id, new_record.id) + end end def import_action_text_rich_texts(zip) - records = read_json_files(zip, "data/action_text_rich_texts").map do |record| - record["body"] = convert_gids_to_sgids(record["body"]) - record + read_json_files(zip, "data/action_text_rich_texts").each do |data| + old_id = data.delete("id") + + # Remap polymorphic record_id + record_type = data["record_type"] + record_mapping = polymorphic_type_to_mapping(record_type) + data["record_id"] = @id_mapper.lookup(record_mapping, data["record_id"]) if record_mapping + + data["body"] = convert_gids_and_fix_links(data["body"]) + data = @id_mapper.remap(data) + new_record = ActionText::RichText.create!(data) + @id_mapper.map(:action_text_rich_texts, old_id, new_record.id) end - ActionText::RichText.insert_all(records) if records.any? end def import_blob_files(zip) zip.glob("storage/*").each do |entry| key = File.basename(entry.name) - blob = ActiveStorage::Blob.find_by(key: key) + blob = ActiveStorage::Blob.find_by(key: key, account: account) next unless blob blob.upload(entry.get_input_stream) end end + # Helper methods + def read_json_files(zip, directory) zip.glob("#{directory}/*.json").map do |entry| JSON.parse(entry.get_input_stream.read) end end - def convert_gids_to_sgids(html) + def convert_gids_and_fix_links(html) return html if html.blank? fragment = Nokogiri::HTML.fragment(html) + + # Convert GIDs to SGIDs fragment.css("action-text-attachment[gid]").each do |node| gid = GlobalID.parse(node["gid"]) - record = gid&.find - next unless record + next unless gid + + type = gid.model_name.plural.underscore.to_sym + new_id = @id_mapper.lookup(type, gid.model_id) + record = gid.model_class.find(new_id) node["sgid"] = record.attachable_sgid node.remove_attribute("gid") end + # Fix links + fragment.css("a[href]").each do |link| + link["href"] = rewrite_link(link["href"]) + end + fragment.to_html end + + def rewrite_link(url) + uri = URI.parse(url) rescue nil + return url unless uri&.path + + path = uri.path + old_slug_pattern = %r{^/#{Regexp.escape(@id_mapper.old_account_slug)}/} + + return url unless path.match?(old_slug_pattern) + + # Replace account slug + path = path.sub(old_slug_pattern, "#{account.slug}/") + + # Try to recognize and remap IDs in the path + begin + params = Rails.application.routes.recognize_path(path) + + case params[:controller] + when "cards" + if params[:id] && @id_mapper[:cards].key?(params[:id]) + new_id = @id_mapper[:cards][params[:id]] + path = Rails.application.routes.url_helpers.card_path(new_id) + end + when "boards" + if params[:id] && @id_mapper[:boards].key?(params[:id]) + new_id = @id_mapper[:boards][params[:id]] + path = Rails.application.routes.url_helpers.board_path(new_id) + end + end + rescue ActionController::RoutingError + # Unknown route, just update the slug + end + + uri.path = path + uri.to_s + end + + def polymorphic_type_to_mapping(type) + case type + when "Card" then :cards + when "Comment" then :comments + when "Board" then :boards + when "User" then :users + when "Tag" then :tags + when "Assignment" then :assignments + when "Tagging" then :taggings + when "Closure" then :closures + when "Step" then :steps + when "Watch" then :watches + when "Pin" then :pins + when "Reaction" then :reactions + when "Mention" then :mentions + when "Event" then :events + when "Access" then :accesses + when "Webhook" then :webhooks + when "Webhook::Delivery" then :webhook_deliveries + when "Card::Goldness" then :card_goldnesses + when "Card::NotNow" then :card_not_nows + when "Card::ActivitySpike" then :card_activity_spikes + when "ActiveStorage::Blob" then :active_storage_blobs + when "ActiveStorage::Attachment" then :active_storage_attachments + when "ActionText::RichText" then :action_text_rich_texts + end + end + + # Data integrity validation + + def validate_export_integrity!(zip) + exported_ids = collect_exported_ids(zip) + + zip.glob("data/**/*.json").each do |entry| + next if entry.name == "data/account.json" + + data = JSON.parse(entry.get_input_stream.read) + validate_account_id(data, entry.name, exported_ids[:account]) + validate_foreign_keys(data, exported_ids, entry.name) + end + end + + def collect_exported_ids(zip) + ids = Hash.new { |h, k| h[k] = Set.new } + + # Account ID + ids[:account] = @old_account_data["id"] + + # Collect IDs from each entity type + entity_directories = { + "data/users" => :users, + "data/tags" => :tags, + "data/boards" => :boards, + "data/columns" => :columns, + "data/cards" => :cards, + "data/comments" => :comments, + "data/steps" => :steps, + "data/accesses" => :accesses, + "data/assignments" => :assignments, + "data/taggings" => :taggings, + "data/closures" => :closures, + "data/card_goldnesses" => :card_goldnesses, + "data/card_not_nows" => :card_not_nows, + "data/card_activity_spikes" => :card_activity_spikes, + "data/watches" => :watches, + "data/pins" => :pins, + "data/reactions" => :reactions, + "data/mentions" => :mentions, + "data/filters" => :filters, + "data/events" => :events, + "data/notifications" => :notifications, + "data/notification_bundles" => :notification_bundles, + "data/webhooks" => :webhooks, + "data/webhook_delinquency_trackers" => :webhook_delinquency_trackers, + "data/webhook_deliveries" => :webhook_deliveries, + "data/entropies" => :entropies, + "data/board_publications" => :board_publications, + "data/active_storage_blobs" => :active_storage_blobs, + "data/active_storage_attachments" => :active_storage_attachments, + "data/action_text_rich_texts" => :action_text_rich_texts + } + + entity_directories.each do |directory, type| + zip.glob("#{directory}/*.json").each do |entry| + data = JSON.parse(entry.get_input_stream.read) + ids[type].add(data["id"]) + end + end + + ids + end + + def validate_account_id(data, filename, expected_account_id) + if data["account_id"] && data["account_id"] != expected_account_id + raise IntegrityError, "#{filename} references foreign account: #{data["account_id"]}" + end + end + + FOREIGN_KEY_VALIDATIONS = { + "board_id" => :boards, + "card_id" => :cards, + "column_id" => :columns, + "user_id" => :users, + "creator_id" => :users, + "assignee_id" => :users, + "assigner_id" => :users, + "closer_id" => :users, + "mentioner_id" => :users, + "mentionee_id" => :users, + "reacter_id" => :users, + "tag_id" => :tags, + "comment_id" => :comments, + "webhook_id" => :webhooks, + "event_id" => :events, + "blob_id" => :active_storage_blobs, + "filter_id" => :filters + }.freeze + + def validate_foreign_keys(data, exported_ids, filename) + FOREIGN_KEY_VALIDATIONS.each do |field, type| + ref_id = data[field] + next unless ref_id + + unless exported_ids[type]&.include?(ref_id) + raise IntegrityError, "#{filename} references unknown #{type}: #{ref_id}" + end + end + end end diff --git a/app/models/account/import/id_mapper.rb b/app/models/account/import/id_mapper.rb new file mode 100644 index 000000000..63e19afd0 --- /dev/null +++ b/app/models/account/import/id_mapper.rb @@ -0,0 +1,60 @@ +class Account::Import::IdMapper + attr_reader :account, :old_account_id, :old_external_account_id, :old_account_slug + + def initialize(account, old_account_data) + @account = account + @old_account_id = old_account_data["id"] + @old_external_account_id = old_account_data["external_account_id"] + @old_account_slug = AccountSlug.encode(@old_external_account_id) + @mappings = Hash.new { |h, k| h[k] = {} } + end + + def map(type, old_id, new_id) + @mappings[type][old_id] = new_id + end + + def [](type) + @mappings[type] + end + + def mapped?(type, old_id) + @mappings[type].key?(old_id) + end + + def lookup(type, old_id) + @mappings[type][old_id] || old_id + end + + # Remap account_id and specified foreign keys in a data hash + # foreign_keys is a Hash of { "field_name" => :type } + def remap(data, foreign_keys: {}) + data = data.dup + data["account_id"] = account.id if data.key?("account_id") + + foreign_keys.each do |field, type| + old_id = data[field] + next unless old_id + next unless @mappings[type].key?(old_id) + + data[field] = @mappings[type][old_id] + end + + data + end + + # Common foreign key mappings for user-related fields + USER_FOREIGN_KEYS = { + "user_id" => :users, + "creator_id" => :users, + "assignee_id" => :users, + "assigner_id" => :users, + "closer_id" => :users, + "mentioner_id" => :users, + "mentionee_id" => :users, + "reacter_id" => :users + }.freeze + + def remap_with_users(data, additional_foreign_keys: {}) + remap(data, foreign_keys: USER_FOREIGN_KEYS.merge(additional_foreign_keys)) + end +end diff --git a/app/models/account/whole_account_export.rb b/app/models/account/whole_account_export.rb index 43865144a..8551463e1 100644 --- a/app/models/account/whole_account_export.rb +++ b/app/models/account/whole_account_export.rb @@ -45,7 +45,10 @@ class Account::WholeAccountExport < Account::Export def export_users(zip) account.users.find_each do |user| - add_file_to_zip(zip, "data/users/#{user.id}.json", JSON.pretty_generate(user.as_json)) + data = user.as_json.except("identity_id").merge( + "email_address" => user.identity&.email_address + ) + add_file_to_zip(zip, "data/users/#{user.id}.json", JSON.pretty_generate(data)) end end diff --git a/app/models/identity.rb b/app/models/identity.rb index 7495e37c3..33955a8b2 100644 --- a/app/models/identity.rb +++ b/app/models/identity.rb @@ -2,6 +2,7 @@ class Identity < ApplicationRecord include Joinable, Transferable has_many :access_tokens, dependent: :destroy + has_many :imports, class_name: "Account::Import", dependent: :destroy has_many :magic_links, dependent: :destroy has_many :sessions, dependent: :destroy has_many :users, dependent: :nullify diff --git a/app/views/imports/new.html.erb b/app/views/imports/new.html.erb new file mode 100644 index 000000000..138be6758 --- /dev/null +++ b/app/views/imports/new.html.erb @@ -0,0 +1,21 @@ +<% @page_title = "Import an account" %> + +
+

Import an account

+ + <%= form_with url: imports_path, class: "flex flex-column gap", data: { controller: "form" }, multipart: true do |form| %> +
+ <%= form.file_field :file, accept: ".zip", required: true, class: "input" %> +

Upload the .zip file from your Fizzy export.

+
+ + + <% end %> +
+ +<% content_for :footer do %> + <%= render "sessions/footer" %> +<% end %> diff --git a/app/views/imports/show.html.erb b/app/views/imports/show.html.erb new file mode 100644 index 000000000..acc2c187e --- /dev/null +++ b/app/views/imports/show.html.erb @@ -0,0 +1,23 @@ +<% @page_title = "Import status" %> + +
+

Import status

+ + <% case @import.status %> + <% when "pending", "processing" %> +

Your import is in progress. This may take a while for large accounts.

+

This page will refresh automatically.

+ + <% when "completed" %> +

Your import has completed successfully!

+ <%= link_to "Go to your account", landing_url(script_name: @import.account.slug), class: "btn btn--link center txt-medium" %> + <% when "failed" %> +

Your import failed.

+

This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export.

+ <%= link_to "Try again", new_import_path, class: "btn btn--plain center txt-medium" %> + <% end %> +
+ +<% content_for :footer do %> + <%= render "sessions/footer" %> +<% end %> diff --git a/app/views/mailers/import_mailer/failed.html.erb b/app/views/mailers/import_mailer/failed.html.erb new file mode 100644 index 000000000..cd3999957 --- /dev/null +++ b/app/views/mailers/import_mailer/failed.html.erb @@ -0,0 +1,6 @@ +

Your Fizzy account import failed

+

Unfortunately, we were unable to import your Fizzy account data.

+ +

This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export, or contact support if the problem persists.

+ + diff --git a/app/views/mailers/import_mailer/failed.text.erb b/app/views/mailers/import_mailer/failed.text.erb new file mode 100644 index 000000000..1a0f0c171 --- /dev/null +++ b/app/views/mailers/import_mailer/failed.text.erb @@ -0,0 +1,7 @@ +Your Fizzy account import failed + +Unfortunately, we were unable to import your Fizzy account data. + +This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export, or contact support if the problem persists. + +Need help? Send us an email at support@fizzy.do diff --git a/app/views/sessions/menus/show.html.erb b/app/views/sessions/menus/show.html.erb index 09383b72a..84dd43c87 100644 --- a/app/views/sessions/menus/show.html.erb +++ b/app/views/sessions/menus/show.html.erb @@ -24,10 +24,13 @@

You don’t have any Fizzy accounts.

<% end %> -
+
<%= link_to new_signup_path, class: "btn btn--plain txt-link center txt-small", data: { turbo_prefetch: false } do %> Sign up for a new Fizzy account <% end %> + <%= link_to new_import_path, class: "btn btn--plain txt-link center txt-small", data: { turbo_prefetch: false } do %> + Import an exported account + <% end %>
<% end %> diff --git a/config/routes.rb b/config/routes.rb index 9ca2eb05b..3c8c66589 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -9,6 +9,8 @@ Rails.application.routes.draw do resources :exports, only: [ :create, :show ] end + resources :imports, only: %i[ new create show ] + resources :users do scope module: :users do resource :avatar @@ -168,6 +170,7 @@ Rails.application.routes.draw do resource :landing + namespace :my do resource :identity, only: :show resources :access_tokens diff --git a/test/models/account/import_test.rb b/test/models/account/import_test.rb new file mode 100644 index 000000000..6a21d8343 --- /dev/null +++ b/test/models/account/import_test.rb @@ -0,0 +1,255 @@ +require "test_helper" + +class Account::ImportTest < ActiveSupport::TestCase + setup do + @identity = identities(:david) + @source_account = accounts("37s") + end + + test "perform_later enqueues ImportAccountDataJob" do + import = create_import_with_file + + assert_enqueued_with(job: ImportAccountDataJob, args: [ import ]) do + import.perform_later + end + end + + test "perform sets status to failed on error" do + import = create_import_with_file + import.stubs(:import_all).raises(StandardError.new("Test error")) + + assert_raises(StandardError) do + import.perform + end + + assert import.failed? + end + + test "perform imports account name from export" do + target_account = create_target_account + import = create_import_for_account(target_account) + + import.perform + + assert_equal @source_account.name, target_account.reload.name + end + + test "perform maps system user" do + target_account = create_target_account + import = create_import_for_account(target_account) + + import.perform + + # The target account should still have exactly one system user + assert_equal 1, target_account.users.where(role: :system).count + end + + test "perform imports users with identity matching" do + target_account = create_target_account + import = create_import_for_account(target_account) + david_email = identities(:david).email_address + + import.perform + + # David's identity should be matched, not duplicated + assert_equal 1, Identity.where(email_address: david_email).count + + # A user with david's email should exist in the new account + new_david = target_account.users.joins(:identity).find_by(identities: { email_address: david_email }) + assert_not_nil new_david + end + + test "perform preserves join code if unique" do + target_account = create_target_account + original_code = target_account.join_code.code + import = create_import_for_account(target_account) + + # Set up a unique code in the export + export_code = "UNIQ-CODE-1234" + Account::JoinCode.where(code: export_code).delete_all + + # Modify the export zip to have this code + import_with_custom_join_code = create_import_for_account(target_account, join_code: export_code) + + import_with_custom_join_code.perform + + assert_equal export_code, target_account.join_code.reload.code + end + + test "perform keeps existing join code on collision" do + target_account = create_target_account + original_code = target_account.join_code.code + + # Create another account with a specific join code + other_account = Account.create!(name: "Other") + other_account.join_code.update!(code: "COLL-ISION-CODE") + + import = create_import_for_account(target_account, join_code: "COLL-ISION-CODE") + + import.perform + + # The target account should keep its original code since there's a collision + assert_equal original_code, target_account.join_code.reload.code + end + + test "perform validates export integrity - rejects foreign account references" do + target_account = create_target_account + import = create_import_with_foreign_account_reference(target_account) + + assert_raises(Account::Import::IntegrityError) do + import.perform + end + end + + test "perform rolls back on ID collision" do + target_account = create_target_account + + # Pre-create a card with a specific ID that will collide + colliding_id = ActiveRecord::Type::Uuid.generate + Card.create!( + id: colliding_id, + account: target_account, + board: target_account.boards.first || Board.create!(account: target_account, name: "Test", creator: target_account.system_user), + creator: target_account.system_user, + title: "Existing card", + number: 999, + status: :open, + last_active_at: Time.current + ) + + import = create_import_for_account(target_account, card_id: colliding_id) + + assert_raises(ActiveRecord::RecordNotUnique) do + import.perform + end + + # Import should be marked as failed + assert import.reload.failed? + end + + test "perform sends completion email and schedules cleanup on success" do + target_account = create_target_account + import = create_import_for_account(target_account) + + assert_enqueued_jobs 2 do # Email + cleanup job + import.perform + end + + assert import.completed? + end + + test "perform sends failure email on error" do + target_account = create_target_account + import = create_import_for_account(target_account) + import.stubs(:import_all).raises(StandardError.new("Test error")) + + assert_enqueued_jobs 1, only: ActionMailer::MailDeliveryJob do + assert_raises(StandardError) do + import.perform + end + end + end + + private + def create_target_account + account = Account.create!(name: "Import Target") + account.users.create!(role: :system, name: "System") + account.users.create!( + role: :owner, + name: "Importer", + identity: @identity, + verified_at: Time.current + ) + account + end + + def create_import_with_file + import = Account::Import.create!(identity: @identity) + import.file.attach(io: generate_export_zip, filename: "export.zip", content_type: "application/zip") + import + end + + def create_import_for_account(target_account, **options) + import = Account::Import.create!(identity: @identity, account: target_account) + import.file.attach(io: generate_export_zip(**options), filename: "export.zip", content_type: "application/zip") + import + end + + def create_import_with_foreign_account_reference(target_account) + import = Account::Import.create!(identity: @identity, account: target_account) + import.file.attach( + io: generate_export_zip(foreign_account_id: "foreign-account-id"), + filename: "export.zip", + content_type: "application/zip" + ) + import + end + + def generate_export_zip(join_code: nil, card_id: nil, foreign_account_id: nil) + Tempfile.new([ "export", ".zip" ]).tap do |tempfile| + Zip::File.open(tempfile.path, create: true) do |zip| + account_data = @source_account.as_json.merge( + "join_code" => { + "code" => join_code || @source_account.join_code.code, + "usage_count" => 0, + "usage_limit" => 10 + } + ) + zip.get_output_stream("data/account.json") { |f| f.write(JSON.generate(account_data)) } + + # Export users + @source_account.users.each do |user| + user_data = user.as_json.except("identity_id").merge( + "email_address" => user.identity&.email_address, + "account_id" => foreign_account_id || @source_account.id + ) + zip.get_output_stream("data/users/#{user.id}.json") { |f| f.write(JSON.generate(user_data)) } + end + + # Export boards + @source_account.boards.each do |board| + board_data = board.as_json + board_data["account_id"] = foreign_account_id if foreign_account_id + zip.get_output_stream("data/boards/#{board.id}.json") { |f| f.write(JSON.generate(board_data)) } + end + + # Export columns + @source_account.columns.each do |column| + zip.get_output_stream("data/columns/#{column.id}.json") { |f| f.write(JSON.generate(column.as_json)) } + end + + # Export cards + @source_account.cards.each do |card| + card_data = card.as_json + card_data["id"] = card_id if card_id + zip.get_output_stream("data/cards/#{card_data['id'] || card.id}.json") { |f| f.write(JSON.generate(card_data)) } + end + + # Export tags + @source_account.tags.each do |tag| + zip.get_output_stream("data/tags/#{tag.id}.json") { |f| f.write(JSON.generate(tag.as_json)) } + end + + # Export comments + Comment.where(account: @source_account).each do |comment| + zip.get_output_stream("data/comments/#{comment.id}.json") { |f| f.write(JSON.generate(comment.as_json)) } + end + + # Export empty directories for other types + %w[ + entropies board_publications webhooks webhook_delinquency_trackers + accesses assignments taggings steps closures card_goldnesses + card_not_nows card_activity_spikes watches pins reactions + mentions filters events notifications notification_bundles + webhook_deliveries active_storage_blobs active_storage_attachments + action_text_rich_texts + ].each do |dir| + # Just create the directory structure + end + end + + tempfile.rewind + StringIO.new(tempfile.read) + end + end +end