Implement full account exports

This commit is contained in:
Stanko K.R.
2025-12-22 17:02:15 +01:00
parent 6f17e3867d
commit 8953920652
10 changed files with 759 additions and 188 deletions
+7
View File
@@ -0,0 +1,7 @@
class ImportAccountDataJob < ApplicationJob
queue_as :backend
def perform(import)
import.build
end
end
+8
View File
@@ -0,0 +1,8 @@
class ImportMailer < ApplicationMailer
def completed(import)
@import = import
@identity = import.identity
mail to: @identity.email_address, subject: "Your Fizzy account import is complete"
end
end
+12 -3
View File
@@ -19,19 +19,24 @@ class Account::Export < ApplicationRecord
def build
processing!
Current.set(account: account) do
with_url_options do
zipfile = generate_zip { |zip| populate_zip(zip) }
file.attach io: File.open(zipfile.path), filename: "fizzy-export-#{id}.zip", content_type: "application/zip"
mark_completed
ExportMailer.completed(self).deliver_later
rescue => e
update!(status: :failed)
raise
ensure
zipfile&.close
zipfile&.unlink
end
end
rescue => e
update!(status: :failed)
raise
end
def mark_completed
update!(status: :completed, completed_at: Time.current)
@@ -42,6 +47,10 @@ class Account::Export < ApplicationRecord
end
private
def with_url_options
ActiveStorage::Current.set(url_options: { host: "localhost" }) { yield }
end
def populate_zip(zip)
raise NotImplementedError, "Subclasses must implement populate_zip"
end
+271
View File
@@ -0,0 +1,271 @@
class Account::Import < ApplicationRecord
belongs_to :account, required: false
belongs_to :identity
has_one_attached :file
enum :status, %w[ pending processing completed failed ].index_by(&:itself), default: :pending
def build_later
ImportAccountDataJob.perform_later(self)
end
def build
processing!
ApplicationRecord.transaction do
populate_from_zip
end
mark_completed
ImportMailer.completed(self).deliver_later
rescue => e
update!(status: :failed)
raise
end
def mark_completed
update!(status: :completed, completed_at: Time.current)
end
private
def populate_from_zip
ApplicationRecord.transaction do
Zip::File.open_buffer(file.download) do |zip|
import_account(zip)
import_users(zip)
import_tags(zip)
import_entropies(zip)
import_columns(zip)
import_board_publications(zip)
import_webhooks(zip)
import_webhook_delinquency_trackers(zip)
import_accesses(zip)
import_assignments(zip)
import_taggings(zip)
import_steps(zip)
import_closures(zip)
import_card_goldnesses(zip)
import_card_not_nows(zip)
import_card_activity_spikes(zip)
import_watches(zip)
import_pins(zip)
import_reactions(zip)
import_mentions(zip)
import_filters(zip)
import_events(zip)
import_notifications(zip)
import_notification_bundles(zip)
import_webhook_deliveries(zip)
import_boards(zip)
import_cards(zip)
import_comments(zip)
import_active_storage_blobs(zip)
import_active_storage_attachments(zip)
import_action_text_rich_texts(zip)
import_blob_files(zip)
end
end
end
def import_account(zip)
entry = zip.find_entry("data/account.json")
raise "Missing account.json in export" unless entry
data = JSON.parse(entry.get_input_stream.read)
join_code_data = data.delete("join_code")
Account.insert(data)
update!(account_id: data["id"])
Account::JoinCode.insert(join_code_data) if join_code_data
end
def import_users(zip)
records = read_json_files(zip, "data/users")
User.insert_all(records) if records.any?
end
def import_tags(zip)
records = read_json_files(zip, "data/tags")
Tag.insert_all(records) if records.any?
end
def import_entropies(zip)
records = read_json_files(zip, "data/entropies")
Entropy.insert_all(records) if records.any?
end
def import_columns(zip)
records = read_json_files(zip, "data/columns")
Column.insert_all(records) if records.any?
end
def import_board_publications(zip)
records = read_json_files(zip, "data/board_publications")
Board::Publication.insert_all(records) if records.any?
end
def import_webhooks(zip)
records = read_json_files(zip, "data/webhooks")
Webhook.insert_all(records) if records.any?
end
def import_webhook_delinquency_trackers(zip)
records = read_json_files(zip, "data/webhook_delinquency_trackers")
Webhook::DelinquencyTracker.insert_all(records) if records.any?
end
def import_accesses(zip)
records = read_json_files(zip, "data/accesses")
Access.insert_all(records) if records.any?
end
def import_assignments(zip)
records = read_json_files(zip, "data/assignments")
Assignment.insert_all(records) if records.any?
end
def import_taggings(zip)
records = read_json_files(zip, "data/taggings")
Tagging.insert_all(records) if records.any?
end
def import_steps(zip)
records = read_json_files(zip, "data/steps")
Step.insert_all(records) if records.any?
end
def import_closures(zip)
records = read_json_files(zip, "data/closures")
Closure.insert_all(records) if records.any?
end
def import_card_goldnesses(zip)
records = read_json_files(zip, "data/card_goldnesses")
Card::Goldness.insert_all(records) if records.any?
end
def import_card_not_nows(zip)
records = read_json_files(zip, "data/card_not_nows")
Card::NotNow.insert_all(records) if records.any?
end
def import_card_activity_spikes(zip)
records = read_json_files(zip, "data/card_activity_spikes")
Card::ActivitySpike.insert_all(records) if records.any?
end
def import_watches(zip)
records = read_json_files(zip, "data/watches")
Watch.insert_all(records) if records.any?
end
def import_pins(zip)
records = read_json_files(zip, "data/pins")
Pin.insert_all(records) if records.any?
end
def import_reactions(zip)
records = read_json_files(zip, "data/reactions")
Reaction.insert_all(records) if records.any?
end
def import_mentions(zip)
records = read_json_files(zip, "data/mentions")
Mention.insert_all(records) if records.any?
end
def import_filters(zip)
records = read_json_files(zip, "data/filters")
Filter.insert_all(records) if records.any?
end
def import_events(zip)
records = read_json_files(zip, "data/events")
Event.insert_all(records) if records.any?
end
def import_notifications(zip)
records = read_json_files(zip, "data/notifications")
Notification.insert_all(records) if records.any?
end
def import_notification_bundles(zip)
records = read_json_files(zip, "data/notification_bundles")
Notification::Bundle.insert_all(records) if records.any?
end
def import_webhook_deliveries(zip)
records = read_json_files(zip, "data/webhook_deliveries")
Webhook::Delivery.insert_all(records) if records.any?
end
def import_boards(zip)
records = read_json_files(zip, "data/boards")
Board.insert_all(records) if records.any?
end
def import_cards(zip)
records = read_json_files(zip, "data/cards")
Card.insert_all(records) if records.any?
end
def import_comments(zip)
records = read_json_files(zip, "data/comments")
Comment.insert_all(records) if records.any?
end
def import_active_storage_blobs(zip)
records = read_json_files(zip, "data/active_storage_blobs")
ActiveStorage::Blob.insert_all(records) if records.any?
end
def import_active_storage_attachments(zip)
records = read_json_files(zip, "data/active_storage_attachments")
ActiveStorage::Attachment.insert_all(records) if records.any?
end
def import_action_text_rich_texts(zip)
records = read_json_files(zip, "data/action_text_rich_texts").map do |record|
record["body"] = convert_gids_to_sgids(record["body"])
record
end
ActionText::RichText.insert_all(records) if records.any?
end
def import_blob_files(zip)
zip.glob("storage/*").each do |entry|
key = File.basename(entry.name)
blob = ActiveStorage::Blob.find_by(key: key)
next unless blob
blob.upload(entry.get_input_stream)
end
end
def read_json_files(zip, directory)
zip.glob("#{directory}/*.json").map do |entry|
JSON.parse(entry.get_input_stream.read)
end
end
def convert_gids_to_sgids(html)
return html if html.blank?
fragment = Nokogiri::HTML.fragment(html)
fragment.css("action-text-attachment[gid]").each do |node|
gid = GlobalID.parse(node["gid"])
record = gid&.find
next unless record
node["sgid"] = record.attachable_sgid
node.remove_attribute("gid")
end
fragment.to_html
end
end
+236 -6
View File
@@ -3,19 +3,249 @@ class Account::WholeAccountExport < Account::Export
def populate_zip(zip)
export_account(zip)
export_users(zip)
export_tags(zip)
export_entropies(zip)
export_columns(zip)
export_board_publications(zip)
export_webhooks(zip)
export_webhook_delinquency_trackers(zip)
export_accesses(zip)
export_assignments(zip)
export_taggings(zip)
export_steps(zip)
export_closures(zip)
export_card_goldnesses(zip)
export_card_not_nows(zip)
export_card_activity_spikes(zip)
export_watches(zip)
export_pins(zip)
export_reactions(zip)
export_mentions(zip)
export_filters(zip)
export_events(zip)
export_notifications(zip)
export_notification_bundles(zip)
export_webhook_deliveries(zip)
export_boards(zip)
export_cards(zip)
export_comments(zip)
export_action_text_rich_texts(zip)
export_active_storage_attachments(zip)
export_active_storage_blobs(zip)
export_blob_files(zip)
end
def export_account(zip)
data = account.as_json.merge(
join_code: account.join_code.as_json,
)
add_file_to_zip(zip, "account.json", JSON.pretty_generate(data))
data = account.as_json.merge(join_code: account.join_code.as_json)
add_file_to_zip(zip, "data/account.json", JSON.pretty_generate(data))
end
def export_users(zip)
account.users.find_each do |user|
add_file_to_zip(zip, "users/#{user.id}.json", user.export_json)
add_file_to_zip(zip, "data/users/#{user.id}.json", JSON.pretty_generate(user.as_json))
end
end
def export_tags(zip)
account.tags.find_each do |tag|
add_file_to_zip(zip, "data/tags/#{tag.id}.json", JSON.pretty_generate(tag.as_json))
end
end
def export_entropies(zip)
Entropy.where(account: account).find_each do |entropy|
add_file_to_zip(zip, "data/entropies/#{entropy.id}.json", JSON.pretty_generate(entropy.as_json))
end
end
def export_columns(zip)
account.columns.find_each do |column|
add_file_to_zip(zip, "data/columns/#{column.id}.json", JSON.pretty_generate(column.as_json))
end
end
def export_board_publications(zip)
Board::Publication.where(account: account).find_each do |publication|
add_file_to_zip(zip, "data/board_publications/#{publication.id}.json", JSON.pretty_generate(publication.as_json))
end
end
def export_webhooks(zip)
Webhook.where(account: account).find_each do |webhook|
add_file_to_zip(zip, "data/webhooks/#{webhook.id}.json", JSON.pretty_generate(webhook.as_json))
end
end
def export_webhook_delinquency_trackers(zip)
Webhook::DelinquencyTracker.joins(:webhook).where(webhook: { account: account }).find_each do |tracker|
add_file_to_zip(zip, "data/webhook_delinquency_trackers/#{tracker.id}.json", JSON.pretty_generate(tracker.as_json))
end
end
def export_accesses(zip)
Access.where(account: account).find_each do |access|
add_file_to_zip(zip, "data/accesses/#{access.id}.json", JSON.pretty_generate(access.as_json))
end
end
def export_assignments(zip)
Assignment.where(account: account).find_each do |assignment|
add_file_to_zip(zip, "data/assignments/#{assignment.id}.json", JSON.pretty_generate(assignment.as_json))
end
end
def export_taggings(zip)
Tagging.where(account: account).find_each do |tagging|
add_file_to_zip(zip, "data/taggings/#{tagging.id}.json", JSON.pretty_generate(tagging.as_json))
end
end
def export_steps(zip)
Step.where(account: account).find_each do |step|
add_file_to_zip(zip, "data/steps/#{step.id}.json", JSON.pretty_generate(step.as_json))
end
end
def export_closures(zip)
Closure.where(account: account).find_each do |closure|
add_file_to_zip(zip, "data/closures/#{closure.id}.json", JSON.pretty_generate(closure.as_json))
end
end
def export_card_goldnesses(zip)
Card::Goldness.where(account: account).find_each do |goldness|
add_file_to_zip(zip, "data/card_goldnesses/#{goldness.id}.json", JSON.pretty_generate(goldness.as_json))
end
end
def export_card_not_nows(zip)
Card::NotNow.where(account: account).find_each do |not_now|
add_file_to_zip(zip, "data/card_not_nows/#{not_now.id}.json", JSON.pretty_generate(not_now.as_json))
end
end
def export_card_activity_spikes(zip)
Card::ActivitySpike.where(account: account).find_each do |activity_spike|
add_file_to_zip(zip, "data/card_activity_spikes/#{activity_spike.id}.json", JSON.pretty_generate(activity_spike.as_json))
end
end
def export_watches(zip)
Watch.where(account: account).find_each do |watch|
add_file_to_zip(zip, "data/watches/#{watch.id}.json", JSON.pretty_generate(watch.as_json))
end
end
def export_pins(zip)
Pin.where(account: account).find_each do |pin|
add_file_to_zip(zip, "data/pins/#{pin.id}.json", JSON.pretty_generate(pin.as_json))
end
end
def export_reactions(zip)
Reaction.where(account: account).find_each do |reaction|
add_file_to_zip(zip, "data/reactions/#{reaction.id}.json", JSON.pretty_generate(reaction.as_json))
end
end
def export_mentions(zip)
Mention.where(account: account).find_each do |mention|
add_file_to_zip(zip, "data/mentions/#{mention.id}.json", JSON.pretty_generate(mention.as_json))
end
end
def export_filters(zip)
Filter.where(account: account).find_each do |filter|
add_file_to_zip(zip, "data/filters/#{filter.id}.json", JSON.pretty_generate(filter.as_json))
end
end
def export_events(zip)
Event.where(account: account).find_each do |event|
add_file_to_zip(zip, "data/events/#{event.id}.json", JSON.pretty_generate(event.as_json))
end
end
def export_notifications(zip)
Notification.where(account: account).find_each do |notification|
add_file_to_zip(zip, "data/notifications/#{notification.id}.json", JSON.pretty_generate(notification.as_json))
end
end
def export_notification_bundles(zip)
Notification::Bundle.where(account: account).find_each do |bundle|
add_file_to_zip(zip, "data/notification_bundles/#{bundle.id}.json", JSON.pretty_generate(bundle.as_json))
end
end
def export_webhook_deliveries(zip)
Webhook::Delivery.where(account: account).find_each do |delivery|
add_file_to_zip(zip, "data/webhook_deliveries/#{delivery.id}.json", JSON.pretty_generate(delivery.as_json))
end
end
def export_boards(zip)
account.boards.find_each do |board|
add_file_to_zip(zip, "data/boards/#{board.id}.json", JSON.pretty_generate(board.as_json))
end
end
def export_cards(zip)
account.cards.find_each do |card|
add_file_to_zip(zip, "data/cards/#{card.id}.json", JSON.pretty_generate(card.as_json))
end
end
def export_comments(zip)
Comment.where(account: account).find_each do |comment|
add_file_to_zip(zip, "data/comments/#{comment.id}.json", JSON.pretty_generate(comment.as_json))
end
end
def export_action_text_rich_texts(zip)
ActionText::RichText.where(account: account).find_each do |rich_text|
data = rich_text.as_json.merge("body" => convert_sgids_to_gids(rich_text.body))
add_file_to_zip(zip, "data/action_text_rich_texts/#{rich_text.id}.json", JSON.pretty_generate(data))
end
end
def convert_sgids_to_gids(content)
return nil if content.blank?
content.send(:attachment_nodes).each do |node|
sgid = SignedGlobalID.parse(node["sgid"], for: ActionText::Attachable::LOCATOR_NAME)
record = sgid&.find
next if record&.account_id != account.id
node["gid"] = record.to_global_id.to_s
node.remove_attribute("sgid")
end
content.fragment.source.to_html
end
def export_active_storage_attachments(zip)
ActiveStorage::Attachment.where(account: account).find_each do |attachment|
add_file_to_zip(zip, "data/active_storage_attachments/#{attachment.id}.json", JSON.pretty_generate(attachment.as_json))
end
end
def export_active_storage_blobs(zip)
ActiveStorage::Blob.where(account: account).find_each do |blob|
add_file_to_zip(zip, "data/active_storage_blobs/#{blob.id}.json", JSON.pretty_generate(blob.as_json))
end
end
def export_blob_files(zip)
ActiveStorage::Blob.where(account: account).find_each do |blob|
add_file_to_zip(zip, "storage/#{blob.key}", compression_method: Zip::Entry::STORED) do |f|
blob.download { |chunk| f.write(chunk) }
end
rescue ActiveStorage::FileNotFoundError
# Skip blobs where the file is missing from storage
end
end
end
@@ -0,0 +1,6 @@
<h1 class="title">Your Fizzy account import is complete</h1>
<p class="subtitle">Your Fizzy account data has been successfully imported.</p>
<p><%= link_to "View your account", root_url %></p>
<p class="footer">Need help? <%= mail_to "support@fizzy.do", "Send us an email" %>.</p>
@@ -0,0 +1,3 @@
Your Fizzy account data has been successfully imported.
View your account: <%= root_url %>
@@ -0,0 +1,14 @@
class CreateAccountImports < ActiveRecord::Migration[8.2]
def change
create_table :account_imports, id: :uuid do |t|
t.uuid :identity_id, null: false
t.uuid :account_id
t.string :status, default: "pending", null: false
t.datetime :completed_at
t.timestamps
t.index :identity_id
t.index :account_id
end
end
end
Generated
+11
View File
@@ -50,6 +50,17 @@ ActiveRecord::Schema[8.2].define(version: 2026_01_21_155752) do
t.index ["value"], name: "index_account_external_id_sequences_on_value", unique: true
end
create_table "account_imports", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
t.uuid "account_id"
t.datetime "completed_at"
t.datetime "created_at", null: false
t.uuid "identity_id", null: false
t.string "status", default: "pending", null: false
t.datetime "updated_at", null: false
t.index ["account_id"], name: "index_account_imports_on_account_id"
t.index ["identity_id"], name: "index_account_imports_on_identity_id"
end
create_table "account_join_codes", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
t.uuid "account_id", null: false
t.string "code", null: false
+12
View File
@@ -38,6 +38,7 @@ ActiveRecord::Schema[8.2].define(version: 2026_01_21_155752) do
t.datetime "completed_at"
t.datetime "created_at", null: false
t.string "status", limit: 255, default: "pending", null: false
t.string "type"
t.datetime "updated_at", null: false
t.uuid "user_id", null: false
t.index ["account_id"], name: "index_account_exports_on_account_id"
@@ -49,6 +50,17 @@ ActiveRecord::Schema[8.2].define(version: 2026_01_21_155752) do
t.index ["value"], name: "index_account_external_id_sequences_on_value", unique: true
end
create_table "account_imports", id: :uuid, force: :cascade do |t|
t.uuid "account_id"
t.datetime "completed_at"
t.datetime "created_at", null: false
t.uuid "identity_id", null: false
t.string "status", limit: 255, default: "pending", null: false
t.datetime "updated_at", null: false
t.index ["account_id"], name: "index_account_imports_on_account_id"
t.index ["identity_id"], name: "index_account_imports_on_identity_id"
end
create_table "account_join_codes", id: :uuid, force: :cascade do |t|
t.uuid "account_id", null: false
t.string "code", limit: 255, null: false