From f50d6d093df48a7b766595119285b6a22f94bd2b Mon Sep 17 00:00:00 2001 From: "Stanko K.R." Date: Thu, 29 Jan 2026 14:46:40 +0100 Subject: [PATCH] Implement cursors for imports --- app/jobs/import_account_data_job.rb | 8 ++- app/mailers/import_mailer.rb | 7 ++- app/models/account/data_transfer/manifest.rb | 55 ++++++++----------- .../account/data_transfer/record_set.rb | 20 ++++++- app/models/account/import.rb | 11 ++-- app/models/export.rb | 9 ++- .../mailers/import_mailer/completed.html.erb | 4 +- .../mailers/import_mailer/completed.text.erb | 4 +- test/models/account/import_test.rb | 6 +- 9 files changed, 70 insertions(+), 54 deletions(-) diff --git a/app/jobs/import_account_data_job.rb b/app/jobs/import_account_data_job.rb index e131a41e9..863aa111f 100644 --- a/app/jobs/import_account_data_job.rb +++ b/app/jobs/import_account_data_job.rb @@ -7,13 +7,17 @@ class ImportAccountDataJob < ApplicationJob step :validate do import.validate \ start: step.cursor, - callback: proc { |record_set:, record_id:| step.set! [ record_set, record_id ] } + callback: proc do |record_set:, file:| + step.set!([ record_set.model.name, file ]) + end end step :process do import.process \ start: step.cursor, - callback: proc { |record_set:, record_id:| step.set! [ record_set, record_id ] } + callback: proc do |record_set:, files:| + step.set!([ record_set.model.name, files.last ]) + end end end end diff --git a/app/mailers/import_mailer.rb b/app/mailers/import_mailer.rb index 5040a224e..df557ad05 100644 --- a/app/mailers/import_mailer.rb +++ b/app/mailers/import_mailer.rb @@ -1,9 +1,12 @@ class ImportMailer < ApplicationMailer - def completed(identity) + def completed(identity, account) + @account = account + @landing_url = landing_url(script_name: account.slug) mail to: identity.email_address, subject: "Your Fizzy account import is complete" end - def failed(identity) + def failed(identity, account) + @account = account mail to: identity.email_address, subject: "Your Fizzy account import failed" end end diff --git a/app/models/account/data_transfer/manifest.rb b/app/models/account/data_transfer/manifest.rb index e764f69c1..33b7e1e93 100644 --- a/app/models/account/data_transfer/manifest.rb +++ b/app/models/account/data_transfer/manifest.rb @@ -1,6 +1,4 @@ class Account::DataTransfer::Manifest - Cursor = Struct.new(:record_class, :last_id) - attr_reader :account def initialize(account) @@ -10,8 +8,16 @@ class Account::DataTransfer::Manifest def each_record_set(start: nil) raise ArgumentError, "No block given" unless block_given? + started = start.nil? + record_class, last_id = start if start + record_sets.each do |record_set| - yield record_set + if started + yield record_set + elsif record_set.model.name == record_class + started = true + yield record_set, last_id + end end end @@ -20,37 +26,24 @@ class Account::DataTransfer::Manifest [ Account::DataTransfer::AccountRecordSet.new(account), Account::DataTransfer::UserRecordSet.new(account), - Account::DataTransfer::RecordSet.new(account: account, model: ::User::Settings), - Account::DataTransfer::RecordSet.new(account: account, model: ::Tag), - Account::DataTransfer::RecordSet.new(account: account, model: ::Board), - Account::DataTransfer::RecordSet.new(account: account, model: ::Column), + *build_record_sets(::User::Settings, ::Tag, ::Board, ::Column), Account::DataTransfer::EntropyRecordSet.new(account), - Account::DataTransfer::RecordSet.new(account: account, model: ::Board::Publication), - Account::DataTransfer::RecordSet.new(account: account, model: ::Webhook), - Account::DataTransfer::RecordSet.new(account: account, model: ::Access), - Account::DataTransfer::RecordSet.new(account: account, model: ::Card), - Account::DataTransfer::RecordSet.new(account: account, model: ::Comment), - Account::DataTransfer::RecordSet.new(account: account, model: ::Step), - Account::DataTransfer::RecordSet.new(account: account, model: ::Assignment), - Account::DataTransfer::RecordSet.new(account: account, model: ::Tagging), - Account::DataTransfer::RecordSet.new(account: account, model: ::Closure), - Account::DataTransfer::RecordSet.new(account: account, model: ::Card::Goldness), - Account::DataTransfer::RecordSet.new(account: account, model: ::Card::NotNow), - Account::DataTransfer::RecordSet.new(account: account, model: ::Card::ActivitySpike), - Account::DataTransfer::RecordSet.new(account: account, model: ::Watch), - Account::DataTransfer::RecordSet.new(account: account, model: ::Pin), - Account::DataTransfer::RecordSet.new(account: account, model: ::Reaction), - Account::DataTransfer::RecordSet.new(account: account, model: ::Mention), - Account::DataTransfer::RecordSet.new(account: account, model: ::Filter), - Account::DataTransfer::RecordSet.new(account: account, model: ::Webhook::DelinquencyTracker), - Account::DataTransfer::RecordSet.new(account: account, model: ::Event), - Account::DataTransfer::RecordSet.new(account: account, model: ::Notification), - Account::DataTransfer::RecordSet.new(account: account, model: ::Notification::Bundle), - Account::DataTransfer::RecordSet.new(account: account, model: ::Webhook::Delivery), - Account::DataTransfer::RecordSet.new(account: account, model: ::ActiveStorage::Blob), - Account::DataTransfer::RecordSet.new(account: account, model: ::ActiveStorage::Attachment), + *build_record_sets( + ::Board::Publication, ::Webhook, ::Access, ::Card, ::Comment, ::Step, + ::Assignment, ::Tagging, ::Closure, ::Card::Goldness, ::Card::NotNow, + ::Card::ActivitySpike, ::Watch, ::Pin, ::Reaction, ::Mention, + ::Filter, ::Webhook::DelinquencyTracker, ::Event, + ::Notification, ::Notification::Bundle, ::Webhook::Delivery, + ::ActiveStorage::Blob, ::ActiveStorage::Attachment + ), Account::DataTransfer::ActionTextRichTextRecordSet.new(account), Account::DataTransfer::BlobFileRecordSet.new(account) ] end + + def build_record_sets(*models) + models.map do |model| + Account::DataTransfer::RecordSet.new(account: account, model: model) + end + end end diff --git a/app/models/account/data_transfer/record_set.rb b/app/models/account/data_transfer/record_set.rb index b3fd9d5c1..61359b0fd 100644 --- a/app/models/account/data_transfer/record_set.rb +++ b/app/models/account/data_transfer/record_set.rb @@ -23,7 +23,10 @@ class Account::DataTransfer::RecordSet def import(from:, start: nil, callback: nil) with_zip(from) do - files.each_slice(IMPORT_BATCH_SIZE) do |file_batch| + file_list = files + file_list = skip_to(file_list, start) if start + + file_list.each_slice(IMPORT_BATCH_SIZE) do |file_batch| import_batch(file_batch) callback&.call(record_set: self, files: file_batch) end @@ -32,7 +35,10 @@ class Account::DataTransfer::RecordSet def validate(from:, start: nil, callback: nil) with_zip(from) do - files.each do |file_path| + file_list = files + file_list = skip_to(file_list, start) if start + + file_list.each do |file_path| validate_record(file_path) callback&.call(record_set: self, file: file_path) end @@ -99,6 +105,16 @@ class Account::DataTransfer::RecordSet end end + def skip_to(file_list, last_id) + index = file_list.index(last_id) + + if index + file_list[(index + 1)..] + else + file_list + end + end + def load(file_path) JSON.parse(zip.read(file_path)) rescue ArgumentError => e diff --git a/app/models/account/import.rb b/app/models/account/import.rb index 1e98ce2e1..5ebb1288b 100644 --- a/app/models/account/import.rb +++ b/app/models/account/import.rb @@ -17,14 +17,15 @@ class Account::Import < ApplicationRecord ensure_downloaded Account::DataTransfer::ZipFile.open(download_path) do |zip| - Account::DataTransfer::Manifest.new(account).each_record_set(start: start&.record_set) do |record_set| - record_set.import(from: zip, start: start&.record_id, callback: callback) + Account::DataTransfer::Manifest.new(account).each_record_set(start: start) do |record_set, last_id| + record_set.import(from: zip, start: last_id, callback: callback) end end mark_completed rescue => e failed! + ImportMailer.failed(identity, account).deliver_later raise e end @@ -33,8 +34,8 @@ class Account::Import < ApplicationRecord ensure_downloaded Account::DataTransfer::ZipFile.open(download_path) do |zip| - Account::DataTransfer::Manifest.new(account).each_record_set(start: start&.record_set) do |record_set| - record_set.validate(from: zip, start: start&.record_id, callback: callback) + Account::DataTransfer::Manifest.new(account).each_record_set(start: start) do |record_set, last_id| + record_set.validate(from: zip, start: last_id, callback: callback) end end end @@ -54,6 +55,6 @@ class Account::Import < ApplicationRecord def mark_completed completed! - # TODO: send email + ImportMailer.completed(identity, account).deliver_later end end diff --git a/app/models/export.rb b/app/models/export.rb index 9986f7539..9560fe122 100644 --- a/app/models/export.rb +++ b/app/models/export.rb @@ -20,20 +20,19 @@ class Export < ApplicationRecord def build processing! + zipfile = nil with_account_context do zipfile = generate_zip { |zip| populate_zip(zip) } - file.attach io: File.open(zipfile.path), filename: "fizzy-export-#{id}.zip", content_type: "application/zip" mark_completed - ExportMailer.completed(self).deliver_later - ensure - zipfile&.close - zipfile&.unlink end rescue => e update!(status: :failed) raise + ensure + zipfile&.close + zipfile&.unlink end def mark_completed diff --git a/app/views/mailers/import_mailer/completed.html.erb b/app/views/mailers/import_mailer/completed.html.erb index 914a182d7..b598ef0be 100644 --- a/app/views/mailers/import_mailer/completed.html.erb +++ b/app/views/mailers/import_mailer/completed.html.erb @@ -1,6 +1,6 @@

Your Fizzy account import is complete

-

Your Fizzy account data has been successfully imported.

+

Your import to <%= @account.name %> is complete.

-

<%= link_to "View your account", root_url %>

+

<%= link_to "Go to your account", @landing_url %>

Need help? <%= mail_to "support@fizzy.do", "Send us an email" %>.

diff --git a/app/views/mailers/import_mailer/completed.text.erb b/app/views/mailers/import_mailer/completed.text.erb index 704df0f6f..df9f0fd54 100644 --- a/app/views/mailers/import_mailer/completed.text.erb +++ b/app/views/mailers/import_mailer/completed.text.erb @@ -1,3 +1,3 @@ -Your Fizzy account data has been successfully imported. +Your import to <%= @account.name %> is complete. -View your account: <%= root_url %> +Go to your account: <%= @landing_url %> diff --git a/test/models/account/import_test.rb b/test/models/account/import_test.rb index 2d357da1f..1a2f8fb99 100644 --- a/test/models/account/import_test.rb +++ b/test/models/account/import_test.rb @@ -67,7 +67,7 @@ class Account::ImportTest < ActiveSupport::TestCase target_account = create_target_account # Pre-create a tag with a specific ID that will collide - colliding_id = SecureRandom.uuid + colliding_id = ActiveRecord::Type::Uuid.generate Tag.create!( id: colliding_id, account: target_account, @@ -189,7 +189,7 @@ class Account::ImportTest < ActiveSupport::TestCase # Export users with new UUIDs (to avoid collisions with fixtures) @source_account.users.each do |user| - new_id = SecureRandom.uuid + new_id = ActiveRecord::Type::Uuid.generate user_data = { "id" => new_id, "account_id" => @source_account.id, @@ -206,7 +206,7 @@ class Account::ImportTest < ActiveSupport::TestCase # Export tags with new UUIDs (to avoid collisions with fixtures) @source_account.tags.each do |tag| - new_id = tag_id || SecureRandom.uuid + new_id = tag_id || ActiveRecord::Type::Uuid.generate tag_data = { "id" => new_id, "account_id" => @source_account.id,