bterkuile/fizzy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement cursors for imports

Browse Source
This commit is contained in:
Stanko K.R.
2026-01-29 14:46:40 +01:00
parent 63f6be4ef5
commit f50d6d093d
9 changed files with 70 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/jobs/import_account_data_job.rb
+6 -2
View File
@@ -7,13 +7,17 @@ class ImportAccountDataJob < ApplicationJob
step :validate do
import.validate \
start: step.cursor,
callback: proc { |record_set:, record_id:| step.set! [ record_set, record_id ] }
callback: proc do |record_set:, file:|
step.set!([ record_set.model.name, file ])
end
end
step :process do
import.process \
start: step.cursor,
callback: proc { |record_set:, record_id:| step.set! [ record_set, record_id ] }
callback: proc do |record_set:, files:|
step.set!([ record_set.model.name, files.last ])
end
end
end
end
app/mailers/import_mailer.rb
+5 -2
View File
@@ -1,9 +1,12 @@
class ImportMailer < ApplicationMailer
def completed(identity)
def completed(identity, account)
@account = account
@landing_url = landing_url(script_name: account.slug)
mail to: identity.email_address, subject: "Your Fizzy account import is complete"
end
def failed(identity)
def failed(identity, account)
@account = account
mail to: identity.email_address, subject: "Your Fizzy account import failed"
end
end
app/models/account/data_transfer/manifest.rb
+24 -31
View File
@@ -1,6 +1,4 @@
class Account::DataTransfer::Manifest
Cursor = Struct.new(:record_class, :last_id)
attr_reader :account
def initialize(account)
@@ -10,8 +8,16 @@ class Account::DataTransfer::Manifest
def each_record_set(start: nil)
raise ArgumentError, "No block given" unless block_given?
started = start.nil?
record_class, last_id = start if start
record_sets.each do |record_set|
yield record_set
if started
yield record_set
elsif record_set.model.name == record_class
started = true
yield record_set, last_id
end
end
end
@@ -20,37 +26,24 @@ class Account::DataTransfer::Manifest
[
Account::DataTransfer::AccountRecordSet.new(account),
Account::DataTransfer::UserRecordSet.new(account),
Account::DataTransfer::RecordSet.new(account: account, model: ::User::Settings),
Account::DataTransfer::RecordSet.new(account: account, model: ::Tag),
Account::DataTransfer::RecordSet.new(account: account, model: ::Board),
Account::DataTransfer::RecordSet.new(account: account, model: ::Column),
*build_record_sets(::User::Settings, ::Tag, ::Board, ::Column),
Account::DataTransfer::EntropyRecordSet.new(account),
Account::DataTransfer::RecordSet.new(account: account, model: ::Board::Publication),
Account::DataTransfer::RecordSet.new(account: account, model: ::Webhook),
Account::DataTransfer::RecordSet.new(account: account, model: ::Access),
Account::DataTransfer::RecordSet.new(account: account, model: ::Card),
Account::DataTransfer::RecordSet.new(account: account, model: ::Comment),
Account::DataTransfer::RecordSet.new(account: account, model: ::Step),
Account::DataTransfer::RecordSet.new(account: account, model: ::Assignment),
Account::DataTransfer::RecordSet.new(account: account, model: ::Tagging),
Account::DataTransfer::RecordSet.new(account: account, model: ::Closure),
Account::DataTransfer::RecordSet.new(account: account, model: ::Card::Goldness),
Account::DataTransfer::RecordSet.new(account: account, model: ::Card::NotNow),
Account::DataTransfer::RecordSet.new(account: account, model: ::Card::ActivitySpike),
Account::DataTransfer::RecordSet.new(account: account, model: ::Watch),
Account::DataTransfer::RecordSet.new(account: account, model: ::Pin),
Account::DataTransfer::RecordSet.new(account: account, model: ::Reaction),
Account::DataTransfer::RecordSet.new(account: account, model: ::Mention),
Account::DataTransfer::RecordSet.new(account: account, model: ::Filter),
Account::DataTransfer::RecordSet.new(account: account, model: ::Webhook::DelinquencyTracker),
Account::DataTransfer::RecordSet.new(account: account, model: ::Event),
Account::DataTransfer::RecordSet.new(account: account, model: ::Notification),
Account::DataTransfer::RecordSet.new(account: account, model: ::Notification::Bundle),
Account::DataTransfer::RecordSet.new(account: account, model: ::Webhook::Delivery),
Account::DataTransfer::RecordSet.new(account: account, model: ::ActiveStorage::Blob),
Account::DataTransfer::RecordSet.new(account: account, model: ::ActiveStorage::Attachment),
*build_record_sets(
::Board::Publication, ::Webhook, ::Access, ::Card, ::Comment, ::Step,
::Assignment, ::Tagging, ::Closure, ::Card::Goldness, ::Card::NotNow,
::Card::ActivitySpike, ::Watch, ::Pin, ::Reaction, ::Mention,
::Filter, ::Webhook::DelinquencyTracker, ::Event,
::Notification, ::Notification::Bundle, ::Webhook::Delivery,
::ActiveStorage::Blob, ::ActiveStorage::Attachment
),
Account::DataTransfer::ActionTextRichTextRecordSet.new(account),
Account::DataTransfer::BlobFileRecordSet.new(account)
]
end
def build_record_sets(*models)
models.map do |model|
Account::DataTransfer::RecordSet.new(account: account, model: model)
end
end
end
app/models/account/data_transfer/record_set.rb
+18 -2
View File
@@ -23,7 +23,10 @@ class Account::DataTransfer::RecordSet
def import(from:, start: nil, callback: nil)
with_zip(from) do
files.each_slice(IMPORT_BATCH_SIZE) do |file_batch|
file_list = files
file_list = skip_to(file_list, start) if start
file_list.each_slice(IMPORT_BATCH_SIZE) do |file_batch|
import_batch(file_batch)
callback&.call(record_set: self, files: file_batch)
end
@@ -32,7 +35,10 @@ class Account::DataTransfer::RecordSet
def validate(from:, start: nil, callback: nil)
with_zip(from) do
files.each do |file_path|
file_list = files
file_list = skip_to(file_list, start) if start
file_list.each do |file_path|
validate_record(file_path)
callback&.call(record_set: self, file: file_path)
end
@@ -99,6 +105,16 @@ class Account::DataTransfer::RecordSet
end
end
def skip_to(file_list, last_id)
index = file_list.index(last_id)
if index
file_list[(index + 1)..]
else
file_list
end
end
def load(file_path)
JSON.parse(zip.read(file_path))
rescue ArgumentError => e
app/models/account/import.rb
+6 -5
View File
@@ -17,14 +17,15 @@ class Account::Import < ApplicationRecord
ensure_downloaded
Account::DataTransfer::ZipFile.open(download_path) do |zip|
Account::DataTransfer::Manifest.new(account).each_record_set(start: start&.record_set) do |record_set|
record_set.import(from: zip, start: start&.record_id, callback: callback)
Account::DataTransfer::Manifest.new(account).each_record_set(start: start) do |record_set, last_id|
record_set.import(from: zip, start: last_id, callback: callback)
end
end
mark_completed
rescue => e
failed!
ImportMailer.failed(identity, account).deliver_later
raise e
end
@@ -33,8 +34,8 @@ class Account::Import < ApplicationRecord
ensure_downloaded
Account::DataTransfer::ZipFile.open(download_path) do |zip|
Account::DataTransfer::Manifest.new(account).each_record_set(start: start&.record_set) do |record_set|
record_set.validate(from: zip, start: start&.record_id, callback: callback)
Account::DataTransfer::Manifest.new(account).each_record_set(start: start) do |record_set, last_id|
record_set.validate(from: zip, start: last_id, callback: callback)
end
end
end
@@ -54,6 +55,6 @@ class Account::Import < ApplicationRecord
def mark_completed
completed!
# TODO: send email
ImportMailer.completed(identity, account).deliver_later
end
end
app/models/export.rb
+4 -5
View File
@@ -20,20 +20,19 @@ class Export < ApplicationRecord
def build
processing!
zipfile = nil
with_account_context do
zipfile = generate_zip { |zip| populate_zip(zip) }
file.attach io: File.open(zipfile.path), filename: "fizzy-export-#{id}.zip", content_type: "application/zip"
mark_completed
ExportMailer.completed(self).deliver_later
ensure
zipfile&.close
zipfile&.unlink
end
rescue => e
update!(status: :failed)
raise
ensure
zipfile&.close
zipfile&.unlink
end
def mark_completed
app/views/mailers/import_mailer/completed.html.erb
+2 -2
View File
@@ -1,6 +1,6 @@
<h1 class="title">Your Fizzy account import is complete</h1>
<p class="subtitle">Your Fizzy account data has been successfully imported.</p>
<p class="subtitle">Your import to <%= @account.name %> is complete.</p>
<p><%= link_to "View your account", root_url %></p>
<p><%= link_to "Go to your account", @landing_url %></p>
<p class="footer">Need help? <%= mail_to "support@fizzy.do", "Send us an email" %>.</p>
app/views/mailers/import_mailer/completed.text.erb
+2 -2
View File
@@ -1,3 +1,3 @@
Your Fizzy account data has been successfully imported.
Your import to <%= @account.name %> is complete.
View your account: <%= root_url %>
Go to your account: <%= @landing_url %>
test/models/account/import_test.rb
+3 -3
View File
@@ -67,7 +67,7 @@ class Account::ImportTest < ActiveSupport::TestCase
target_account = create_target_account
# Pre-create a tag with a specific ID that will collide
colliding_id = SecureRandom.uuid
colliding_id = ActiveRecord::Type::Uuid.generate
Tag.create!(
id: colliding_id,
account: target_account,
@@ -189,7 +189,7 @@ class Account::ImportTest < ActiveSupport::TestCase
# Export users with new UUIDs (to avoid collisions with fixtures)
@source_account.users.each do |user|
new_id = SecureRandom.uuid
new_id = ActiveRecord::Type::Uuid.generate
user_data = {
"id" => new_id,
"account_id" => @source_account.id,
@@ -206,7 +206,7 @@ class Account::ImportTest < ActiveSupport::TestCase
# Export tags with new UUIDs (to avoid collisions with fixtures)
@source_account.tags.each do |tag|
new_id = tag_id || SecureRandom.uuid
new_id = tag_id || ActiveRecord::Type::Uuid.generate
tag_data = {
"id" => new_id,
"account_id" => @source_account.id,