diff --git a/AGENTS.md b/AGENTS.md index 80bb887a2..d1c837079 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,6 +137,13 @@ Key recurring tasks (via `config/recurring.yml`): - Search records denormalized for performance - Models in `app/models/search/` +### Imports and exports + +Allow people to move between OSS and SAAS Fizzy instances: +- Exports/Imports can be wirtten to/read from local or S3 storage depending on the config of the instance (both myst be supported) +- Must be able to handle very large ZIP files (500+GB) +- Models in `app/models/account/data_transfer/`, `app/models/zip_file` + ## Tools ### Chrome MCP (Local Dev) diff --git a/app/models/zip_file.rb b/app/models/zip_file.rb index 2f135fd8d..8c8299bc9 100644 --- a/app/models/zip_file.rb +++ b/app/models/zip_file.rb @@ -39,9 +39,15 @@ class ZipFile ) writer = Writer.new - service.upload(blob.key, writer.io) do |io| - writer.stream_to(io) + + # Use S3's upload_stream directly for write-based streaming. + # ActiveStorage's upload method expects a read-based IO, but ZipKit + # needs a write-based stream. The TransferManager's upload_stream + # yields a writable IO that we can stream directly to. + service.send(:upload_stream, key: blob.key, part_size: 100.megabytes) do |write_stream| + writer.stream_to(write_stream) yield writer + writer.close end blob.update!(byte_size: writer.byte_size, checksum: writer.checksum) diff --git a/app/models/zip_file/writer.rb b/app/models/zip_file/writer.rb index 8e9665563..8291c4948 100644 --- a/app/models/zip_file/writer.rb +++ b/app/models/zip_file/writer.rb @@ -9,10 +9,6 @@ class ZipFile::Writer @digest = Digest::MD5.new end - def io - @io ||= ZipFile::Writer::IO.new(self) - end - def stream_to(io) @output_io = io end diff --git a/app/models/zip_file/writer/io.rb b/app/models/zip_file/writer/io.rb deleted file mode 100644 index 451dd0d07..000000000 --- a/app/models/zip_file/writer/io.rb +++ /dev/null @@ -1,18 +0,0 @@ -class ZipFile::Writer::IO - def initialize(writer) - @writer = writer - end - - def write(data) - @writer.write(data) - end - - def <<(data) - write(data) - self - end - - def size - @writer.byte_size - end -end diff --git a/test/models/zip_file_test.rb b/test/models/zip_file_test.rb index 257b9adf4..42db30f9e 100644 --- a/test/models/zip_file_test.rb +++ b/test/models/zip_file_test.rb @@ -1,13 +1,6 @@ require "test_helper" class ZipFileTest < ActiveSupport::TestCase - test "writer io exposes size for S3 multipart upload detection" do - writer = ZipFile::Writer.new - - assert_respond_to writer.io, :size - assert_equal writer.byte_size, writer.io.size - end - test "writer adds files with content" do tempfile = Tempfile.new([ "test", ".zip" ]) tempfile.binmode