Merge pull request #1768 from basecamp/data-exports

Add "data export" feature

Gemfile

@@ -33,6 +33,7 @@ gem "platform_agent"
 gem "aws-sdk-s3", require: false
 gem "web-push"
 gem "net-http-persistent"
+gem "rubyzip", require: "zip"
 gem "mittens"
 gem "useragent", bc: "useragent"
 

Gemfile.lock

@@ -503,6 +503,7 @@ DEPENDENCIES
   rouge
   rqrcode
   rubocop-rails-omakase
+  rubyzip
   selenium-webdriver
   solid_cable (>= 3.0)
   solid_cache (~> 1.0)

Gemfile.saas.lock

@@ -643,6 +643,7 @@ DEPENDENCIES
   rouge
   rqrcode
   rubocop-rails-omakase
+  rubyzip
   selenium-webdriver
   sentry-rails
   sentry-ruby

app/controllers/account/exports_controller.rb

@@ -0,0 +1,23 @@
+class Account::ExportsController < ApplicationController
+  before_action :ensure_export_limit_not_exceeded, only: :create
+  before_action :set_export, only: :show
+
+  CURRENT_EXPORT_LIMIT = 10
+
+  def show
+  end
+
+  def create
+    Current.account.exports.create!(user: Current.user).build_later
+    redirect_to account_settings_path, notice: "Export started. You'll receive an email when it's ready."
+  end
+
+  private
+    def ensure_export_limit_not_exceeded
+      head :too_many_requests if Current.user.exports.current.count >= CURRENT_EXPORT_LIMIT
+    end
+
+    def set_export
+      @export = Current.account.exports.completed.find_by(id: params[:id], user: Current.user)
+    end
+end

app/javascript/controllers/auto_click_controller.js

@@ -0,0 +1,7 @@
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+  connect() {
+    this.element.click()
+  }
+}

app/jobs/export_account_data_job.rb

@@ -0,0 +1,7 @@
+class ExportAccountDataJob < ApplicationJob
+  queue_as :backend
+
+  def perform(export)
+    export.build
+  end
+end

app/mailers/export_mailer.rb

@@ -0,0 +1,8 @@
+class ExportMailer < ApplicationMailer
+  def completed(export)
+    @export = export
+    @user = export.user
+
+    mail to: @user.identity.email_address, subject: "Your Fizzy data export is ready for download"
+  end
+end

app/models/account.rb

@@ -8,6 +8,7 @@ class Account < ApplicationRecord
   has_many :webhooks, dependent: :destroy
   has_many :tags, dependent: :destroy
   has_many :columns, dependent: :destroy
+  has_many :exports, class_name: "Account::Export", dependent: :destroy
 
   has_many_attached :uploads
 

app/models/account/export.rb

@@ -0,0 +1,73 @@
+class Account::Export < ApplicationRecord
+  belongs_to :account
+  belongs_to :user
+
+  has_one_attached :file
+
+  enum :status, %w[ pending processing completed failed ].index_by(&:itself), default: :pending
+
+  scope :current, -> { where(created_at: 24.hours.ago..) }
+  scope :expired, -> { where(completed_at: ...24.hours.ago) }
+
+  def self.cleanup
+    expired.destroy_all
+  end
+
+  def build_later
+    ExportAccountDataJob.perform_later(self)
+  end
+
+  def build
+    processing!
+    zipfile = generate_zip
+
+    file.attach io: File.open(zipfile.path), filename: "fizzy-export-#{id}.zip", content_type: "application/zip"
+    mark_completed
+
+    ExportMailer.completed(self).deliver_later
+  rescue => e
+    update!(status: :failed)
+    raise
+  ensure
+    zipfile&.close
+    zipfile&.unlink
+  end
+
+  def mark_completed
+    update!(status: :completed, completed_at: Time.current)
+  end
+
+  private
+    def generate_zip
+      Tempfile.new([ "export", ".zip" ]).tap do |tempfile|
+        Zip::File.open(tempfile.path, create: true) do |zip|
+          exportable_cards.find_each do |card|
+            add_card_to_zip(zip, card)
+          end
+        end
+      end
+    end
+
+    def exportable_cards
+      user.accessible_cards.includes(
+        :board,
+        creator: :identity,
+        comments: { creator: :identity },
+        rich_text_description: { embeds_attachments: :blob }
+      )
+    end
+
+    def add_card_to_zip(zip, card)
+      zip.get_output_stream("#{card.number}.json") do |f|
+        f.write(card.export_json)
+      end
+
+      card.export_attachments.each do |attachment|
+        zip.get_output_stream(attachment[:path]) do |f|
+          attachment[:blob].download { |chunk| f.write(chunk) }
+        end
+      rescue ActiveStorage::FileNotFoundError
+        # Skip attachments where the file is missing from storage
+      end
+    end
+end

app/models/card.rb

@@ -1,6 +1,6 @@
 class Card < ApplicationRecord
   include Assignable, Attachments, Broadcastable, Closeable, Colored, Entropic, Eventable,
-    Golden, Mentions, Multistep, Pinnable, Postponable, Promptable,
+    Exportable, Golden, Mentions, Multistep, Pinnable, Postponable, Promptable,
     Readable, Searchable, Stallable, Statuses, Taggable, Triageable, Watchable
 
   belongs_to :account, default: -> { board.account }

app/models/card/exportable.rb

@@ -0,0 +1,85 @@
+module Card::Exportable
+  extend ActiveSupport::Concern
+  include ActionView::Helpers::TagHelper
+
+  def export_json
+    JSON.pretty_generate({
+      number: number,
+      title: title,
+      board: board.name,
+      status: export_status,
+      creator: export_user(creator),
+      description: export_html(description),
+      created_at: created_at.iso8601,
+      updated_at: updated_at.iso8601,
+      comments: comments.chronologically.map do |comment|
+        {
+          id: comment.id,
+          body: export_html(comment.body),
+          creator: export_user(comment.creator),
+          created_at: comment.created_at.iso8601
+        }
+      end
+    })
+  end
+
+  def export_attachments
+    collect_attachments.map do |attachment|
+      { path: export_attachment_path(attachment.blob), blob: attachment.blob }
+    end
+  end
+
+  private
+    def export_html(rich_text)
+      return "" if rich_text.blank?
+
+      rich_text.body.render_attachments do |attachment|
+        attachment_representation(attachment)
+      end.to_html
+    end
+
+    def attachment_representation(attachment)
+      case attachable = attachment.attachable
+      when ActiveStorage::Blob
+        path = export_attachment_path(attachable)
+        if attachable.image?
+          tag.img(src: path, alt: attachable.filename)
+        else
+          tag.a(attachable.filename, href: path)
+        end
+      when ActionText::Attachables::RemoteImage
+        tag.img(src: attachable.url, alt: "Remote image")
+      else
+        attachment.to_html
+      end
+    end
+
+    def export_user(user)
+      {
+        id: user.id,
+        name: user.name,
+        email: user.identity&.email_address
+      }
+    end
+
+    def export_attachment_path(blob)
+      "#{number}/#{blob.key}_#{blob.filename}"
+    end
+
+    def collect_attachments
+      attachments.to_a + comments.flat_map { |c| c.attachments.to_a }
+    end
+
+    def export_status
+      case
+      when closed?
+        "Done"
+      when postponed?
+        "Not now"
+      when column.present?
+        column.name
+      else
+        "Maybe?"
+      end
+    end
+end

app/models/user.rb

@@ -16,6 +16,7 @@ class User < ApplicationRecord
   has_many :closures, dependent: :nullify
   has_many :pins, dependent: :destroy
   has_many :pinned_cards, through: :pins, source: :card
+  has_many :exports, class_name: "Account::Export", dependent: :destroy
 
   scope :with_avatars, -> { preload(:account, :avatar_attachment) }
 

app/views/account/exports/show.html.erb

@@ -0,0 +1,29 @@
+<% if @export.present? %>
+  <% @page_title = "Download Export" %>
+<% else %>
+  <% @page_title = "Download Expired" %>
+<% end %>
+
+<% content_for :header do %>
+  <div class="header__actions header__actions--start">
+    <%= back_link_to "Account Settings", account_settings_path, "keydown.left@document->hotkey#click keydown.esc@document->hotkey#click" %>
+  </div>
+<% end %>
+
+<div class="panel panel--wide shadow center flex flex-column gap">
+  <h2 class="txt-large margin-none font-weight-black"><%= @page_title %></h2>
+
+  <% if @export.present? %>
+    <p class="margin-none-block-start">Your export is ready. The download should start automatically.</p>
+
+    <%= link_to rails_blob_path(@export.file, disposition: "attachment"),
+          id: "download-link",
+          class: "btn btn--link",
+          data: { turbo: false, controller: "auto-click" } do %>
+      Download your data
+    <% end %>
+  <% else %>
+    <p class="margin-none-block-start">That download link has expired. You’ll need to <%= link_to "request a new export", account_settings_path, class: "txt-lnk" %>.</p>
+  <% end %>
+
+</div>

app/views/account/settings/_entropy.html.erb

@@ -1,8 +1,7 @@
-<div class="settings__panel settings__panel--entropy panel shadow center">
-  <header>
-    <h2 class="divider txt-large">Auto close</h2>
-    <p class="margin-none-block-start">Fizzy doesn’t let stale cards stick around forever. Cards automatically move to “Not Now” if there is no activity for specific period of time. <em>This is the default, global setting — you can override it on each board.</em></p>
-  </header>
+<header>
+  <h2 class="divider txt-large">Auto close</h2>
+  <p class="margin-none-block-start">Fizzy doesn’t let stale cards stick around forever. Cards automatically move to “Not Now” if there is no activity for specific period of time. <em>This is the default, global setting — you can override it on each board.</em></p>
+</header>
+
+<%= render "entropy/auto_close", model: account.entropy, url: account_entropy_path, disabled: !Current.user.admin? %>
 
-  <%= render "entropy/auto_close", model: account.entropy, url: account_entropy_path, disabled: !Current.user.admin? %>
-</div>

app/views/account/settings/_export.html.erb

@@ -0,0 +1,19 @@
+<header class="margin-block-start-double">
+  <h2 class="divider txt-large">Export your data</h2>
+  <p class="margin-none-block">Download an archive of your Fizzy data.</p>
+</header>
+
+<div data-controller="dialog" data-dialog-modal-value="true">
+  <button type="button" class="btn" data-action="dialog#open">Begin export...</button>
+
+  <dialog class="dialog panel panel--wide shadow" data-dialog-target="dialog">
+    <h2 class="margin-none txt-large">Export your account data</h2>
+    <p class="margin-none-block-start">This will kick off a request to generate a ZIP archive of all the data in boards you have access to.</p>
+    <p>When the file is ready, we’ll email you a link to download it. The link will expire after 24 hours.</p>
+
+    <div class="flex gap justify-center">
+      <%= button_to "Start export", account_exports_path, method: :post, class: "btn btn--link", form: { data: { action: "submit->dialog#close" } } %>
+      <button type="button" class="btn" data-action="dialog#close">Cancel</button>
+    </div>
+  </dialog>
+</div>

app/views/account/settings/show.html.erb

@@ -15,5 +15,8 @@
     <%= render "account/settings/users", users: @users %>
   </div>
 
-  <%= render "account/settings/entropy", account: @account %>
+  <div class="settings__panel settings__panel--entropy panel shadow center">
+    <%= render "account/settings/entropy", account: @account %>
+    <%= render "account/settings/export" %>
+  </div>
 </section>

app/views/mailers/export_mailer/completed.html.erb

@@ -0,0 +1,6 @@
+<h1 class="title">Download your Fizzy data</h1>
+<p class="subtitle">Your Fizzy data export has finished processing and is ready to download.</p>
+
+<p><%= link_to "Download your data", account_export_url(@export) %></p>
+
+<p class="footer">Need help? <%= mail_to "support@fizzy.do", "Send us an email" %>.</p>

app/views/mailers/export_mailer/completed.text.erb

@@ -0,0 +1,3 @@
+Your Fizzy data export has finished processing and is ready to download.
+
+Download your data: <%= account_export_url(@export) %>

config/recurring.yml

@@ -27,6 +27,9 @@ production: &production
   cleanup_magic_links:
     command: "MagicLink.cleanup"
     schedule: every 4 hours
+  cleanup_exports:
+    command: "Account::Export.cleanup"
+    schedule: every hour at minute 20
 
 <% if Fizzy.saas? %>
   # Metrics

config/routes.rb

@@ -5,6 +5,7 @@ Rails.application.routes.draw do
     resource :join_code
     resource :settings
     resource :entropy
+    resources :exports, only: [ :create, :show ]
   end
 
   resources :users do

db/migrate/20251201100607_create_account_exports.rb

@@ -0,0 +1,14 @@
+class CreateAccountExports < ActiveRecord::Migration[8.2]
+  def change
+    create_table :account_exports, id: :uuid do |t|
+      t.uuid :account_id, null: false
+      t.uuid :user_id, null: false
+      t.string :status, default: "pending", null: false
+      t.datetime :completed_at
+      t.timestamps
+
+      t.index :account_id
+      t.index :user_id
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.2].define(version: 2025_11_29_175717) do
+ActiveRecord::Schema[8.2].define(version: 2025_12_01_100607) do
   create_table "accesses", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
     t.datetime "accessed_at"
     t.uuid "account_id", null: false
@@ -25,6 +25,17 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_29_175717) do
     t.index ["user_id"], name: "index_accesses_on_user_id"
   end
 
+  create_table "account_exports", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
+    t.uuid "account_id", null: false
+    t.datetime "completed_at"
+    t.datetime "created_at", null: false
+    t.string "status", default: "pending", null: false
+    t.datetime "updated_at", null: false
+    t.uuid "user_id", null: false
+    t.index ["account_id"], name: "index_account_exports_on_account_id"
+    t.index ["user_id"], name: "index_account_exports_on_user_id"
+  end
+
   create_table "account_external_id_sequences", id: :uuid, charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
     t.bigint "value", default: 0, null: false
     t.index ["value"], name: "index_account_external_id_sequences_on_value", unique: true

db/schema_sqlite.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.2].define(version: 2025_11_29_175717) do
+ActiveRecord::Schema[8.2].define(version: 2025_12_01_100607) do
   create_table "accesses", id: :uuid, force: :cascade do |t|
     t.datetime "accessed_at"
     t.uuid "account_id", null: false
@@ -25,6 +25,17 @@ ActiveRecord::Schema[8.2].define(version: 2025_11_29_175717) do
     t.index ["user_id"], name: "index_accesses_on_user_id"
   end
 
+  create_table "account_exports", id: :uuid, force: :cascade do |t|
+    t.uuid "account_id", null: false
+    t.datetime "completed_at"
+    t.datetime "created_at", null: false
+    t.string "status", limit: 255, default: "pending", null: false
+    t.datetime "updated_at", null: false
+    t.uuid "user_id", null: false
+    t.index ["account_id"], name: "index_account_exports_on_account_id"
+    t.index ["user_id"], name: "index_account_exports_on_user_id"
+  end
+
   create_table "account_external_id_sequences", id: :uuid, force: :cascade do |t|
     t.bigint "value", default: 0, null: false
     t.index ["value"], name: "index_account_external_id_sequences_on_value", unique: true

test/controllers/accounts/exports_controller_test.rb

@@ -0,0 +1,78 @@
+require "test_helper"
+
+class Account::ExportsControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    sign_in_as :david
+  end
+
+  test "create creates an export record and enqueues job" do
+    assert_difference -> { Account::Export.count }, 1 do
+      assert_enqueued_with(job: ExportAccountDataJob) do
+        post account_exports_path
+      end
+    end
+
+    assert_redirected_to account_settings_path
+    assert_equal "Export started. You'll receive an email when it's ready.", flash[:notice]
+  end
+
+  test "create associates export with current user" do
+    post account_exports_path
+
+    export = Account::Export.last
+    assert_equal users(:david), export.user
+    assert_equal Current.account, export.account
+    assert export.pending?
+  end
+
+  test "create rejects request when current export limit is reached" do
+    Account::ExportsController::CURRENT_EXPORT_LIMIT.times do
+      Account::Export.create!(account: Current.account, user: users(:david))
+    end
+
+    assert_no_difference -> { Account::Export.count } do
+      post account_exports_path
+    end
+
+    assert_response :too_many_requests
+  end
+
+  test "create allows request when exports are older than one day" do
+    Account::ExportsController::CURRENT_EXPORT_LIMIT.times do
+      Account::Export.create!(account: Current.account, user: users(:david), created_at: 2.days.ago)
+    end
+
+    assert_difference -> { Account::Export.count }, 1 do
+      post account_exports_path
+    end
+
+    assert_redirected_to account_settings_path
+  end
+
+  test "show displays completed export with download link" do
+    export = Account::Export.create!(account: Current.account, user: users(:david))
+    export.build
+
+    get account_export_path(export)
+
+    assert_response :success
+    assert_select "a#download-link"
+  end
+
+  test "show displays a warning if the export is missing" do
+    get account_export_path("not-really-an-export")
+
+    assert_response :success
+    assert_select "h2", "Download Expired"
+  end
+
+  test "show does not allow access to another user's export" do
+    export = Account::Export.create!(account: Current.account, user: users(:kevin))
+    export.build
+
+    get account_export_path(export)
+
+    assert_response :success
+    assert_select "h2", "Download Expired"
+  end
+end

test/fixtures/account/exports.yml

@@ -0,0 +1,12 @@
+pending_export:
+  id: <%= ActiveRecord::FixtureSet.identify("pending_export", :uuid) %>
+  account: 37s_uuid
+  user: david_uuid
+  status: pending
+
+completed_export:
+  id: <%= ActiveRecord::FixtureSet.identify("completed_export", :uuid) %>
+  account: 37s_uuid
+  user: david_uuid
+  status: completed
+  completed_at: <%= 1.hour.ago.to_fs(:db) %>

test/mailers/export_mailer_test.rb

@@ -0,0 +1,16 @@
+require "test_helper"
+
+class ExportMailerTest < ActionMailer::TestCase
+  test "completed" do
+    export = Account::Export.create!(account: Current.account, user: users(:david))
+    email = ExportMailer.completed(export)
+
+    assert_emails 1 do
+      email.deliver_now
+    end
+
+    assert_equal [ "david@37signals.com" ], email.to
+    assert_equal "Your Fizzy data export is ready for download", email.subject
+    assert_match %r{/exports/#{export.id}}, email.body.encoded
+  end
+end

test/mailers/previews/export_mailer_preview.rb

@@ -0,0 +1,11 @@
+class ExportMailerPreview < ActionMailer::Preview
+  def completed
+    export = Account::Export.new(
+      id: "preview-export-id",
+      account: Account.first,
+      user: User.first
+    )
+
+    ExportMailer.completed(export)
+  end
+end

test/models/account/export_test.rb

@@ -0,0 +1,95 @@
+require "test_helper"
+
+class Account::ExportTest < ActiveSupport::TestCase
+  test "build_later enqueues ExportAccountDataJob" do
+    export = Account::Export.create!(account: Current.account, user: users(:david))
+
+    assert_enqueued_with(job: ExportAccountDataJob, args: [ export ]) do
+      export.build_later
+    end
+  end
+
+  test "build generates zip with card JSON files" do
+    export = Account::Export.create!(account: Current.account, user: users(:david))
+
+    export.build
+
+    assert export.completed?
+    assert export.file.attached?
+    assert_equal "application/zip", export.file.content_type
+  end
+
+  test "build sets status to processing then completed" do
+    export = Account::Export.create!(account: Current.account, user: users(:david))
+
+    export.build
+
+    assert export.completed?
+    assert_not_nil export.completed_at
+  end
+
+  test "build sends email when completed" do
+    export = Account::Export.create!(account: Current.account, user: users(:david))
+
+    assert_enqueued_jobs 1, only: ActionMailer::MailDeliveryJob do
+      export.build
+    end
+  end
+
+  test "build sets status to failed on error" do
+    export = Account::Export.create!(account: Current.account, user: users(:david))
+    export.stubs(:generate_zip).raises(StandardError.new("Test error"))
+
+    assert_raises(StandardError) do
+      export.build
+    end
+
+    assert export.failed?
+  end
+
+  test "cleanup deletes exports completed more than 24 hours ago" do
+    old_export = Account::Export.create!(account: Current.account, user: users(:david), status: :completed, completed_at: 25.hours.ago)
+    recent_export = Account::Export.create!(account: Current.account, user: users(:david), status: :completed, completed_at: 23.hours.ago)
+    pending_export = Account::Export.create!(account: Current.account, user: users(:david), status: :pending)
+
+    Account::Export.cleanup
+
+    assert_not Account::Export.exists?(old_export.id)
+    assert Account::Export.exists?(recent_export.id)
+    assert Account::Export.exists?(pending_export.id)
+  end
+
+  test "build includes only accessible cards for user" do
+    user = users(:david)
+    export = Account::Export.create!(account: Current.account, user: user)
+
+    export.build
+
+    assert export.completed?
+    assert export.file.attached?
+
+    # Verify zip contents
+    Tempfile.create([ "test", ".zip" ]) do |temp|
+      temp.binmode
+      export.file.download { |chunk| temp.write(chunk) }
+      temp.rewind
+
+      Zip::File.open(temp.path) do |zip|
+        json_files = zip.glob("*.json")
+        assert json_files.any?, "Zip should contain at least one JSON file"
+
+        # Verify structure of a JSON file
+        json_content = JSON.parse(zip.read(json_files.first.name))
+        assert json_content.key?("number")
+        assert json_content.key?("title")
+        assert json_content.key?("board")
+        assert json_content.key?("creator")
+        assert json_content["creator"].key?("id")
+        assert json_content["creator"].key?("name")
+        assert json_content["creator"].key?("email")
+        assert json_content.key?("description")
+        assert json_content.key?("comments")
+      end
+    end
+  end
+end

test/models/card/exportable_test.rb

@@ -0,0 +1,39 @@
+require "test_helper"
+
+class Card::ExportableTest < ActiveSupport::TestCase
+  test "export_json returns card data as JSON" do
+    card = cards(:logo)
+
+    json = JSON.parse(card.export_json)
+
+    assert_equal 1, json["number"]
+    assert_equal "The logo isn't big enough", json["title"]
+    assert_equal "Writebook", json["board"]
+    assert_equal "Triage", json["status"]
+    assert_equal users(:david).id, json["creator"]["id"]
+    assert_equal "David", json["creator"]["name"]
+    assert_equal "david@37signals.com", json["creator"]["email"]
+    assert_equal "", json["description"]
+    assert_equal 5, json["comments"].count
+    assert_equal card.created_at.iso8601, json["created_at"]
+    assert_equal card.updated_at.iso8601, json["updated_at"]
+  end
+
+  test "export_attachments returns attachment paths and blobs" do
+    card = cards(:logo)
+
+    blob = ActiveStorage::Blob.create_and_upload!(
+      io: file_fixture("moon.jpg").open,
+      filename: "moon.jpg",
+      content_type: "image/jpeg"
+    )
+    attachment_html = ActionText::Attachment.from_attachable(blob).to_html
+    card.update!(description: "<p>Here is an image:</p>#{attachment_html}")
+
+    attachments = card.export_attachments
+
+    assert_equal 1, attachments.count
+    assert_equal file_fixture("moon.jpg").binread, attachments.first[:blob].download
+    assert attachments.first[:path].start_with?("#{card.number}/")
+  end
+end