Fix attachemnts (#1627)

* Load keys from tenanted dbs

* Implement sgid swapping

* Fix sgid generation

* Flip the logic to do lookups by tenant and id

* Create missing blobs

* Copy over variations

* fix-active-storage-links: fix mentions

* Fix variant generation

* Fix avatars

* Fix avatars

---------

Co-authored-by: Mike Dalessio <mike@37signals.com>

script/fix-active-storage-links.rb

@@ -1,33 +1,278 @@
 #!/usr/bin/env ruby
 
 require_relative "../config/environment"
+require "pathname"
 require "uri"
 require "base64"
 require "json"
 
-ActionText::RichText.all.where("body LIKE '%/rails/active_storage/%'").find_each do |rich_text|
+class FixActiveStorage
-  next unless rich_text.body
+  attr_reader :skipped, :processed, :scope
 
-  blobs = rich_text.embeds.map(&:blob)
+  def initialize(scope = nil)
+    @scope = scope || ActionText::RichText.all.where("body LIKE '%/rails/active_storage/%'")
+    @mapping = {}
 
-  rich_text.body.send(:attachment_nodes).each do |node|
+    @skipped = 0
-    url = node["url"]
+    @processed = 0
-    next unless url
 
-    url_encoded_filename = url.split("/").last
+    @users = {}
-    filename = URI.decode_www_form_component(url_encoded_filename)
+    @memberships = {}
+    @attachments = {}
+    @identities = {}
+  end
 
-    counter += 1
+  def ingest_blob_keys(db_path)
-    blob = blobs.select { |b| b.filename == filename }
+    models = Models.new(db_path)
-    raise "Multiple blobs with filename #{filename}" if blob.size > 1
-    blob = blob.first
 
-    if blob
+    @mapping[models.accounts.sole.external_account_id.to_s] = models.blobs.all.index_by(&:id)
-      node["sgid"] = blob.attachable_sgid
+    @attachments[models.accounts.sole.external_account_id.to_s] = models.attachments.all.index_by(&:id)
+    @users[models.accounts.sole.external_account_id.to_s] = models.users.all.index_by(&:id)
+  end
+
+  def ingest_untenanted(untenanted_db_path)
+    untenanted = Models.new(untenanted_db_path)
+
+    @memberships = untenanted.memberships.all.index_by(&:id)
+    @identities = untenanted.identities.all.index_by(&:id)
+  end
+
+  def perform
+    fix_avatars
+    fix_mentions
+    fix_attachments
+
+    pp [ @processed, @skipped ]
+  end
+
+  private
+    def fix_avatars
+      User.all.active.preload(:identity).find_each do |user|
+        tenant = user.account.external_account_id.to_s
+        email_address = user.identity.email_address
+
+        membership = @memberships.values.find { |m| m.tenant == tenant && @identities[m.identity_id]&.email_address == email_address }
+        old_user = @users[tenant]&.values&.find { |u| u.membership_id == membership&.id }
+
+        next if user.avatar.attached? || old_user.nil?
+
+        old_avatar_attachment = @attachments[tenant]&.values&.find do |attachment|
+          attachment.record_type == "User" && attachment.record_id == old_user.id && attachment.name == "avatar"
+        end
+
+        if old_avatar_attachment.nil?
+          @skipped += 1
+          next
+        end
+
+
+        old_blob = old_avatar_attachment.blob
+
+        if old_blob.nil?
+          @skipped += 1
+          next
+        end
+
+        new_blob = ActiveStorage::Blob.find_by(key: old_blob.key)
+
+        unless new_blob
+          new_blob = ActiveStorage::Blob.create!(
+            account_id: user.account_id,
+            byte_size: old_blob.byte_size,
+            checksum: old_blob.checksum,
+            content_type: old_blob.content_type,
+            created_at: old_blob.created_at,
+            filename: old_blob.filename,
+            key: old_blob.key,
+            metadata: old_blob.metadata,
+            service_name: old_blob.service_name
+          )
+        end
+
+          ActiveStorage::Attachment.find_or_create_by!(
+            account_id: user.account_id,
+            blob_id: new_blob.id,
+            name: "avatar",
+            record: user
+          )
+
+        @processed += 1
+      end
+    end
+
+    def fix_mentions
+      ActionText::RichText.where("body LIKE '%action-text-attachment%'").find_each do |rich_text|
+        rich_text.body.send(:attachment_nodes).each do |node|
+          next unless node["content-type"] == "application/vnd.actiontext.mention"
+
+          sgid = SignedGlobalID.parse(node["sgid"], for: ActionText::Attachable::LOCATOR_NAME)
+
+          user = @users.dig(sgid.params[:tenant], sgid.model_id.to_i)
+          membership = @memberships[user&.membership_id]
+          unless membership
+            @skipped += 1
+            next
+          end
+          identity = @identities[membership&.identity_id]
+          unless identity
+            @skipped += 1
+            next
+          end
+
+          new_identity = Identity.find_by(email_address: identity.email_address)
+          new_account = Account.find_by(external_account_id: sgid.params[:tenant])
+          new_user = User.find_by(identity: new_identity, account: new_account)
+          new_sgid = new_user.attachable_sgid
+
+          node["sgid"] = new_sgid.to_s
+        end
+        rich_text.save!
+      end
+    end
+
+    def fix_attachments
+      scope.find_each do |rich_text|
+        next unless rich_text.body
+
+        rich_text.body.send(:attachment_nodes).each do |node|
+          sgid = node["sgid"]
+          url = node["url"]
+          next if url.blank? || sgid.blank?
+
+          sgid = SignedGlobalID.parse(node["sgid"], for: ActionText::Attachable::LOCATOR_NAME)
+          old_blob = @mapping.dig(sgid.params[:tenant], sgid.model_id.to_i)
+
+          # There are some old files that got lost in a previous migration
+          unless old_blob
+            @skipped += 1
+            next
+          end
+
+          new_blob = ActiveStorage::Blob.find_by(key: old_blob.key)
+
+          unless new_blob
+            new_blob = ActiveStorage::Blob.create!(
+              account_id: rich_text.account_id,
+              byte_size: old_blob.byte_size,
+              checksum: old_blob.checksum,
+              content_type: old_blob.content_type,
+              created_at: old_blob.created_at,
+              filename: old_blob.filename,
+              key: old_blob.key,
+              metadata: old_blob.metadata,
+              service_name: old_blob.service_name
+            )
+
+            ActiveStorage::Attachment.create!(
+              account_id: rich_text.account_id,
+              blob_id: new_blob.id,
+              created_at: old_blob.created_at,
+              name: "embeds",
+              record: rich_text
+            )
+          end
+
+          node["sgid"] = new_blob.attachable_sgid
+
+          @processed += 1
+        end
+
+        rich_text.save!
+      rescue ActiveStorage::FileNotFoundError
+        @skipped += 1
+        next
+      end
+    end
+end
+
+class Models
+  attr_reader :application_record
+
+  def initialize(db_path)
+    const_name = "ImportBase#{db_path.hash.abs}"
+
+    if self.class.const_defined?(const_name)
+      @application_record = self.class.const_get(const_name)
     else
-      skipped += 1
+      @application_record = Class.new(ActiveRecord::Base) do
+        self.abstract_class = true
+
+        def self.models
+          const_get("MODELS")
+        end
+
+        delegate :models, to: :class
+      end
+      self.class.const_set(const_name, @application_record)
+    end
+
+    @application_record.establish_connection adapter: "sqlite3", database: db_path
+    @application_record.const_set("MODELS", self)
+  end
+
+  def accounts
+    @accounts ||= Class.new(application_record) do
+      self.table_name = "accounts"
     end
   end
 
-  rich_text.save!
+  def blobs
+    models = self
+    @blobs ||= Class.new(application_record) do
+      self.table_name = "active_storage_blobs"
+
+      def attachments
+        models.attachments.where(blob_id: id)
+      end
+    end
+  end
+
+  def attachments
+    models = self
+    @attachments ||= Class.new(application_record) do
+      self.table_name = "active_storage_attachments"
+
+      def blob
+        models.blobs.find_by(id: blob_id)
+      end
+    end
+  end
+
+  def users
+    @users ||= Class.new(application_record) do
+      self.table_name = "users"
+    end
+  end
+
+  def identities
+    @identities ||= Class.new(application_record) do
+      self.table_name = "identities"
+    end
+  end
+
+  def memberships
+    @memberships ||= Class.new(application_record) do
+      self.table_name = "memberships"
+    end
+  end
 end
+
+# tenanted_db_paths = ARGV
+tenanted_db_paths = Dir[Rails.root.join("storage/tenants/production/*/db/main.sqlite3")]
+untenanted_db_path = Rails.root.join("storage/untenanted/production.sqlite3")
+
+if tenanted_db_paths.empty?
+  $stderr.puts "Error: at least one tenanted database path is required"
+  $stderr.puts
+  exit 1
+end
+
+fix = FixActiveStorage.new
+
+fix.ingest_untenanted(untenanted_db_path)
+
+tenanted_db_paths.each_with_index do |db_path, _index|
+  fix.ingest_blob_keys(db_path)
+end
+
+fix.perform