backups: sweep them into "/backups" if it exists

and delete any backups older than our data retention policy

ref: https://3.basecamp.com/2914079/buckets/37331921/todos/9175385847

app/jobs/sqlite_backups_job.rb

@@ -1,3 +1,5 @@
+require "find"
+
 #
 #  This job backs up all the tenant databases using the SQLite Backup API, which should allow the
 #  application to continue running against the database while it is backed up.
@@ -6,7 +8,7 @@
 #
 #  It will keep N files around, like this:
 #
-#    storage/tenants/development/honcho/db/
+#    storage/tenants/development/12345678/db/
 #    ├─ main.sqlite3
 #    ├─ main.sqlite3.1
 #    ├─ main.sqlite3.2
@@ -14,16 +16,29 @@
 #    ├─ main.sqlite3.4
 #    └─ main.sqlite3.5
 #
+#  On some systems, notably in production, we have an NFS-mounted filesystem into which the
+#  application copies the backup files for disaster recovery. We copy into an environment- and
+#  tenant-specific directory. The file, when copied, will be renamed with the timestamp of the file
+#  creation time. For example:
+#
+#    /backups/production/12345678/main.sqlite3.20251014194804
+#
+#  It will also clean up old backups in the NFS-mounted filesystem following our data retention policy.
+#
 class SQLiteBackupsJob < ApplicationJob
   DEFAULT_NUMBER_OF_BACKUPS = 5
   DEFAULT_STEP_PAGES = 1024
+  DEFAULT_SWEEP_DIR = "/backups"
+  DEFAULT_SWEEP_RETENTION = 30.days # https://37signals.com/policies/privacy
 
-  def perform(keep: DEFAULT_NUMBER_OF_BACKUPS, step: DEFAULT_STEP_PAGES)
+  def perform(keep: DEFAULT_NUMBER_OF_BACKUPS, step: DEFAULT_STEP_PAGES, sweep_dir: DEFAULT_SWEEP_DIR, sweep_retention: DEFAULT_SWEEP_RETENTION)
     @failures = []
 
     ApplicationRecord.with_each_tenant do |tenant|
-      perform_file_rollover tenant, keep: keep
-      perform_backup tenant, step: step
+      perform_file_rollover(tenant, keep:)
+      perform_backup(tenant, step:)
+      perform_sweep(tenant, sweep_dir:)
+      enforce_retention(tenant, sweep_dir:, sweep_retention:)
     end
 
     if @failures.present?
@@ -66,11 +81,11 @@ class SQLiteBackupsJob < ApplicationJob
             when SQLite3::Constants::ErrorCode::OK
               total = backup.pagecount
               progress = total - backup.remaining
-              Rails.logger.debug { "SQLiteBackupsJob: #{tenant.inspect}: Wrote #{progress} of #{total} pages." }
+              log(tenant, :debug) { "Wrote #{progress} of #{total} pages." }
             when SQLite3::Constants::ErrorCode::BUSY, SQLite3::Constants::ErrorCode::LOCKED
-              Rails.logger.debug { "SQLiteBackupsJob: #{tenant.inspect}: Busy, retrying." }
+              log(tenant, :debug) { "Busy, retrying." }
             else
-              Rails.logger.error "SQLiteBackupsJob: #{tenant.inspect}: Failed with status #{status}."
+              log(tenant, :error) { "Failed with status #{status}." }
               @failures << tenant
             end
           end
@@ -79,11 +94,45 @@ class SQLiteBackupsJob < ApplicationJob
           backup.finish
         end
 
-        message = sprintf(
-          "SQLiteBackupsJob: %{tenant}: Backup complete in %<elapsed>.1f ms. Wrote %{pages} pages to %{path}",
-          tenant: tenant.inspect, path: backup_db.inspect, pages: pages, elapsed: elapsed
-        )
-        Rails.logger.info message
+        log(tenant) { sprintf("Backup complete in %<elapsed>.1f ms. Wrote %{pages} pages to %{path}", path: backup_db.inspect, pages: pages, elapsed: elapsed) }
+      end
+    end
+
+    def perform_sweep(tenant, sweep_dir:)
+      unless File.directory?(sweep_dir) && File.writable?(sweep_dir)
+        log(tenant, :warn) { "Skipping sweep, #{sweep_dir.inspect} does not exist or is not writable." }
+        return
+      end
+
+      backup_file = backup_path(tenant, 1)
+      sweep_path = File.join(sweep_dir, Rails.env, tenant)
+      FileUtils.mkdir_p(sweep_path)
+
+      if File.exist?(backup_file)
+        timestamp = File.ctime(backup_file).utc.strftime("%Y%m%d%H%M%S")
+        swept_file = File.join(sweep_path, File.basename(db_path(tenant)) + ".#{timestamp}")
+        FileUtils.cp(backup_file, swept_file)
+        log(tenant) { "Swept backup to #{swept_file.inspect}." }
+      else
+        log(tenant, :warn) { "No backup file found at #{backup_file.inspect} to sweep." }
+      end
+    end
+
+    def enforce_retention(tenant, sweep_dir:, sweep_retention:)
+      unless File.directory?(sweep_dir) && File.writable?(sweep_dir)
+        log(tenant, :warn) { "Skipping retention enforcement, #{sweep_dir.inspect} does not exist or is not writable." }
+        return
+      end
+
+      sweep_path = File.join(sweep_dir, Rails.env, tenant)
+      if File.directory?(sweep_path)
+        cutoff_time = Time.now - sweep_retention
+        Find.find(sweep_path) do |path|
+          if File.file?(path) && File.ctime(path) < cutoff_time
+            FileUtils.rm(path)
+            log(tenant) { "Removed old swept backup #{path.inspect}." }
+          end
+        end
       end
     end
 
@@ -98,4 +147,9 @@ class SQLiteBackupsJob < ApplicationJob
     def db_config
       ApplicationRecord.connection_pool.db_config
     end
+
+    def log(tenant, level = :info, &block)
+      message = block.call
+      Rails.logger.send(level, "[tenant=#{tenant}] SQLiteBackupsJob: #{message}")
+    end
 end