Backup the tenant databases nightly.

This is probably not a permanent solution, but should provide some
disaster recovery capability while we're building up Fizzy operationally.

app/jobs/sqlite_backups_job.rb

@@ -0,0 +1,101 @@
+#
+#  This job backs up all the tenant databases using the SQLite Backup API, which should allow the
+#  application to continue running against the database while it is backed up.
+#
+#  ref: https://www.sqlite.org/c3ref/backup_finish.html
+#
+#  It will keep N files around, like this:
+#
+#    storage/tenants/development/honcho/db/
+#    ├─ main.sqlite3
+#    ├─ main.sqlite3.1
+#    ├─ main.sqlite3.2
+#    ├─ main.sqlite3.3
+#    ├─ main.sqlite3.4
+#    └─ main.sqlite3.5
+#
+class SQLiteBackupsJob < ApplicationJob
+  DEFAULT_NUMBER_OF_BACKUPS = 5
+  DEFAULT_STEP_PAGES = 1024
+
+  def perform(keep: DEFAULT_NUMBER_OF_BACKUPS, step: DEFAULT_STEP_PAGES)
+    @failures = []
+
+    ApplicationRecord.with_each_tenant do |tenant|
+      perform_file_rollover tenant, keep: keep
+      perform_backup tenant, step: step
+    end
+
+    if @failures.present?
+      raise "SQLiteBackupsJob: failed to backup tenants: #{@failures.join(", ")}"
+    end
+  end
+
+  private
+    def perform_file_rollover(tenant, keep:)
+      keep.downto(2) do |j|
+        fresher = backup_path(tenant, j - 1)
+        staler = backup_path(tenant, j)
+
+        if j == keep && File.exist?(staler)
+          FileUtils.rm(staler)
+        end
+
+        if File.exist?(fresher)
+          # TODO: It may be worth benchmarking whether backing up into the previous backup is faster
+          # than backing up into an empty file.
+          FileUtils.mv(fresher, staler)
+        end
+      end
+    end
+
+    def perform_backup(tenant, step:)
+      ApplicationRecord.with_connection do |conn|
+        current_adapter = conn.raw_connection
+        backup_db = backup_path(tenant, 1)
+        backup_adapter = SQLite3::Database.new(backup_db)
+        backup = SQLite3::Backup.new(backup_adapter, "main", current_adapter, "main")
+
+        pages = 0
+        elapsed = ActiveSupport::Benchmark.realtime(:float_millisecond) do
+          loop do
+            status = backup.step(step)
+            case status
+            when SQLite3::Constants::ErrorCode::DONE
+              break
+            when SQLite3::Constants::ErrorCode::OK
+              total = backup.pagecount
+              progress = total - backup.remaining
+              Rails.logger.debug { "SQLiteBackupsJob: #{tenant.inspect}: Wrote #{progress} of #{total} pages." }
+            when SQLite3::Constants::ErrorCode::BUSY, SQLite3::Constants::ErrorCode::LOCKED
+              Rails.logger.debug { "SQLiteBackupsJob: #{tenant.inspect}: Busy, retrying." }
+            else
+              Rails.logger.error "SQLiteBackupsJob: #{tenant.inspect}: Failed with status #{status}."
+              @failures << tenant
+            end
+          end
+
+          pages = backup.pagecount
+          backup.finish
+        end
+
+        message = sprintf(
+          "SQLiteBackupsJob: %{tenant}: Backup complete in %<elapsed>.1f ms. Wrote %{pages} pages to %{path}",
+          tenant: tenant.inspect, path: backup_db.inspect, pages: pages, elapsed: elapsed
+        )
+        Rails.logger.info message
+      end
+    end
+
+    def backup_path(tenant, index)
+      db_path(tenant) + ".#{index}"
+    end
+
+    def db_path(tenant)
+      db_config.database
+    end
+
+    def db_config
+      ApplicationRecord.connection_pool.db_config
+    end
+end

config/environments/development.rb

@@ -74,4 +74,7 @@ Rails.application.configure do
 
   # Allow all hosts in development
   config.hosts = nil
+
+  config.active_job.queue_adapter = :solid_queue
+  config.solid_queue.connects_to = { database: { writing: :queue } }
 end

config/initializers/inflections.rb

@@ -11,6 +11,6 @@
 # end
 
 # These inflection rules are supported but not enabled by default:
-# ActiveSupport::Inflector.inflections(:en) do |inflect|
-#   inflect.acronym "RESTful"
-# end
+ActiveSupport::Inflector.inflections(:en) do |inflect|
+  inflect.acronym "SQLite"
+end

config/recurring.yml

@@ -8,3 +8,6 @@ production:
   remove_abandoned_creations:
     class: RemoveAbandonedCreationsJob
     schedule: every hour
+  sqlite_backups:
+    class: SQLiteBackupsJob
+    schedule: every day at 05:00