dunlop-core/app/models/dunlop/source_file_model.rb

module Dunlop::SourceFileModel
  extend ActiveSupport::Concern
  PAPERCLIP_S3_HEADERS = Proc.new do |attachment|
    if attachment.persisted?
      {}
    else
      # This strategy fails when the original_file of a model is changed. But is for here no usecase
      {'Content-Disposition' => %|attachment; filename="#{attachment.pre_set_original_file&.original_filename.presence || attachment.class.name + Time.now.getutc.iso8601}"|}
    end
  end

  include ::Dunlop::SourceFileModel::ConversionHelpers

  included do
    include Dunlop::Loggable
    has_many :downloads, class_name: 'Dunlop::FileDownload', as: :file
    attr_accessor :working_file
    belongs_to :creator, polymorphic: true, optional: true
    Dunlop.source_file_models << self
    scope :processed, -> { where state: ['active', 'inactive'] }

    mount_uploader :original_file, ::Dunlop::SourceFileUploader unless defined?(Paperclip) # CarrierWave still default.

    # used by dunlop-file_transfer
    def default_file
      original_file
    end

    state_machine initial: :scheduled do
      event(:scheduled) { transition [:inactive,:active] => :scheduled }
      event(:loading)   { transition  scheduled: :loading   }
      event(:active)    { transition  loading: :active    }
      event(:inactive)  { transition  active: :inactive  }
      event(:failed)    { transition  any                 => :failed    }
    end

    after_commit(on: :create) { autoload! } # always autoload!, is empty method, can be implemented

    # Auto generate the SourceFile::Job that allows processing by worker. For example
    #   def autoload
    #     SourceFile::Job.perform_later(id)
    #   end
    self::Job = Class.new "#{name.deconstantize}::ApplicationJob".constantize do
      def perform(source_file_id)
        return unless source_file = self.class.name.sub(/::Job$/, '').constantize.find(source_file_id)
        return unless source_file.scheduled?
        source_file.execute_loading_process
      end
    end
  end

  # NOTE:/TODO: this strategy is incompatible with handling multiple files of the same type at the same time. Should be completely substituted by
  # a sequential (elastic beanstalk is not sequential when worker has more than 1 thread) queue based worder implementation
  def deactivate_loaded_source_file
    self.class.active.update_all state: 'inactive'
  end

  def load_source_file
    self.working_file = new_working_file
    unzip_working_file
    pre_process_working_file
    load_source_records
  ensure
    working_file.try(:unlink)
  end

  def new_working_file
    filename = original_file.original_filename.to_s
    local_working_file = Tempfile.new([filename, File.extname(filename)], Rails.application.config.working_path)
    FileUtils.chmod(0644, local_working_file.path)
    local_working_file
  end

  def unzip_working_file
    source_path = if original_file.try(:options).try(:[], :storage) == :s3
      filename = original_file.original_filename.to_s
      temp = Tempfile.new([filename, File.extname(filename)], Rails.application.config.working_path) # keep reference for tempfile till unzip completed and keep extension preserved
      temp.binmode
      open( original_file.expiring_url ) { |data| temp.write data.read }
      temp.close
      temp.path
    else
      original_file.path
    end
    ::Dunlop::FileZip.unzip(source_path, working_file.path)
  end

  def pre_process_working_file(target_path = working_file.path)
    ::Dunlop::FileSed.fix_line_endings(target_path)
  end

  def load_source_records
    raise 'override in subclass'
  end

  def autoload!
    #optionally override in subclass
  end

  def assert_first_line(expected, target_path = working_file.path)
    raise "Header of #{self.class.model_name.human} is not as expected:\n\n #{expected}\n\nActual value was:\n#{first_line}" unless first_line == expected
  end

  def assert_first_line_starts_with(expected, target_path = working_file.path)
    first_line_start = first_line[0...expected.length]
    raise "First header part of file #{self.class.model_name.human} is not as expected:\n\n #{expected}\n\nActual value was:\n#{first_line_start}" unless first_line_start == expected
  end

  def first_line(target_path = working_file.path)
    #@first_line ||= File.open(target_path).each_line{|l| break l unless l.starts_with? '#'}.to_s.strip # skip comments
    @first_line ||= File.open(target_path, &:readline).to_s.strip
  end

  def sql_template_path
    Rails.root.join('app/models/source_file/sql_templates')
  end

  def execute_loading_process
    with_nested_logger_and_catch_failed(nil, benchmark: true) do
      logger.info I18n.t('dunlop.source_file.start_loading')
      loading!
      deactivate_loaded_source_file
      load_source_file
      active!
      after_activate_hook
      logger.info I18n.t('dunlop.source_file.finished_loading')
      self
    end
  end

  # Empty hook placeholder to be overwritten by subclass
  def after_activate_hook
    #optionally implement in subclass
  end

  def execute_sql_erb_script(name, template_binding=binding)
    sql_erb_filename = File.join(sql_template_path, "#{name}.sql.erb")
    sql_erb_script = File.read(sql_erb_filename)
    template = ERB.new(sql_erb_script)
    sql_script = template.result(template_binding)
    sql_script.split(";\n").each do |sql_statement|
      ActiveRecord::Base.connection.execute(sql_statement) unless sql_statement.strip.empty?
    end
  end

  module ClassMethods

    def factory(attrs)
      attrs ||= {}
      klass = classes.find { |c| c.to_s == attrs[:sti_type].to_s } || self
      klass.new(attrs)
    end

    def classes
      class_names.map(&:constantize)
    end

    def class_names
      source_file_names.map{|sfn| sfn.is_a?(Class) ? sfn.name : "#{self.name}::#{sfn.to_s.camelize}"}
    end

    def setup_source_files(source_file_names)
      @source_file_names = source_file_names
    end

    def source_file_names
      @source_file_names
    end

    def for_select
      classes.map{|cls| [cls.model_name.human, cls.name]}
    end

    def latest
      active.last
    end

    def last_current_day
      where.not(current_at_day: nil).order(current_at_day: :desc).where.not(state: 'failed').first.try(:current_at_day)
    end

    def active
      where(state: 'active')
    end

    # braindump towards state => processed support for source-files
    def latest_processed
      if base_class == self # return a scope having only the latest processed source-files
        join_select = processed.select('MAX(id) AS max_id').group(inheritance_column)
        #join_select = processed.selecting{ id.maximum.as('max_id')}.group(:sti_type)
        joins("INNER JOIN (#{join_select.to_sql}) tmp ON id = tmp.max_id")
      else # return the latest process of the specific source-file class
        # 2 query based solution, probably faster than the one (sub) query solution
        find_by(id: processed.maximum(:id))

        ## ONE QUERY SOLUTIONS
        #   find_by(id: processed.select('MAX(id)'))
        # OR using baby squeel
        #   find_by(id: processed.selecting{ id.maximum })
      end
    end

    def ransackable_scopes(auth_object = nil)
      [:latest_processed]
    end

    def marker_updated_at(source_file_type)
      SourceFile.
        where(sti_type: source_file_type).
        where(state: 'active').
        order('updated_at DESC').
        pluck(:updated_at).
        first
    end

    #TODO: this is not compatible with multiple source_file impolementations in one application
    def recent_ids
      group(:sti_type).map do |combination|
        sti_type = combination.sti_type

        if marker = marker_updated_at(sti_type)
          SourceFile.
            where(sti_type: sti_type).
            where.not(state: 'inactive').
            #where{ updated_at >= marker }. # squeel
            where("updated_at >= ?", marker).
            pluck(:id)
        else
          SourceFile.
            where(sti_type: sti_type).
            pluck(:id)
        end
      end.flatten.compact
    end

    def recent
      where(id: recent_ids)
    end

    def archive
      #where('id not in (?)',recent_ids)
      where.not(id: recent_ids)
    end

    def cleanup
      archive.find_each(&:destroy)
    end

    def load_scheduled_source_files
      where(state: 'scheduled').find_each do |scheduled_source_file|
        scheduled_source_file.execute_loading_process
      end
    end
    alias_method :load_scheduled!, :load_scheduled_source_files

    def has_panda_original_file(options = {})
      environments = Array.wrap(options.delete(:environments)) || %w[production staging]
      if environments.include?(Rails.env)
        has_attached_file :original_file,
          #url: '/system/organization/:id/logos/:style.:extension',
          storage: :s3,
          path: ':rails_env/:class/:id/:filename',
          hash_secret: '8da20e64945d2164c31357c6052be2a5',
          s3_credentials: {
            bucket:            ENV['S3_FILES_BUCKET'],
            access_key_id:     ENV['S3_ACCESS_KEY_ID'],
            secret_access_key: ENV['S3_SECRET_ACCESS_KEY'],
            s3_region:         ENV['AWS_REGION'],
          },
          s3_headers: paperclip_s3_headers,
          s3_permissions: :private,
          s3_protocol: :https

        attr_accessor :pre_set_original_file
        # overload original_file assignment since on attachment initialization the file is not yet 'known' for the s3 headers
        define_method :original_file= do |file|
          self.pre_set_original_file = file
          original_file.assign(file)
        end
      else
        has_attached_file :original_file,
          url: '/system/:class/:attachment/:id/:filename',
          hash_secret: 'dunlop-source-files-for-panda'
      end
      do_not_validate_attachment_file_type :original_file
    end

    def paperclip_s3_headers
      PAPERCLIP_S3_HEADERS
    end

    def activate_dunlop!
    end
  end

end