dunlop-core/app/models/dunlop/source_file_model.rb

module Dunlop::SourceFileModel
  extend ActiveSupport::Concern

  include Dunlop::SourceFileModel::ConversionHelpers

  included do
    include Dunlop::Loggable
    has_many :downloads, class_name: 'Dunlop::FileDownload', as: :file
    attr_accessor :working_file
    belongs_to :creator, polymorphic: true, optional: true

    mount_uploader :original_file, Dunlop::SourceFileUploader
    # used by dunlop-file_transfer
    def default_file
      original_file
    end

    state_machine initial: :scheduled do
      event(:scheduled) { transition [:inactive,:active] => :scheduled }
      event(:loading)   { transition  scheduled: :loading   }
      event(:active)    { transition  loading: :active    }
      event(:inactive)  { transition  active: :inactive  }
      event(:failed)    { transition  any                 => :failed    }
    end

  end

  def deactivate_loaded_source_file
    self.class.find_each do |source_file|
      source_file.inactive! if source_file.active?
    end
  end

  def load_source_file
    self.working_file = new_working_file
    unzip_working_file
    pre_process_working_file
    load_source_records
  ensure
    working_file.try(:unlink)
  end

  def new_working_file
    local_working_file = Tempfile.new("source_file", Rails.application.config.working_path)
    FileUtils.chmod(0644, local_working_file.path)
    local_working_file
  end

  def unzip_working_file
    source_path = if original_file.try(:options).try(:[], :storage) == :s3
      temp = Tempfile.new 'temp' # keep reference for tempfile till unzip completed
      temp.binmode
      open( original_file.expiring_url ) { |data| temp.write data.read }
      temp.close
      temp.path
    else
      original_file.path
    end
    Dunlop::FileZip.unzip(source_path, working_file.path)
  end

  def pre_process_working_file(target_path = working_file.path)
    Dunlop::FileSed.fix_line_endings(target_path)
    Dunlop::FileSed.in_place(target_path, %Q{s/"//g})
  end

  def load_source_records
    raise 'override in subclass'
  end

  def autoload!
    #optionally override in subclass
  end

  def assert_first_line(expected, target_path = working_file.path)
    raise "Header of #{self.class.model_name.human} is not as expected:\n\n #{expected}\n\nActual value was:\n#{first_line}" unless first_line == expected
  end

  def assert_first_line_starts_with(expected, target_path = working_file.path)
    first_line_start = first_line[0...expected.length]
    raise "First header part of file #{self.class.model_name.human} is not as expected:\n\n #{expected}\n\nActual value was:\n#{first_line_start}" unless first_line_start == expected
  end

  def first_line(target_path = working_file.path)
    #@first_line ||= File.open(target_path).each_line{|l| break l unless l.starts_with? '#'}.to_s.strip # skip comments
    @first_line ||= File.open(target_path, &:readline).to_s.strip
  end

  def sql_template_path
    Rails.root.join('app/models/source_file/sql_templates')
  end

  def execute_loading_process
    with_nested_logger_and_catch_failed(nil, benchmark: true) do
      logger.info I18n.t('dunlop.source_file.start_loading')
      loading!
      deactivate_loaded_source_file
      load_source_file
      active!
      after_activate_hook
      logger.info I18n.t('dunlop.source_file.finished_loading')
      self
    end
  end

  # Empty hook placeholder to be overwritten by subclass
  def after_activate_hook
  end

  def execute_sql_erb_script(name, template_binding=binding)
    sql_erb_filename = File.join(sql_template_path, "#{name}.sql.erb")
    sql_erb_script = File.read(sql_erb_filename)
    template = ERB.new(sql_erb_script)
    sql_script = template.result(template_binding)
    sql_script.split(";\n").each do |sql_statement|
      ActiveRecord::Base.connection.execute(sql_statement) unless sql_statement.strip.empty?
    end
  end

  module ClassMethods

    def factory(attrs)
      attrs ||= {}
      klass = classes.find { |c| c.to_s == attrs[:sti_type].to_s } || self
      klass.new(attrs)
    end

    def classes
      class_names.map(&:constantize)
    end

    def class_names
      source_file_names.map{|sfn| sfn.is_a?(Class) ? sfn.name : "#{self.name}::#{sfn.to_s.camelize}"}
    end

    def setup_source_files(source_file_names)
      @source_file_names = source_file_names
    end

    def source_file_names
      @source_file_names
    end

    def for_select
      classes.map{|cls| [cls.model_name.human, cls.name]}
    end

    def latest
      active.last
    end

    def last_current_day
      where.not(current_at_day: nil).order(current_at_day: :desc).where.not(state: 'failed').first.try(:current_at_day)
    end

    def active
      where(state: 'active')
    end

    def marker_updated_at(source_file_type)
      SourceFile.
        where(sti_type: source_file_type).
        where(state: 'active').
        order('updated_at DESC').
        pluck(:updated_at).
        first
    end

    def recent_ids
      group(:sti_type).map do |combination|
        sti_type = combination.sti_type

        if marker = marker_updated_at(sti_type)
          SourceFile.
            where(sti_type: sti_type).
            where.not(state: 'inactive').
            #where{ updated_at >= marker }. # squeel
            where("updated_at >= ?", marker).
            pluck(:id)
        else
          SourceFile.
            where(sti_type: sti_type).
            pluck(:id)
        end
      end.flatten.compact
    end

    def recent
      where(id: recent_ids)
    end

    def archive
      #where('id not in (?)',recent_ids)
      where.not(id: recent_ids)
    end

    def cleanup
      archive.find_each(&:destroy)
    end

    def load_scheduled_source_files
      where(state: 'scheduled').find_each do |scheduled_source_file|
        scheduled_source_file.execute_loading_process
      end
    end
    alias_method :load_scheduled!, :load_scheduled_source_files

    def for_select
      classes.map{|klass| [klass.model_name.human, klass.name]}
    end

    def activate_dunlop!
    end
  end

end