module Dunlop::TargetFileModel extend ActiveSupport::Concern extend Memoist # Looking for the ::Job implementation? This is implemented in the to_prepare engine statement # lib/dunlop/engine.rb PAPERCLIP_S3_HEADERS = Proc.new do |attachment| if attachment.persisted? {} else {'Content-Disposition' => %|attachment; filename="#{File.basename(attachment.local_target_filename)}"|} # can be resource based and only available on initialization. Therefore persisted? end end included do include Dunlop::Loggable has_many :downloads, class_name: 'Dunlop::FileDownload', as: :file attr_accessor :working_file, :original_file attr_accessor :resource Dunlop.target_file_models << self mount_uploader :original_file, Dunlop::TargetFileUploader unless defined?(Paperclip) # CarrierWave still default. # used by dunlop-file_transfer def default_file original_file end state_machine initial: :new do before_transition on: :generating, do: :set_start_time before_transition on: [:completed,:failed], do: :set_end_time event(:scheduled) { transition new: :scheduled } event(:generating) { transition scheduled: :generating } event(:completed) { transition generating: :completed } event(:failed) { transition any => :failed } end end def set_start_time self.start_time = Time.zone.now end def set_end_time self.end_time = Time.zone.now end def duration if start_time and end_time end_time - start_time elsif start_time Time.zone.now - start_time else 0 end end def generate_and_post_process_file self.working_file = new_working_file generate_file post_process_file attach_target_file post_generate_hook ensure working_file.close! rescue SystemCallError # container permission issue end def local_target_filename @local_target_filename ||= begin result = File.join(Rails.application.config.working_path, target_basename) gzip_file? ? result + ".gz" : result end end # Target file basename used on creation to deliver the desired file_name. # On actul persisted records this method cannot be used for naming. Then # @record.original_file&.original_filename or @record.original_file_file_name # should be used def target_basename @target_basename ||= generate_file_name end # This method should contain the dynamically generated filename, could include precise timestamp, that differs per inquiry. Should be cached on instance creation def generate_file_name "#{self.class.name.demodulize.underscore}-#{date_number}.csv" end def post_generate_hook # hook to implement post generation actions like sending by email and/or to other systems by means of web call and/or to create a transfer instance end # Implement #data or this method in Subclass. Example: # CsvBuilder::BladiBla.new(resource).data def generate_file working_file.puts builder.data end def builder @builder ||= self.class.builder_class.new(resource, builder_options) end # can be implemented by specific builder classes, for customized builder behaviour def builder_options {} end def header_line_count builder.headers.present? ? 1 : 0 end def post_process_file self.working_file.close self.number_of_records = line_count self.number_of_records -= header_line_count gzip_file if gzip_file? self.size = working_file_size self.save! end def attach_target_file %x[cp #{Shellwords.escape(working_file.path)} #{Shellwords.escape(local_target_filename)}] self.original_file = File.open(local_target_filename, 'r') self.save! ensure FileUtils.rm_f(local_target_filename) end def new_working_file #local_working_file = Tempfile.new("target_file", Rails.application.config.working_path) local_working_file = Tempfile.new([target_basename, File.extname(target_basename)], Rails.application.config.working_path) # keep reference for tempfile till unzip completed and keep extension preserved FileUtils.chmod(0666, local_working_file.path) local_working_file end def gzip_file? false #override in subclass end def gzip_file return if Dunlop::FileZip.mime_type(working_file.path) == "application/x-gzip" # no double action and working file replacement local_working_file = new_working_file Dunlop::FileZip.gzip(working_file.path,local_working_file.path) working_file.unlink rescue SystemCallError # container permission issue self.working_file = local_working_file end def line_count %x[wc -l #{Shellwords.escape(working_file.path)}].to_i end def sql_template_path Rails.root.join("app/models/target_file/sql_templates") end def prepare_working_file_for_sql_generation working_file.close FileUtils.rm_f(working_file.path) end def execute_generation_process with_nested_logger_and_catch_failed do scheduled! unless scheduled? generating! generate_and_post_process_file completed! self # return generated file instance end end def date_number Date.current.to_s(:number) end module ClassMethods def factory(attrs) attrs ||= {} klass = classes.find { |c| c.to_s == attrs[:sti_type].to_s } || self klass.new(attrs) end def classes @classes ||= class_names.map(&:constantize) end def class_names target_file_names.map{|tfn| tfn.is_a?(Class) ? tfn.name : "#{self.name}::#{tfn.to_s.camelize}"} end def setup_target_files(target_file_names) @target_file_names = target_file_names end def target_file_names @target_file_names end def latest completed.last end def completed where(state: 'completed') end def activate_dunlop! end # Allow resource as argument. Changed from resource to rsrc to be sure to avoid method naming collision confusion def generate!(rsrc = nil) new_record = create(resource: rsrc) #new_record.resource = rsrc if rsrc.present? new_record.execute_generation_process new_record end def cleanup #noop, can't make assumptions here end def create_from_source_file!(source_file) target_file = self.new( working_file: File.open(source_file.original_file.path, 'r'), state: 'generating', start_time: Time.now, number_of_records: source_file.number_of_records, ) target_file.size = File.size(target_file.working_file.path) target_file.attach_target_file target_file.completed! target_file end def builder_class "#{name}CsvBuilder".safe_constantize || ::CsvBuilder end def paperclip_s3_headers PAPERCLIP_S3_HEADERS end def has_panda_original_file(options = {}) environments = Array.wrap(options.delete(:environments)) || %w[production staging] if environments.include?(Rails.env) has_attached_file :original_file, #url: '/system/organization/:id/logos/:style.:extension', storage: :s3, path: ':rails_env/:class/:id/:filename', hash_secret: '8da20e64945d2164c31357c6052be2a5', s3_credentials: { bucket: ENV['S3_FILES_BUCKET'], access_key_id: ENV['S3_ACCESS_KEY_ID'], secret_access_key: ENV['S3_SECRET_ACCESS_KEY'], s3_region: ENV['AWS_REGION'], }, s3_headers: paperclip_s3_headers, s3_permissions: :private, s3_protocol: :https else has_attached_file :original_file, url: '/system/:class/:attachment/:id/:filename', hash_secret: 'dunlop-target-files-for-panda' end do_not_validate_attachment_file_type :original_file end end private def working_file_size File.size(working_file.path) end end