Green enougn for dependency and test upgrades

Gemfile

@@ -6,9 +6,10 @@ source 'https://rubygems.org'
 #
 gemspec
 
-gem "rails", "~> 5.1.3"
+gem "rails", "~> 5.1"
 
 group :development, :test do
+  gem 'bootsnap', require: false
   gem 'rspec-rails'
   gem 'sqlite3'
   gem 'pry-rails'
@@ -20,6 +21,7 @@ group :development, :test do
   gem 'foundation-rails', '~> 5.4'
   gem "select2-rails"
   gem 'puma'
+  gem 'paperclip', '~> 5.3'
   #gem 'carrierwave'
   #gem 'fog-aws'
   gem 'uuidtools'

app/models/dunlop/source_file_model.rb

@@ -263,19 +263,19 @@ module Dunlop::SourceFileModel
           s3_headers: paperclip_s3_headers,
           s3_permissions: :private,
           s3_protocol: :https
+
+        attr_accessor :pre_set_original_file
+        # overload original_file assignment since on attachment initialization the file is not yet 'known' for the s3 headers
+        define_method :original_file= do |file|
+          self.pre_set_original_file = file
+          original_file.assign(file)
+        end
       else
         has_attached_file :original_file,
           url: '/system/:class/:attachment/:id/:filename',
           hash_secret: 'dunlop-source-files-for-panda'
       end
       do_not_validate_attachment_file_type :original_file
-
-      attr_accessor :pre_set_original_file
-      # overload original_file assignment since on attachment initialization the file is not yet 'known' for the s3 headers
-      define_method :original_file= do |file|
-        self.pre_set_original_file = file
-        original_file.assign(file)
-      end
     end
 
     def paperclip_s3_headers

spec/dummy/app/models/source_file.rb

@@ -1,6 +1,8 @@
 class SourceFile < ApplicationRecord
   include Dunlop::SourceFileModel
 
+  has_panda_original_file environments: %w[production staging]
+
   setup_source_files %i[
     my_source
   ]

spec/dummy/app/models/target_file.rb

@@ -1,6 +1,8 @@
 class TargetFile < ApplicationRecord
   include Dunlop::TargetFileModel
 
+  has_panda_original_file environments: %w[production staging]
+
   setup_target_files %i[
     my_target
     file_with_builder

spec/dummy/bin/bundle

@@ -1,3 +1,3 @@
 #!/usr/bin/env ruby
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
 load Gem.bin_path('bundler', 'bundle')

spec/dummy/bin/rails

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
-APP_PATH = File.expand_path('../../config/application', __FILE__)
+APP_PATH = File.expand_path('../config/application', __dir__)
 require_relative '../config/boot'
 require 'rails/commands'

spec/dummy/bin/setup

@@ -1,29 +1,36 @@
 #!/usr/bin/env ruby
-require 'pathname'
+require 'fileutils'
+include FileUtils
 
 # path to your application root.
-APP_ROOT = Pathname.new File.expand_path('../../',  __FILE__)
+APP_ROOT = File.expand_path('..', __dir__)
 
-Dir.chdir APP_ROOT do
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+chdir APP_ROOT do
   # This script is a starting point to setup your application.
-  # Add necessary setup steps to this file:
+  # Add necessary setup steps to this file.
 
-  puts "== Installing dependencies =="
-  system "gem install bundler --conservative"
-  system "bundle check || bundle install"
+  puts '== Installing dependencies =='
+  system! 'gem install bundler --conservative'
+  system('bundle check') || system!('bundle install')
+
+  # Install JavaScript dependencies if using Yarn
+  # system('bin/yarn')
 
   # puts "\n== Copying sample files =="
-  # unless File.exist?("config/database.yml")
-  #   system "cp config/database.yml.sample config/database.yml"
+  # unless File.exist?('config/database.yml')
+  #   cp 'config/database.yml.sample', 'config/database.yml'
   # end
 
   puts "\n== Preparing database =="
-  system "bin/rake db:setup"
+  system! 'bin/rails db:setup'
 
   puts "\n== Removing old logs and tempfiles =="
-  system "rm -f log/*"
-  system "rm -rf tmp/cache"
+  system! 'bin/rails log:clear tmp:clear'
 
   puts "\n== Restarting application server =="
-  system "touch tmp/restart.txt"
+  system! 'bin/rails restart'
 end

spec/dummy/bin/update

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
+
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+chdir APP_ROOT do
+  # This script is a way to update your development environment automatically.
+  # Add necessary update steps to this file.
+
+  puts '== Installing dependencies =='
+  system! 'gem install bundler --conservative'
+  system('bundle check') || system!('bundle install')
+
+  # Install JavaScript dependencies if using Yarn
+  # system('bin/yarn')
+
+  puts "\n== Updating database =="
+  system! 'bin/rails db:migrate'
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! 'bin/rails log:clear tmp:clear'
+
+  puts "\n== Restarting application server =="
+  system! 'bin/rails restart'
+end

spec/dummy/bin/yarn

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path('..', __dir__)
+Dir.chdir(APP_ROOT) do
+  begin
+    exec "yarnpkg", *ARGV
+  rescue Errno::ENOENT
+    $stderr.puts "Yarn executable was not detected in the system."
+    $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install"
+    exit 1
+  end
+end

spec/dummy/config/application.rb

@@ -2,11 +2,14 @@ require File.expand_path('../boot', __FILE__)
 
 # Pick the frameworks you want:
 require "active_record/railtie"
+require "active_storage/engine"
 require "action_controller/railtie"
-require "action_mailer/railtie"
 require "action_view/railtie"
+require "action_mailer/railtie"
+require "active_job/railtie"
+require "action_cable/engine"
+#require "rails/test_unit/railtie"
 require "sprockets/railtie"
-# require "rails/test_unit/railtie"
 
 Bundler.require(*Rails.groups)
 require "dunlop"

spec/dummy/config/boot.rb

@@ -1,5 +1,4 @@
-# Set up gems listed in the Gemfile.
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__)
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
 
-require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
-$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)
+require 'bundler/setup' # Set up gems listed in the Gemfile.
+require 'bootsnap/setup' # Speed up boot time by caching expensive operations.

spec/dummy/config/environment.rb

@@ -1,7 +1,5 @@
 # Load the Rails application.
-require File.expand_path('../application', __FILE__)
-
-#require 'carrierwave/storage/fog'
+require_relative 'application'
 
 # Initialize the Rails application.
 Rails.application.initialize!

spec/dummy/config/environments/production.rb

@@ -14,15 +14,13 @@ Rails.application.configure do
   config.consider_all_requests_local       = false
   config.action_controller.perform_caching = true
 
-  # Enable Rack::Cache to put a simple HTTP cache in front of your application
-  # Add `rack-cache` to your Gemfile before enabling this.
-  # For large-scale production use, consider using a caching reverse proxy like
-  # NGINX, varnish or squid.
-  # config.action_dispatch.rack_cache = true
+  # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"]
+  # or in config/master.key. This key is used to decrypt credentials (and other encrypted files).
+  # config.require_master_key = true
 
   # Disable serving static files from the `/public` folder by default since
   # Apache or NGINX already handles this.
-  config.serve_static_files = ENV['RAILS_SERVE_STATIC_FILES'].present?
+  config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present?
 
   # Compress JavaScripts and CSS.
   config.assets.js_compressor = :uglifier
@@ -31,16 +29,18 @@ Rails.application.configure do
   # Do not fallback to assets pipeline if a precompiled asset is missed.
   config.assets.compile = false
 
-  # Asset digests allow you to set far-future HTTP expiration dates on all assets,
-  # yet still be able to expire them through the digest params.
-  config.assets.digest = true
-
   # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb
 
+  # Enable serving of images, stylesheets, and JavaScripts from an asset server.
+  # config.action_controller.asset_host = 'http://assets.example.com'
+
   # Specifies the header that your server uses for sending files.
   # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache
   # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX
 
+  # Store uploaded files on the local file system (see config/storage.yml for options)
+  config.active_storage.service = :local
+
   # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
   # config.force_ssl = true
 
@@ -49,16 +49,16 @@ Rails.application.configure do
   config.log_level = :debug
 
   # Prepend all log lines with the following tags.
-  # config.log_tags = [ :subdomain, :uuid ]
-
-  # Use a different logger for distributed setups.
-  # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new)
+  config.log_tags = [ :request_id ]
 
   # Use a different cache store in production.
   # config.cache_store = :mem_cache_store
 
-  # Enable serving of images, stylesheets, and JavaScripts from an asset server.
-  # config.action_controller.asset_host = 'http://assets.example.com'
+  # Use a real queuing backend for Active Job (and separate queues per environment)
+  # config.active_job.queue_adapter     = :resque
+  # config.active_job.queue_name_prefix = "dummy_#{Rails.env}"
+
+  config.action_mailer.perform_caching = false
 
   # Ignore bad email addresses and do not raise email delivery errors.
   # Set this to true and configure the email server for immediate delivery to raise delivery errors.
@@ -74,6 +74,16 @@ Rails.application.configure do
   # Use default logging formatter so that PID and timestamp are not suppressed.
   config.log_formatter = ::Logger::Formatter.new
 
+  # Use a different logger for distributed setups.
+  # require 'syslog/logger'
+  # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
+
+  if ENV["RAILS_LOG_TO_STDOUT"].present?
+    logger           = ActiveSupport::Logger.new(STDOUT)
+    logger.formatter = config.log_formatter
+    config.logger    = ActiveSupport::TaggedLogging.new(logger)
+  end
+
   # Do not dump schema after migrations.
   config.active_record.dump_schema_after_migration = false
 end

spec/dummy/config/environments/test.rb

@@ -6,22 +6,17 @@ Rails.application.configure do
   # your test database is "scratch space" for the test suite and is wiped
   # and recreated between test runs. Don't rely on the data there!
   config.cache_classes = true
-  config.time_zone = 'Moscow'
-  config.git_version = 'test'
 
   # Do not eager load code on boot. This avoids loading your whole application
   # just for the purpose of running a single test. If you are using a tool that
   # preloads Rails for running tests, you may have to set it to true.
   config.eager_load = false
 
-  # Configure static file server for tests with Cache-Control for performance.
-  if Rails.version.starts_with?('4')
-    config.serve_static_files   = true # rails < 5
-    config.static_cache_control = 'public, max-age=3600' # rails < 5
-  else
-    config.public_file_server.enabled = true
-    config.public_file_server.headers = { 'Cache-Control' => 'public, max-age=3600' }
-  end
+  # Configure public file server for tests with Cache-Control for performance.
+  config.public_file_server.enabled = true
+  config.public_file_server.headers = {
+    'Cache-Control' => "public, max-age=#{1.hour.to_i}"
+  }
 
   # Show full error reports and disable caching.
   config.consider_all_requests_local       = true
@@ -33,22 +28,19 @@ Rails.application.configure do
   # Disable request forgery protection in test environment.
   config.action_controller.allow_forgery_protection = false
 
-  config.to_prepare do
-    require Rails.root.join("config/initializers/dunlop_workflow_steps_in_batch_finished_config")
-    require Rails.root.join("config/initializers/dunlop_overdue_config")
-  end
+  # Store uploaded files on the local file system in a temporary directory
+  config.active_storage.service = :test
+
+  config.action_mailer.perform_caching = false
 
   # Tell Action Mailer not to deliver emails to the real world.
   # The :test delivery method accumulates sent emails in the
   # ActionMailer::Base.deliveries array.
   config.action_mailer.delivery_method = :test
 
-  # Randomize the order test cases are executed.
-  config.active_support.test_order = :random
-
   # Print deprecation notices to the stderr.
   config.active_support.deprecation = :stderr
 
   # Raises error for missing translations
-  config.action_view.raise_on_missing_translations = true
+  # config.action_view.raise_on_missing_translations = true
 end

spec/dummy/config/initializers/application_controller_renderer.rb

@@ -0,0 +1,8 @@
+# Be sure to restart your server when you modify this file.
+
+# ActiveSupport::Reloader.to_prepare do
+#   ApplicationController.renderer.defaults.merge!(
+#     http_host: 'example.org',
+#     https: false
+#   )
+# end

spec/dummy/config/initializers/assets.rb

@@ -3,9 +3,12 @@
 # Version of your assets, change this if you want to expire all your assets.
 Rails.application.config.assets.version = '1.0'
 
-# Add additional assets to the asset load path
+# Add additional assets to the asset load path.
 # Rails.application.config.assets.paths << Emoji.images_path
+# Add Yarn node_modules folder to the asset load path.
+Rails.application.config.assets.paths << Rails.root.join('node_modules')
 
 # Precompile additional assets.
-# application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
-# Rails.application.config.assets.precompile += %w( search.js )
+# application.js, application.css, and all non-JS/CSS in the app/assets
+# folder are already added.
+# Rails.application.config.assets.precompile += %w( admin.js admin.css )

spec/dummy/config/initializers/content_security_policy.rb

@@ -0,0 +1,25 @@
+# Be sure to restart your server when you modify this file.
+
+# Define an application-wide content security policy
+# For further information see the following documentation
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
+
+# Rails.application.config.content_security_policy do |policy|
+#   policy.default_src :self, :https
+#   policy.font_src    :self, :https, :data
+#   policy.img_src     :self, :https, :data
+#   policy.object_src  :none
+#   policy.script_src  :self, :https
+#   policy.style_src   :self, :https
+
+#   # Specify URI for violation reports
+#   # policy.report_uri "/csp-violation-report-endpoint"
+# end
+
+# If you are using UJS then enable automatic nonce generation
+# Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) }
+
+# Report CSP violations to a specified URI
+# For further information see the following documentation:
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only
+# Rails.application.config.content_security_policy_report_only = true

spec/dummy/config/initializers/cookies_serializer.rb

@@ -1,3 +1,5 @@
 # Be sure to restart your server when you modify this file.
 
+# Specify a serializer for the signed and encrypted cookie jars.
+# Valid options are :json, :marshal, and :hybrid.
 Rails.application.config.action_dispatch.cookies_serializer = :json

spec/dummy/config/initializers/wrap_parameters.rb

@@ -5,10 +5,10 @@
 
 # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
 ActiveSupport.on_load(:action_controller) do
-  wrap_parameters format: [:json] if respond_to?(:wrap_parameters)
+  wrap_parameters format: [:json]
 end
 
 # To enable root element in JSON for ActiveRecord objects.
 # ActiveSupport.on_load(:active_record) do
-#  self.include_root_in_json = true
+#   self.include_root_in_json = true
 # end

spec/dummy/config/locales/en.yml

@@ -16,6 +16,16 @@
 #
 # This would use the information in config/locales/es.yml.
 #
+# The following keys must be escaped otherwise they will not be retrieved by
+# the default I18n backend:
+#
+# true, false, on, off, yes, no
+#
+# Instead, surround them with single quotes.
+#
+# en:
+#   'true': 'foo'
+#
 # To learn more, please read the Rails Internationalization guide
 # available at http://guides.rubyonrails.org/i18n.html.
 

spec/dummy/config/puma.rb

@@ -0,0 +1,34 @@
+# Puma can serve each request in a thread from an internal thread pool.
+# The `threads` method setting takes two numbers: a minimum and maximum.
+# Any libraries that use thread pools should be configured to match
+# the maximum value specified for Puma. Default is set to 5 threads for minimum
+# and maximum; this matches the default thread size of Active Record.
+#
+threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 }
+threads threads_count, threads_count
+
+# Specifies the `port` that Puma will listen on to receive requests; default is 3000.
+#
+port        ENV.fetch("PORT") { 3000 }
+
+# Specifies the `environment` that Puma will run in.
+#
+environment ENV.fetch("RAILS_ENV") { "development" }
+
+# Specifies the number of `workers` to boot in clustered mode.
+# Workers are forked webserver processes. If using threads and workers together
+# the concurrency of the application would be max `threads` * `workers`.
+# Workers do not work on JRuby or Windows (both of which do not support
+# processes).
+#
+# workers ENV.fetch("WEB_CONCURRENCY") { 2 }
+
+# Use the `preload_app!` method when specifying a `workers` number.
+# This directive tells Puma to first boot the application and load code
+# before forking the application. This takes advantage of Copy On Write
+# process behavior so workers use less memory.
+#
+# preload_app!
+
+# Allow puma to be restarted by `rails restart` command.
+plugin :tmp_restart

spec/dummy/config/spring.rb

@@ -0,0 +1,6 @@
+%w[
+  .ruby-version
+  .rbenv-vars
+  tmp/restart.txt
+  tmp/caching-dev.txt
+].each { |path| Spring.watch(path) }

spec/dummy/config/storage.yml

@@ -0,0 +1,34 @@
+test:
+  service: Disk
+  root: <%= Rails.root.join("tmp/storage") %>
+
+local:
+  service: Disk
+  root: <%= Rails.root.join("storage") %>
+
+# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
+# amazon:
+#   service: S3
+#   access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %>
+#   secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %>
+#   region: us-east-1
+#   bucket: your_own_bucket
+
+# Remember not to checkin your GCS keyfile to a repository
+# google:
+#   service: GCS
+#   project: your_project
+#   credentials: <%= Rails.root.join("path/to/gcs.keyfile") %>
+#   bucket: your_own_bucket
+
+# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key)
+# microsoft:
+#   service: AzureStorage
+#   storage_account_name: your_account_name
+#   storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %>
+#   container: your_container_name
+
+# mirror:
+#   service: Mirror
+#   primary: local
+#   mirrors: [ amazon, google, microsoft ]

spec/dummy/db/migrate/20180607211734_add_paperclip_fields_to_source_files.rb

@@ -0,0 +1,8 @@
+class AddPaperclipFieldsToSourceFiles < ActiveRecord::Migration[5.2]
+  def change
+    add_column :source_files, :original_file_file_name, :string
+    add_column :source_files, :original_file_content_type, :string
+    add_column :source_files, :original_file_file_size, :integer
+    add_column :source_files, :original_file_updated_at, :datetime
+  end
+end

spec/dummy/db/migrate/20180607212010_add_paperclip_fields_to_target_files.rb

@@ -0,0 +1,8 @@
+class AddPaperclipFieldsToTargetFiles < ActiveRecord::Migration[5.2]
+  def change
+    add_column :target_files, :original_file_file_name, :string
+    add_column :target_files, :original_file_content_type, :string
+    add_column :target_files, :original_file_file_size, :integer
+    add_column :target_files, :original_file_updated_at, :datetime
+  end
+end

spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171128140114) do
+ActiveRecord::Schema.define(version: 2018_06_07_212010) do
 
   create_table "contractor_initializations", force: :cascade do |t|
     t.integer "workflow_instance_id"
@@ -101,6 +101,10 @@ ActiveRecord::Schema.define(version: 20171128140114) do
     t.date "current_at_day"
     t.integer "creator_id"
     t.string "creator_type"
+    t.string "original_file_file_name"
+    t.string "original_file_content_type"
+    t.integer "original_file_file_size"
+    t.datetime "original_file_updated_at"
   end
 
   create_table "source_record_my_sources", force: :cascade do |t|
@@ -123,6 +127,10 @@ ActiveRecord::Schema.define(version: 20171128140114) do
     t.string "original_file"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.string "original_file_file_name"
+    t.string "original_file_content_type"
+    t.integer "original_file_file_size"
+    t.datetime "original_file_updated_at"
   end
 
   create_table "users", force: :cascade do |t|

spec/dummy/public/system/source_file/my_sources/original_files/1/my_source_1.csv

@@ -0,0 +1,3 @@
+my_date,my_var,My count,my_num,my_truth
+2015-05-09,abc D,19,4.8,true
+2015-11-09,def G,19,7.8,false

spec/dummy/public/system/source_file/my_sources/original_files/2/my_source_1.csv

@@ -0,0 +1,3 @@
+my_date,my_var,My count,my_num,my_truth
+2015-05-09,abc D,19,4.8,true
+2015-11-09,def G,19,7.8,false

spec/dummy/public/system/source_file/my_sources/original_files/3/my_source_1.csv

@@ -0,0 +1,3 @@
+my_date,my_var,My count,my_num,my_truth
+2015-05-09,abc D,19,4.8,true
+2015-11-09,def G,19,7.8,false