From cf297801fd13073f509050bfc588d8e22f98dd35 Mon Sep 17 00:00:00 2001 From: Francisco Brito Date: Fri, 15 Mar 2024 20:26:51 -0300 Subject: [PATCH] feature/#7907-script-to-change-local-storage-to-aws --- lib/active_storage/migrator.rb | 69 +++++++++++++++-------- lib/tasks/storage_migrations.rake | 8 +-- spec/lib/active_storage/migrator_spec.rb | 27 ++++----- spec/lib/tasks/storage_migrations_spec.rb | 41 ++++++++------ 4 files changed, 86 insertions(+), 59 deletions(-) diff --git a/lib/active_storage/migrator.rb b/lib/active_storage/migrator.rb index 55787b962..e13530b4a 100644 --- a/lib/active_storage/migrator.rb +++ b/lib/active_storage/migrator.rb @@ -1,35 +1,60 @@ require 'yaml' require 'erb' +require 'logger' -module ActiveStorage - class Migrator - def self.migrate(from_service_name, to_service_name) - yaml_with_env = ERB.new(File.read('config/storage.yml')).result - configs = YAML.load(yaml_with_env) +class ActiveStorage::Migrator + Rails.logger = Logger.new($stdout) + Rails.logger.level = Logger::DEBUG - from_service = ActiveStorage::Service.configure(from_service_name, { from_service_name.to_sym => configs[from_service_name.to_s] }) - to_service = ActiveStorage::Service.configure(to_service_name, { to_service_name.to_sym => configs[to_service_name.to_s] }) + def self.migrate(from_service_name, to_service_name) + configs = load_storage_config + # Check if services are configured correctly + if configs[from_service_name.to_s].nil? || configs[to_service_name.to_s].nil? + raise "Error: The services '#{from_service_name}' or '#{to_service_name}' are not configured correctly." + end - # Check if services are configured correctly - if from_service.nil? || to_service.nil? - puts "Error: The services '#{from_service_name}' or '#{to_service_name}' are not configured correctly." - return - end + from_service = ActiveStorage::Service.configure(from_service_name, { from_service_name.to_sym => configs[from_service_name.to_s] }) + to_service = ActiveStorage::Service.configure(to_service_name, { to_service_name.to_sym => configs[to_service_name.to_s] }) - # Configure the blob service for the source service - ActiveStorage::Blob.service = from_service + configure_blob_service(from_service) - puts "#{ActiveStorage::Blob.count} Blobs to migrate from #{from_service_name} to #{to_service_name}" - ActiveStorage::Blob.find_each do |blob| - next unless blob.image? + Rails.logger.debug { "#{ActiveStorage::Blob.count} Blobs to migrate from #{from_service_name} to #{to_service_name}" } - print '.' + migrate_blobs(from_service, to_service) + end - blob.open do |io| - checksum = blob.checksum - to_service.upload(blob.key, io, checksum: checksum) - end + def self.load_storage_config + yaml_with_env = ERB.new(File.read('config/storage.yml')).result + YAML.load(yaml_with_env) + end + + def self.configure_services(from_service_name, to_service_name, configs) + from_service = ActiveStorage::Service.configure(from_service_name, { from_service_name.to_sym => configs[from_service_name.to_s] }) + to_service = ActiveStorage::Service.configure(to_service_name, { to_service_name.to_sym => configs[to_service_name.to_s] }) + [from_service, to_service] + end + + def self.configure_service(service_name, configs) + service_config = configs[service_name.to_s] + ActiveStorage::Service.configure(service_name, { service_name.to_sym => service_config }) + end + + def self.configure_blob_service(service) + ActiveStorage::Blob.service = service + end + + def self.migrate_blobs(_from_service, to_service) + # Configure the blob service for the source service + ActiveStorage::Blob.find_each do |blob| + next unless blob.image? + + Rails.logger.debug { '.' } + + blob.open do |io| + checksum = blob.checksum + to_service.upload(blob.key, io, checksum: checksum) end end + Rails.logger.debug { 'Successful migration' } end end diff --git a/lib/tasks/storage_migrations.rake b/lib/tasks/storage_migrations.rake index 93d085545..816290d2d 100644 --- a/lib/tasks/storage_migrations.rake +++ b/lib/tasks/storage_migrations.rake @@ -1,12 +1,10 @@ namespace :storage do desc 'Migrate blobs from one storage service to another' task migrate: :environment do - from_service = ENV['FROM'] - to_service = ENV['TO'] + from_service = ENV.fetch('FROM', nil) + to_service = ENV.fetch('TO', nil) - if from_service.nil? || to_service.nil? - raise 'Missing FROM or TO argument. Usage: FROM=service_name TO=service_name rake storage:migrate' - end + raise 'Missing FROM or TO argument. Usage: FROM=service_name TO=service_name rake storage:migrate' if from_service.nil? || to_service.nil? ActiveStorage::Migrator.migrate(from_service.to_sym, to_service.to_sym) end diff --git a/spec/lib/active_storage/migrator_spec.rb b/spec/lib/active_storage/migrator_spec.rb index 78cd687be..bdc62d22d 100644 --- a/spec/lib/active_storage/migrator_spec.rb +++ b/spec/lib/active_storage/migrator_spec.rb @@ -2,32 +2,29 @@ require 'rails_helper' RSpec.describe ActiveStorage::Migrator do describe '.migrate' do - let(:from_service_stub) { double('from_service') } - let(:to_service_stub) { double('to_service') } - let(:blob_spy) { spy('ActiveStorage::Blob') } + let(:from_service_stub) { instance_double(ActiveStorage::Service) } + let(:to_service_stub) { instance_double(ActiveStorage::Service) } before do - allow(ActiveStorage::Service).to receive(:configure).with(:from_service, any_args).and_return(from_service_stub) - allow(ActiveStorage::Service).to receive(:configure).with(:to_service, any_args).and_return(to_service_stub) - allow(ActiveStorage::Blob).to receive(:find_each).and_yield(blob_spy) + allow(ActiveStorage::Service).to receive(:configure).with('local', any_args).and_return(from_service_stub) + allow(ActiveStorage::Service).to receive(:configure).with('amazon', any_args).and_return(to_service_stub) end context 'when services are configured correctly' do it 'migrates blobs from one service to another' do - allow(blob_spy).to receive(:image?).and_return(true) - - expect(ActiveStorage::Service).to receive(:configure).with(:from_service, any_args) - expect(ActiveStorage::Service).to receive(:configure).with(:to_service, any_args) - - described_class.migrate(:from_service, :to_service) + expect(ActiveStorage::Service).to receive(:configure).with('local', any_args) + expect(ActiveStorage::Service).to receive(:configure).with('amazon', any_args) + expect(described_class).to receive(:migrate_blobs).with(from_service_stub, to_service_stub) + expect { described_class.migrate('local', 'amazon') }.not_to raise_error end end context 'when services are not configured correctly' do it 'prints an error message' do - allow(from_service_stub).to receive(:nil?).and_return(true) - - expect { described_class.migrate(:from_service, :to_service) }.to output("Error: The services 'from_service' or 'to_service' are not configured correctly.\n").to_stdout + allow(ActiveStorage::Service).to receive(:configure).and_return(nil) + expect do + described_class.migrate('random', 'random') + end.to raise_error(RuntimeError, "Error: The services 'random' or 'random' are not configured correctly.") end end end diff --git a/spec/lib/tasks/storage_migrations_spec.rb b/spec/lib/tasks/storage_migrations_spec.rb index 6d7d517ed..d334e3de0 100644 --- a/spec/lib/tasks/storage_migrations_spec.rb +++ b/spec/lib/tasks/storage_migrations_spec.rb @@ -1,27 +1,34 @@ require 'rails_helper' require 'rake' -RSpec.describe 'storage_migrations' do - describe 'rake task' do +RSpec.describe Rake::Task do + describe 'storage_migrations' do + describe 'rake task' do + context 'when FROM argument is missing' do + before do + ENV['FROM'] = nil + end - context 'when FROM argument is missing' do - before do - ENV['FROM'] = nil + it 'raises an error' do + expect do + described_class['storage:migrate'].invoke + end.to raise_error(RuntimeError, + 'Missing FROM or TO argument. Usage: FROM=service_name TO=service_name rake storage:migrate') + end end - it 'raises an error' do - expect { Rake::Task['storage:migrate'].invoke }.to raise_error(RuntimeError, 'Missing FROM or TO argument. Usage: FROM=service_name TO=service_name rake storage:migrate') - end - end + context 'when TO argument is missing' do + before do + ENV['FROM'] = 'service_name' + ENV['TO'] = nil + end - context 'when TO argument is missing' do - before do - ENV['FROM'] = 'service_name' - ENV['TO'] = nil - end - - it 'raises an error' do - expect { Rake::Task['storage:migrate'].invoke }.to raise_error(RuntimeError, 'Missing FROM or TO argument. Usage: FROM=service_name TO=service_name rake storage:migrate') + it 'raises an error' do + expect do + described_class['storage:migrate'].invoke + end.to raise_error(RuntimeError, + 'Missing FROM or TO argument. Usage: FROM=service_name TO=service_name rake storage:migrate') + end end end end