From 64b0ebd8dc082fad4fe06cb14da9c5dd72f23b28 Mon Sep 17 00:00:00 2001 From: Francisco Brito <90341639+FranciscoJBrito@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:42:49 -0300 Subject: [PATCH] feat: Script to migrate images between providers (#9117) # Pull Request Template ## Description Storage migration functionality, which allows the transfer of images from one on-premises provider to another (e.g. AWS, Google, etc.) using Active Storage. I ran the unit and integration tests and they all passed correctly. **The command to execute the migration is `FROM=from_service TO=to_service rake storage:migrate`** Fixes #7907 ## Type of change Please delete options that are not relevant. - [x] New feature (non-breaking change which adds functionality) ## How Has This Been Tested? I have tested the storage migration functionality by running the provided unit and integration tests. Additionally, I manually tested the migration process in a local development environment by following these steps: Set up two storage services (e.g., AWS S3 and Google Cloud Storage) with valid configurations. Execute the `FROM=local TO=amazon rake storage:migrate` task with appropriate FROM and TO arguments to migrate blobs between the services. Verified that the blobs were successfully transferred to the target storage service. Checked for any error messages or unexpected behavior during the migration process. ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] Any dependent changes have been merged and published in downstream modules --------- Co-authored-by: Sony Mathew Co-authored-by: Sony Mathew <2040199+sony-mathew@users.noreply.github.com> --- lib/active_storage/migrator.rb | 52 ++++++ lib/tasks/storage_migrations.rake | 13 ++ spec/lib/active_storage/migrator_spec.rb | 175 ++++++++++++++++++ .../rake/task_storage_migrations_spec.rb | 64 +++++++ spec/rails_helper.rb | 4 + 5 files changed, 308 insertions(+) create mode 100644 lib/active_storage/migrator.rb create mode 100644 lib/tasks/storage_migrations.rake create mode 100644 spec/lib/active_storage/migrator_spec.rb create mode 100644 spec/lib/tasks/rake/task_storage_migrations_spec.rb diff --git a/lib/active_storage/migrator.rb b/lib/active_storage/migrator.rb new file mode 100644 index 000000000..0fdf1669d --- /dev/null +++ b/lib/active_storage/migrator.rb @@ -0,0 +1,52 @@ +require 'yaml' +require 'erb' + +class ActiveStorage::Migrator + def self.migrate(from_service_name, to_service_name, should_update_service_name: true) + configs = load_storage_config + # Check if services are configured correctly + if configs[from_service_name.to_s].nil? || configs[to_service_name.to_s].nil? + raise "Error: The services '#{from_service_name}' or '#{to_service_name}' are not configured correctly." + end + + from_service = configure_service(from_service_name, configs) + + configure_blob_service(from_service) + + migrate_blobs(from_service_name, to_service_name, should_update_service_name: should_update_service_name) + end + + def self.load_storage_config + yaml_with_env = ERB.new(File.read('config/storage.yml')).result + YAML.load(yaml_with_env) + end + + def self.configure_blob_service(service) + ActiveStorage::Blob.service = service + end + + def self.configure_service(service_name, configs = load_storage_config) + service_config = configs[service_name.to_s] + ActiveStorage::Service.configure(service_name, { service_name.to_sym => service_config }) + end + + def self.migrate_blobs(from_service_name, to_service_name, should_update_service_name: true) + to_service = configure_service(to_service_name) + blobs = ActiveStorage::Blob.where(service_name: from_service_name.to_s) + Rails.logger.debug { "#{blobs.count} Blobs to migrate from #{from_service_name} to #{to_service_name}" } + + blobs.find_each do |blob| + Rails.logger.debug { '.' } + + blob.open do |io| + checksum = blob.checksum + to_service.upload(blob.key, io, checksum: checksum) + end + + blob.update!(service_name: to_service_name.to_s) if should_update_service_name + rescue ActiveStorage::FileNotFoundError => e + Rails.logger.warn { "Skipping missing blob #{blob.id} (#{blob.key}): #{e.message}" } + end + Rails.logger.debug { 'Successful migration' } + end +end diff --git a/lib/tasks/storage_migrations.rake b/lib/tasks/storage_migrations.rake new file mode 100644 index 000000000..9a864db4e --- /dev/null +++ b/lib/tasks/storage_migrations.rake @@ -0,0 +1,13 @@ +namespace :storage do + desc 'Migrate blobs from one storage service to another' + # Example: FROM=local TO=amazon UPDATE_BLOB_SERVICE_NAME=true bundle exec rake storage:migrate + task migrate: :environment do + from_service = ENV.fetch('FROM', nil) + to_service = ENV.fetch('TO', nil) + should_update_service_name = ActiveModel::Type::Boolean.new.cast(ENV.fetch('UPDATE_BLOB_SERVICE_NAME', true)) + + raise 'Missing FROM or TO argument. Usage: FROM=service_name TO=service_name rake storage:migrate' if from_service.nil? || to_service.nil? + + ActiveStorage::Migrator.migrate(from_service.to_sym, to_service.to_sym, should_update_service_name: should_update_service_name) + end +end diff --git a/spec/lib/active_storage/migrator_spec.rb b/spec/lib/active_storage/migrator_spec.rb new file mode 100644 index 000000000..748e18174 --- /dev/null +++ b/spec/lib/active_storage/migrator_spec.rb @@ -0,0 +1,175 @@ +require 'rails_helper' +require 'fileutils' +require 'stringio' +require 'tmpdir' + +RSpec.describe ActiveStorage::Migrator do + around do |example| + original_services = ActiveStorage::Blob.services + original_service = ActiveStorage::Blob.service + + example.run + ensure + ActiveStorage::Blob.services = original_services + ActiveStorage::Blob.service = original_service + end + + describe '.migrate' do + let(:from_service_stub) { instance_double(ActiveStorage::Service, name: 'local') } + let(:to_service_stub) { instance_double(ActiveStorage::Service, name: 'amazon') } + + before do + allow(ActiveStorage::Service).to receive(:configure).with('local', any_args).and_return(from_service_stub) + allow(ActiveStorage::Service).to receive(:configure).with('amazon', any_args).and_return(to_service_stub) + end + + context 'when services are configured correctly' do + it 'migrates blobs from one service to another' do + expect(ActiveStorage::Service).to receive(:configure).with('local', any_args) + expect(described_class).to receive(:migrate_blobs).with('local', 'amazon', should_update_service_name: true) + expect { described_class.migrate('local', 'amazon') }.not_to raise_error + end + + it 'passes the service-name update flag to blob migration' do + expect(described_class).to receive(:migrate_blobs).with('local', 'amazon', should_update_service_name: false) + + described_class.migrate('local', 'amazon', should_update_service_name: false) + end + + it 'does not override the application logger' do + allow(described_class).to receive(:migrate_blobs) + + expect(Rails).not_to receive(:logger=) + + described_class.migrate('local', 'amazon') + end + end + + context 'when services are not configured correctly' do + it 'prints an error message' do + allow(ActiveStorage::Service).to receive(:configure).and_return(nil) + expect do + described_class.migrate('random', 'random') + end.to raise_error(RuntimeError, "Error: The services 'random' or 'random' are not configured correctly.") + end + end + end + + describe '.migrate_blobs' do + let(:from_root) { Dir.mktmpdir('active-storage-local') } + let(:to_root) { Dir.mktmpdir('active-storage-amazon') } + let(:other_root) { Dir.mktmpdir('active-storage-other') } + let(:from_service) { build_disk_service('local', from_root) } + let(:to_service) { build_disk_service('amazon', to_root) } + let(:other_service) { build_disk_service('other', other_root) } + + before do + allow(described_class).to receive(:configure_service).with(:amazon).and_return(to_service) + end + + around do |example| + original_services = ActiveStorage::Blob.services + original_service = ActiveStorage::Blob.service + + ActiveStorage::Blob.services = { + 'local' => from_service, + 'amazon' => to_service, + 'other' => other_service + } + ActiveStorage::Blob.service = from_service + + example.run + ensure + ActiveStorage::Blob.services = original_services + ActiveStorage::Blob.service = original_service + FileUtils.rm_rf([from_root, to_root, other_root]) + end + + it 'uploads non-image blobs and updates their service name' do + blob = create_blob(service_name: 'local', filename: 'invoice.pdf', content_type: 'application/pdf', content: 'pdf-content') + + described_class.migrate_blobs(:local, :amazon, should_update_service_name: true) + + expect(blob).not_to be_image + expect(blob.reload.service_name).to eq('amazon') + expect(to_service.download(blob.key)).to eq('pdf-content') + end + + it 'only migrates blobs from the source service' do + local_blob = create_blob(service_name: 'local', filename: 'audio.mp3', content_type: 'audio/mpeg', content: 'audio-content') + other_blob = create_blob(service_name: 'other', filename: 'other.pdf', content_type: 'application/pdf', content: 'other-content') + + described_class.migrate_blobs(:local, :amazon, should_update_service_name: true) + + expect(local_blob.reload.service_name).to eq('amazon') + expect(to_service.download(local_blob.key)).to eq('audio-content') + expect(other_blob.reload.service_name).to eq('other') + expect(to_service.exist?(other_blob.key)).to be(false) + expect(other_service.download(other_blob.key)).to eq('other-content') + end + + it 'skips missing source files and continues migration' do + missing_blob = create_blob(service_name: 'local', filename: 'missing.pdf', content_type: 'application/pdf', content: 'missing-content') + existing_blob = create_blob(service_name: 'local', filename: 'existing.pdf', content_type: 'application/pdf', content: 'existing-content') + + from_service.delete(missing_blob.key) + + expect do + described_class.migrate_blobs(:local, :amazon, should_update_service_name: true) + end.not_to raise_error + + expect(missing_blob.reload.service_name).to eq('local') + expect(to_service.exist?(missing_blob.key)).to be(false) + expect(existing_blob.reload.service_name).to eq('amazon') + expect(to_service.download(existing_blob.key)).to eq('existing-content') + end + + it 'does not update the blob service when upload fails' do + blob = create_blob(service_name: 'local', filename: 'report.pdf', content_type: 'application/pdf', content: 'report-content') + + allow(to_service).to receive(:upload).and_raise(ActiveStorage::IntegrityError) + + expect do + described_class.migrate_blobs(:local, :amazon, should_update_service_name: true) + end.to raise_error(ActiveStorage::IntegrityError) + + expect(blob.reload.service_name).to eq('local') + expect(to_service.exist?(blob.key)).to be(false) + end + + it 'skips the blob service update when the flag is disabled' do + blob = create_blob(service_name: 'local', filename: 'manual.pdf', content_type: 'application/pdf', content: 'manual-content') + + described_class.migrate_blobs(:local, :amazon, should_update_service_name: false) + + expect(blob.reload.service_name).to eq('local') + expect(to_service.download(blob.key)).to eq('manual-content') + end + + it 'does not rewrite blobs that were already migrated' do + blob = create_blob(service_name: 'local', filename: 'archive.pdf', content_type: 'application/pdf', content: 'archive-content') + + allow(to_service).to receive(:upload).and_call_original + + 2.times { described_class.migrate_blobs(:local, :amazon, should_update_service_name: true) } + + expect(to_service).to have_received(:upload).with(blob.key, an_instance_of(Tempfile), checksum: blob.checksum).once + expect(blob.reload.service_name).to eq('amazon') + expect(ActiveStorage::Blob.where(id: blob.id, service_name: 'local')).to be_empty + end + + def build_disk_service(service_name, root) + ActiveStorage::Service.configure(service_name, { service_name => { service: 'Disk', root: root } }) + end + + def create_blob(service_name:, filename:, content_type:, content:) + ActiveStorage::Blob.create_and_upload!( + io: StringIO.new(content), + filename: filename, + content_type: content_type, + service_name: service_name, + identify: false + ) + end + end +end diff --git a/spec/lib/tasks/rake/task_storage_migrations_spec.rb b/spec/lib/tasks/rake/task_storage_migrations_spec.rb new file mode 100644 index 000000000..552c814b3 --- /dev/null +++ b/spec/lib/tasks/rake/task_storage_migrations_spec.rb @@ -0,0 +1,64 @@ +require 'rake' +require 'rails_helper' + +RSpec.describe Rake::Task do + describe 'storage_migrations' do + describe 'rake task' do + subject(:task) { described_class['storage:migrate'] } + + before do + task.reenable + end + + context 'when FROM argument is missing' do + it 'raises an error' do + with_modified_env FROM: nil, TO: 'amazon' do + expect do + task.invoke + end.to raise_error(RuntimeError, + 'Missing FROM or TO argument. Usage: FROM=service_name TO=service_name rake storage:migrate') + end + end + end + + context 'when TO argument is missing' do + it 'raises an error' do + with_modified_env FROM: 'service_name', TO: nil do + expect do + task.invoke + end.to raise_error(RuntimeError, + 'Missing FROM or TO argument. Usage: FROM=service_name TO=service_name rake storage:migrate') + end + end + end + + context 'when required arguments are present' do + it 'updates blob service names by default' do + expect(ActiveStorage::Migrator).to receive(:migrate).with(:local, :amazon, should_update_service_name: true) + + with_modified_env FROM: 'local', TO: 'amazon', UPDATE_BLOB_SERVICE_NAME: nil do + task.invoke + end + end + + it 'skips updating blob service names when disabled' do + expect(ActiveStorage::Migrator).to receive(:migrate).with(:local, :amazon, should_update_service_name: false) + + with_modified_env FROM: 'local', TO: 'amazon', UPDATE_BLOB_SERVICE_NAME: 'false' do + task.invoke + end + end + + it 'can be invoked again after the task is re-enabled' do + expect(ActiveStorage::Migrator).to receive(:migrate).twice.with(:local, :amazon, should_update_service_name: true) + + with_modified_env FROM: 'local', TO: 'amazon', UPDATE_BLOB_SERVICE_NAME: nil do + task.invoke + task.reenable + task.invoke + end + end + end + end + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index e83db4046..984cd045f 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -7,6 +7,10 @@ require 'rspec/rails' require 'pundit/rspec' require 'sidekiq/testing' +# Load Rake tasks +require 'rake' +Rails.application.load_tasks + # test-prof helpers for tests optimization require 'test_prof/recipes/rspec/before_all' require 'test_prof/recipes/rspec/let_it_be'