From 9a89e1f5227bd9cce9dfbcb69b6a60321c79bb03 Mon Sep 17 00:00:00 2001 From: Kenta Ishizaki <153918146+55728@users.noreply.github.com> Date: Fri, 24 Apr 2026 20:33:03 +0900 Subject: [PATCH] fix: Strip UTF-8 BOM in DataImportJob#csv_reader before parsing CSV (#14126) ## Description `DataImportJob#csv_reader` reads CSV data with `force_encoding('UTF-8')` but does not strip the UTF-8 Byte Order Mark (`EF BB BF`). If a CSV file containing a BOM is imported, the first header key is prefixed with `\uFEFF`, which causes key mismatches in `DataImport::ContactManager` when the first column is one of the recognized keys (`:email`, `:identifier`, `:phone_number`, `:name`). This was identified during review of #14123 (see #14124 for the tracking issue). Fixes #14124 ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? - Added a new fixture (`spec/fixtures/data_import/with_bom.csv`) containing a UTF-8 BOM followed by valid contact data. - Added a new spec (`will strip UTF-8 BOM and import contacts correctly`) that imports the BOM fixture and verifies that `name`, `email`, and `phone_number` are all correctly parsed. - All existing examples in `spec/jobs/data_import_job_spec.rb` continue to pass. ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] Any dependent changes have been merged and published in downstream modules --- app/jobs/data_import_job.rb | 1 + spec/fixtures/data_import/with_bom.csv | 2 ++ spec/jobs/data_import_job_spec.rb | 14 ++++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 spec/fixtures/data_import/with_bom.csv diff --git a/app/jobs/data_import_job.rb b/app/jobs/data_import_job.rb index d8bbeb992..4149ee16f 100644 --- a/app/jobs/data_import_job.rb +++ b/app/jobs/data_import_job.rb @@ -106,6 +106,7 @@ class DataImportJob < ApplicationJob raw_data = file.read utf8_data = raw_data.force_encoding('UTF-8') clean_data = utf8_data.valid_encoding? ? utf8_data : utf8_data.encode('UTF-16le', invalid: :replace, replace: '').encode('UTF-8') + clean_data = clean_data.delete_prefix("\xEF\xBB\xBF") CSV.new(StringIO.new(clean_data), headers: true) end diff --git a/spec/fixtures/data_import/with_bom.csv b/spec/fixtures/data_import/with_bom.csv new file mode 100644 index 000000000..8b1850620 --- /dev/null +++ b/spec/fixtures/data_import/with_bom.csv @@ -0,0 +1,2 @@ +name,email,phone_number +Ahmed,ahmed@example.com,+971501234567 diff --git a/spec/jobs/data_import_job_spec.rb b/spec/jobs/data_import_job_spec.rb index 19178b177..88b268ef6 100644 --- a/spec/jobs/data_import_job_spec.rb +++ b/spec/jobs/data_import_job_spec.rb @@ -91,6 +91,20 @@ RSpec.describe DataImportJob do expect(invalid_data_import.account.contacts.first.name).to eq(csv_data[0]['name'].encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')) end + + it 'will strip UTF-8 BOM and import contacts correctly' do + bom_data_import = create(:data_import, + import_file: Rack::Test::UploadedFile.new(Rails.root.join('spec/fixtures/data_import/with_bom.csv'), + 'text/csv')) + + described_class.perform_now(bom_data_import) + expect(bom_data_import.account.contacts.count).to eq(1) + + contact = bom_data_import.account.contacts.first + expect(contact.name).to eq('Ahmed') + expect(contact.email).to eq('ahmed@example.com') + expect(contact.phone_number).to eq('+971501234567') + end end context 'when the data contains existing records' do