Fetch IMAP message content using `BODY.PEEK[]` instead of `RFC822` to avoid provider-specific parser failures while preserving unread state. This also applies the existing SMTP timeout configuration to custom SMTP email-channel replies, so provider SMTP responses have enough time to complete. Fixes: https://github.com/chatwoot/chatwoot/issues/12762 ## Why Some IMAP providers can return responses for `FETCH RFC822` that Ruby `net-imap` fails to parse with: `Net::IMAP::ResponseParseError: unexpected RPAR (expected ATOM or NIL)` We reproduced this with iCloud IMAP. Authentication, `INBOX` selection, and header fetches worked, but fetching full message content with `RFC822` failed before Chatwoot received a `Mail::Message`. The same mailbox successfully returned full message content when fetched with `BODY.PEEK[]`. > During end-to-end iCloud validation, inbound fetch worked after the IMAP change, but outbound replies through the custom SMTP settings could still fail with a socket read timeout. The OAuth SMTP path already used explicit SMTP timeout values; the custom SMTP path was relying on mailer defaults instead. ## What this change does - Replaces the full message fetch from `RFC822` to `BODY.PEEK[]` - Reads the returned message content from `BODY[]`, which is how `net-imap` exposes the response attribute - Keeps the existing `BODY.PEEK[HEADER]` header-fetch behavior unchanged - Applies `SMTP_OPEN_TIMEOUT` and `SMTP_READ_TIMEOUT` to custom SMTP email-channel replies - Defaults custom SMTP reply delivery to `open_timeout: 15` and `read_timeout: 30` - Updates IMAP service specs for standard and Microsoft IMAP fetch flows - Updates mailer specs for custom SMTP timeout settings `BODY.PEEK[]` is preferable here because it fetches the full message content without marking messages as read. ## Validation - Configured a local email inbox against iCloud IMAP and SMTP - Confirmed `FETCH RFC822` reproduces `Net::IMAP::ResponseParseError: unexpected RPAR (expected ATOM or NIL)` - Confirmed `BODY[]` and `BODY.PEEK[]` fetch the same mailbox successfully - Confirmed Chatwoot imports iCloud messages after the IMAP change - Sent two outbound replies from the Chatwoot UI through iCloud SMTP after applying the timeout settings - Confirmed both UI-created outbound messages were marked `sent`, had iCloud SMTP `source_id` values, and had no `external_error` - Ran `bundle exec rspec spec/services/imap/fetch_email_service_spec.rb spec/services/imap/microsoft_fetch_email_service_spec.rb` - Ran `bundle exec rspec spec/mailers/conversation_reply_mailer_spec.rb`
152 lines
7.3 KiB
Ruby
152 lines
7.3 KiB
Ruby
require 'rails_helper'
|
|
|
|
RSpec.describe Imap::FetchEmailService do
|
|
include ActionMailbox::TestHelper
|
|
let(:logger) { instance_double(ActiveSupport::Logger, info: true, error: true) }
|
|
let(:account) { create(:account) }
|
|
let(:imap_email_channel) { create(:channel_email, :imap_email, account: account) }
|
|
let(:imap) { instance_double(Net::IMAP) }
|
|
let(:eml_content_with_message_id) { Rails.root.join('spec/fixtures/files/only_text.eml').read }
|
|
let(:eml_content_without_message_id) { eml_content_with_message_id.sub(/^Message-ID:.*\n/, '') }
|
|
|
|
describe '#perform' do
|
|
before do
|
|
allow(Rails).to receive(:logger).and_return(logger)
|
|
allow(Net::IMAP).to receive(:new).with(
|
|
imap_email_channel.imap_address, port: imap_email_channel.imap_port, ssl: imap_email_channel.imap_enable_ssl
|
|
).and_return(imap)
|
|
allow(imap).to receive(:authenticate).with(
|
|
'plain', imap_email_channel.imap_login, imap_email_channel.imap_password
|
|
)
|
|
allow(imap).to receive(:select).with('INBOX')
|
|
end
|
|
|
|
context 'when using CRAM-MD5 authentication' do
|
|
let(:cram_md5_channel) { create(:channel_email, :imap_email, account: account, imap_authentication: 'cram-md5') }
|
|
|
|
before do
|
|
allow(Net::IMAP).to receive(:new).with(
|
|
cram_md5_channel.imap_address, port: cram_md5_channel.imap_port, ssl: cram_md5_channel.imap_enable_ssl
|
|
).and_return(imap)
|
|
allow(imap).to receive(:authenticate).with(
|
|
'CRAM-MD5', cram_md5_channel.imap_login, cram_md5_channel.imap_password
|
|
)
|
|
allow(imap).to receive(:select).with('INBOX')
|
|
end
|
|
|
|
it 'uses CRAM-MD5 authentication' do
|
|
travel_to '26.10.2020 10:00'.to_datetime do
|
|
allow(imap).to receive(:search).with(%w[SINCE 25-Oct-2020]).and_return([])
|
|
allow(imap).to receive(:logout)
|
|
|
|
described_class.new(channel: cram_md5_channel).perform
|
|
|
|
expect(imap).to have_received(:authenticate).with(
|
|
'CRAM-MD5', cram_md5_channel.imap_login, cram_md5_channel.imap_password
|
|
)
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when using LOGIN authentication' do
|
|
let(:login_channel) { create(:channel_email, :imap_email, account: account, imap_authentication: 'login') }
|
|
|
|
before do
|
|
allow(Net::IMAP).to receive(:new).with(
|
|
login_channel.imap_address, port: login_channel.imap_port, ssl: login_channel.imap_enable_ssl
|
|
).and_return(imap)
|
|
allow(imap).to receive(:login).with(
|
|
login_channel.imap_login, login_channel.imap_password
|
|
)
|
|
allow(imap).to receive(:select).with('INBOX')
|
|
end
|
|
|
|
it 'uses LOGIN authentication' do
|
|
travel_to '26.10.2020 10:00'.to_datetime do
|
|
allow(imap).to receive(:search).with(%w[SINCE 25-Oct-2020]).and_return([])
|
|
allow(imap).to receive(:logout)
|
|
|
|
described_class.new(channel: login_channel).perform
|
|
|
|
expect(imap).to have_received(:login).with(
|
|
login_channel.imap_login, login_channel.imap_password
|
|
)
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when new emails are available in the mailbox' do
|
|
it 'fetches the emails and returns the emails that are not present in the db' do
|
|
travel_to '26.10.2020 10:00'.to_datetime do
|
|
email_object = create_inbound_email_from_fixture('only_text.eml')
|
|
email_header = Net::IMAP::FetchData.new(1, 'BODY[HEADER]' => eml_content_with_message_id)
|
|
imap_fetch_mail = Net::IMAP::FetchData.new(1, 'BODY[]' => eml_content_with_message_id)
|
|
|
|
allow(imap).to receive(:search).with(%w[SINCE 25-Oct-2020]).and_return([1])
|
|
allow(imap).to receive(:fetch).with([1], 'BODY.PEEK[HEADER]').and_return([email_header])
|
|
allow(imap).to receive(:fetch).with(1, 'BODY.PEEK[]').and_return([imap_fetch_mail])
|
|
allow(imap).to receive(:logout)
|
|
|
|
result = described_class.new(channel: imap_email_channel).perform
|
|
|
|
expect(result.length).to eq 1
|
|
expect(result[0].message_id).to eq email_object.message_id
|
|
expect(imap).to have_received(:search).with(%w[SINCE 25-Oct-2020])
|
|
expect(imap).to have_received(:fetch).with([1], 'BODY.PEEK[HEADER]')
|
|
expect(imap).to have_received(:fetch).with(1, 'BODY.PEEK[]')
|
|
expect(logger).to have_received(:info).with("[IMAP::FETCH_EMAIL_SERVICE] Fetching mails from #{imap_email_channel.email}, found 1.")
|
|
expect(imap).to have_received(:logout)
|
|
end
|
|
end
|
|
|
|
it 'fetches the emails and returns the mail objects that are not present in the db' do
|
|
travel_to '26.10.2020 10:00'.to_datetime do
|
|
email_object = create_inbound_email_from_fixture('only_text.eml')
|
|
create(:message, source_id: email_object.message_id, account: account, inbox: imap_email_channel.inbox)
|
|
|
|
email_header = Net::IMAP::FetchData.new(1, 'BODY[HEADER]' => eml_content_with_message_id)
|
|
|
|
allow(imap).to receive(:search).with(%w[SINCE 25-Oct-2020]).and_return([1])
|
|
allow(imap).to receive(:fetch).with([1], 'BODY.PEEK[HEADER]').and_return([email_header])
|
|
allow(imap).to receive(:logout)
|
|
|
|
result = described_class.new(channel: imap_email_channel).perform
|
|
|
|
expect(result.length).to eq 0
|
|
expect(imap).to have_received(:search).with(%w[SINCE 25-Oct-2020])
|
|
expect(imap).to have_received(:fetch).with([1], 'BODY.PEEK[HEADER]')
|
|
expect(imap).not_to have_received(:fetch).with(1, 'BODY.PEEK[]')
|
|
end
|
|
end
|
|
|
|
it 'does not count emails without message ids toward the sync limit' do
|
|
travel_to '26.10.2020 10:00'.to_datetime do
|
|
email_object = create_inbound_email_from_fixture('only_text.eml')
|
|
max_messages_per_sync = Imap::BaseFetchEmailService::MAX_MESSAGES_PER_SYNC
|
|
empty_message_id_seq_nums = (1..max_messages_per_sync).to_a
|
|
valid_message_seq_num = max_messages_per_sync + 1
|
|
empty_message_id_headers = empty_message_id_seq_nums.map do |seq_num|
|
|
Net::IMAP::FetchData.new(seq_num, 'BODY[HEADER]' => eml_content_without_message_id)
|
|
end
|
|
valid_email_header = Net::IMAP::FetchData.new(valid_message_seq_num, 'BODY[HEADER]' => eml_content_with_message_id)
|
|
imap_fetch_mail = Net::IMAP::FetchData.new(valid_message_seq_num, 'BODY[]' => eml_content_with_message_id)
|
|
|
|
allow(imap).to receive(:search).with(%w[SINCE 25-Oct-2020]).and_return(empty_message_id_seq_nums + [valid_message_seq_num])
|
|
allow(imap).to receive(:fetch).with(empty_message_id_seq_nums, 'BODY.PEEK[HEADER]').and_return(empty_message_id_headers)
|
|
allow(imap).to receive(:fetch).with([valid_message_seq_num], 'BODY.PEEK[HEADER]').and_return([valid_email_header])
|
|
allow(imap).to receive(:fetch).with(valid_message_seq_num, 'BODY.PEEK[]').and_return([imap_fetch_mail])
|
|
allow(imap).to receive(:logout)
|
|
|
|
result = described_class.new(channel: imap_email_channel).perform
|
|
|
|
expect(result.length).to eq 1
|
|
expect(result[0].message_id).to eq email_object.message_id
|
|
expect(imap).to have_received(:fetch).with(empty_message_id_seq_nums, 'BODY.PEEK[HEADER]')
|
|
expect(imap).to have_received(:fetch).with([valid_message_seq_num], 'BODY.PEEK[HEADER]')
|
|
expect(imap).to have_received(:fetch).with(valid_message_seq_num, 'BODY.PEEK[]')
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|