diff --git a/app/builders/messages/messenger/message_builder.rb b/app/builders/messages/messenger/message_builder.rb index de23af619..a4fb8b073 100644 --- a/app/builders/messages/messenger/message_builder.rb +++ b/app/builders/messages/messenger/message_builder.rb @@ -1,33 +1,55 @@ class Messages::Messenger::MessageBuilder include ::FileTypeHelper - # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity def process_attachment(attachment) # This check handles very rare case if there are multiple files to attach with only one unsupported file return if unsupported_file_type?(attachment['type']) + attachment_obj, downloaded_file = prepare_attachment(attachment) + return unless attachment_obj + + finalize_attachment(attachment_obj) + ensure + downloaded_file&.close! + end + + def prepare_attachment(attachment) params = attachment_params(attachment) attachment_obj = @message.attachments.new(params.except(:remote_file_url)) - downloaded_file = nil - if facebook_reel?(attachment) - attachment_obj.save! - update_facebook_reel_content(attachment) - elsif params[:remote_file_url] - downloaded_file = attach_file(attachment_obj, params[:remote_file_url]) - return unless downloaded_file + downloaded_file = persist_attachment(attachment, attachment_obj, params[:remote_file_url]) + return [nil, downloaded_file] if params[:remote_file_url].present? && downloaded_file.blank? - attachment_obj.save! - else - attachment_obj.save! - end + [attachment_obj, downloaded_file] + end + + def persist_attachment(attachment, attachment_obj, remote_file_url) + return save_facebook_reel_attachment(attachment, attachment_obj) if facebook_reel?(attachment) + return save_remote_file_attachment(attachment_obj, remote_file_url) if remote_file_url.present? + + attachment_obj.save! + nil + end + + def save_facebook_reel_attachment(attachment, attachment_obj) + attachment_obj.save! + update_facebook_reel_content(attachment) + nil + end + + def save_remote_file_attachment(attachment_obj, remote_file_url) + downloaded_file = attach_file(attachment_obj, remote_file_url) + return if downloaded_file.blank? + + attachment_obj.save! + downloaded_file + end + + def finalize_attachment(attachment_obj) fetch_story_link(attachment_obj) if attachment_obj.file_type == 'story_mention' fetch_ig_story_link(attachment_obj) if attachment_obj.file_type == 'ig_story' fetch_ig_post_link(attachment_obj) if attachment_obj.file_type == 'ig_post' update_attachment_file_type(attachment_obj) - ensure - downloaded_file&.close! end - # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity def attach_file(attachment, file_url) downloaded_file = nil diff --git a/app/services/concerns/downloaded_file_tracking.rb b/app/services/concerns/downloaded_file_tracking.rb new file mode 100644 index 000000000..793adc49b --- /dev/null +++ b/app/services/concerns/downloaded_file_tracking.rb @@ -0,0 +1,14 @@ +module DownloadedFileTracking + private + + def with_downloaded_files + @downloaded_files = [] + yield + ensure + Array(@downloaded_files).each(&:close!) + end + + def track_downloaded_file(downloaded_file) + (@downloaded_files ||= []) << downloaded_file + end +end diff --git a/app/services/sms/incoming_message_service.rb b/app/services/sms/incoming_message_service.rb index 4143d32a3..f9940896e 100644 --- a/app/services/sms/incoming_message_service.rb +++ b/app/services/sms/incoming_message_service.rb @@ -1,24 +1,24 @@ class Sms::IncomingMessageService include ::FileTypeHelper + include ::DownloadedFileTracking pattr_initialize [:inbox!, :params!] def perform - @downloaded_files = [] - set_contact - set_conversation - @message = @conversation.messages.create!( - content: params[:text], - account_id: @inbox.account_id, - inbox_id: @inbox.id, - message_type: :incoming, - sender: @contact, - source_id: params[:id] - ) - attach_files - @message.save! - ensure - close_downloaded_files + with_downloaded_files do + set_contact + set_conversation + @message = @conversation.messages.create!( + content: params[:text], + account_id: @inbox.account_id, + inbox_id: @inbox.id, + message_type: :incoming, + sender: @contact, + source_id: params[:id] + ) + attach_files + @message.save! + end end private @@ -111,12 +111,4 @@ class Sms::IncomingMessageService rescue SafeFetch::Error => e Rails.logger.info "Error downloading SMS attachment from #{media_url}: #{e.message}: Skipping" end - - def track_downloaded_file(attachment_file) - @downloaded_files << attachment_file - end - - def close_downloaded_files - Array(@downloaded_files).each(&:close!) - end end diff --git a/app/services/telegram/attachment_handling.rb b/app/services/telegram/attachment_handling.rb new file mode 100644 index 000000000..207983ad9 --- /dev/null +++ b/app/services/telegram/attachment_handling.rb @@ -0,0 +1,78 @@ +module Telegram::AttachmentHandling + private + + def file_content_type + return :image if image_message? + return :audio if audio_message? + return :video if video_message? + + file_type(params[:message][:document][:mime_type]) + end + + def image_message? + params[:message][:photo].present? || params.dig(:message, :sticker, :thumb).present? + end + + def audio_message? + params[:message][:voice].present? || params[:message][:audio].present? + end + + def video_message? + params[:message][:video].present? || params[:message][:video_note].present? + end + + def attach_files + return unless file + + file_download_path = telegram_file_download_path + return unless file_download_path + + SafeFetch.fetch(file_download_path, allowed_content_types: Attachment::ACCEPTABLE_FILE_TYPES) do |attachment_file| + build_file_attachment(attachment_file) + end + rescue SafeFetch::Error => e + Rails.logger.info "Error downloading Telegram attachment from #{file_download_path}: #{e.message}: Skipping" + end + + def telegram_file_download_path + file_download_path = inbox.channel.get_telegram_file_path(file[:file_id]) + return file_download_path if file_download_path.present? + + Rails.logger.info "Telegram file download path is blank for #{file[:file_id]} : inbox_id: #{inbox.id}" + nil + end + + def build_file_attachment(attachment_file) + track_downloaded_file(attachment_file) + @message.attachments.new( + account_id: @message.account_id, + file_type: file_content_type, + file: { + io: attachment_file.tempfile, + filename: attachment_file.original_filename, + content_type: attachment_file.content_type + } + ) + end + + def file + @file ||= visual_media_params || params[:message][:voice].presence || params[:message][:audio].presence || params[:message][:document].presence + end + + def location_fallback_title + return '' if venue.blank? + + venue[:title] || '' + end + + def venue + @venue ||= params.dig(:message, :venue).presence + end + + def visual_media_params + params[:message][:photo].presence&.last || + params.dig(:message, :sticker, :thumb).presence || + params[:message][:video].presence || + params[:message][:video_note].presence + end +end diff --git a/app/services/telegram/incoming_message_service.rb b/app/services/telegram/incoming_message_service.rb index 4b18d5ee2..d7cf84bae 100644 --- a/app/services/telegram/incoming_message_service.rb +++ b/app/services/telegram/incoming_message_service.rb @@ -1,42 +1,42 @@ # Find the various telegram payload samples here: https://core.telegram.org/bots/webhooks#testing-your-bot-with-updates # https://core.telegram.org/bots/api#available-types -# rubocop:disable Metrics/ClassLength class Telegram::IncomingMessageService include ::FileTypeHelper + include ::DownloadedFileTracking include ::Telegram::ParamHelpers + include ::Telegram::AttachmentHandling pattr_initialize [:inbox!, :params!] def perform - @downloaded_files = [] - # chatwoot doesn't support group conversations at the moment - transform_business_message! - return unless private_message? + with_downloaded_files do + # chatwoot doesn't support group conversations at the moment + transform_business_message! + return unless private_message? - set_contact - update_contact_avatar - set_conversation - # TODO: Since the recent Telegram Business update, we need to explicitly mark messages as read using an additional request. - # Otherwise, the client will see their messages as unread. - # Chatwoot defines a 'read' status in its enum but does not currently update this status for Telegram conversations. - # We have two options: - # 1. Send the read request to Telegram here, immediately when the message is created. - # 2. Properly update the read status in the Chatwoot UI and trigger the Telegram request when the agent actually reads the message. - # See: https://core.telegram.org/bots/api#readbusinessmessage - @message = @conversation.messages.build( - content: telegram_params_message_content, - account_id: @inbox.account_id, - inbox_id: @inbox.id, - message_type: message_type, - sender: message_sender, - content_attributes: telegram_params_content_attributes, - source_id: telegram_params_message_id.to_s - ) + set_contact + update_contact_avatar + set_conversation + # TODO: Since the recent Telegram Business update, we need to explicitly mark messages as read using an additional request. + # Otherwise, the client will see their messages as unread. + # Chatwoot defines a 'read' status in its enum but does not currently update this status for Telegram conversations. + # We have two options: + # 1. Send the read request to Telegram here, immediately when the message is created. + # 2. Properly update the read status in the Chatwoot UI and trigger the Telegram request when the agent actually reads the message. + # See: https://core.telegram.org/bots/api#readbusinessmessage + @message = @conversation.messages.build( + content: telegram_params_message_content, + account_id: @inbox.account_id, + inbox_id: @inbox.id, + message_type: message_type, + sender: message_sender, + content_attributes: telegram_params_content_attributes, + source_id: telegram_params_message_id.to_s + ) - process_message_attachments if message_params? - @message.save! - ensure - close_downloaded_files + process_message_attachments if message_params? + @message.save! + end end private @@ -125,63 +125,6 @@ class Telegram::IncomingMessageService business_message_outgoing? ? nil : @contact end - def file_content_type - return :image if image_message? - return :audio if audio_message? - return :video if video_message? - - file_type(params[:message][:document][:mime_type]) - end - - def image_message? - params[:message][:photo].present? || params.dig(:message, :sticker, :thumb).present? - end - - def audio_message? - params[:message][:voice].present? || params[:message][:audio].present? - end - - def video_message? - params[:message][:video].present? || params[:message][:video_note].present? - end - - def attach_files - return unless file - - file_download_path = telegram_file_download_path - return unless file_download_path - - SafeFetch.fetch( - file_download_path, - allowed_content_types: Attachment::ACCEPTABLE_FILE_TYPES - ) do |attachment_file| - build_file_attachment(attachment_file) - end - rescue SafeFetch::Error => e - Rails.logger.info "Error downloading Telegram attachment from #{file_download_path}: #{e.message}: Skipping" - end - - def telegram_file_download_path - file_download_path = inbox.channel.get_telegram_file_path(file[:file_id]) - return file_download_path if file_download_path.present? - - Rails.logger.info "Telegram file download path is blank for #{file[:file_id]} : inbox_id: #{inbox.id}" - nil - end - - def build_file_attachment(attachment_file) - track_downloaded_file(attachment_file) - @message.attachments.new( - account_id: @message.account_id, - file_type: file_content_type, - file: { - io: attachment_file.tempfile, - filename: attachment_file.original_filename, - content_type: attachment_file.content_type - } - ) - end - def attach_location return unless location @@ -208,20 +151,6 @@ class Telegram::IncomingMessageService ) end - def file - @file ||= visual_media_params || params[:message][:voice].presence || params[:message][:audio].presence || params[:message][:document].presence - end - - def location_fallback_title - return '' if venue.blank? - - venue[:title] || '' - end - - def venue - @venue ||= params.dig(:message, :venue).presence - end - def location @location ||= params.dig(:message, :location).presence end @@ -230,23 +159,7 @@ class Telegram::IncomingMessageService @contact_card ||= params.dig(:message, :contact).presence end - def visual_media_params - params[:message][:photo].presence&.last || - params.dig(:message, :sticker, :thumb).presence || - params[:message][:video].presence || - params[:message][:video_note].presence - end - - def track_downloaded_file(attachment_file) - @downloaded_files << attachment_file - end - - def close_downloaded_files - Array(@downloaded_files).each(&:close!) - end - def transform_business_message! params[:message] = params[:business_message] if params[:business_message] && !params[:message] end end -# rubocop:enable Metrics/ClassLength diff --git a/app/services/tiktok/message_service.rb b/app/services/tiktok/message_service.rb index 1f1b0ad15..c52306f63 100644 --- a/app/services/tiktok/message_service.rb +++ b/app/services/tiktok/message_service.rb @@ -1,4 +1,5 @@ class Tiktok::MessageService + include DownloadedFileTracking include Tiktok::MessagingHelpers pattr_initialize [:channel!, :content!, :outgoing_echo] @@ -32,25 +33,12 @@ class Tiktok::MessageService end def create_message - @downloaded_files = [] - message = conversation.messages.build( - content: message_content, - account_id: channel.inbox.account_id, - inbox_id: channel.inbox.id, - message_type: incoming_message? ? :incoming : :outgoing, - content_attributes: message_content_attributes, - source_id: tt_message_id, - created_at: tt_message_time, - updated_at: tt_message_time - ) - - message.sender = contact_inbox.contact if incoming_message? && !outgoing_echo - message.status = :delivered if outgoing_message? - - create_message_attachments(message) - message.save! - ensure - close_downloaded_files + with_downloaded_files do + message = build_message + apply_message_direction(message) + create_message_attachments(message) + message.save! + end end def message_content @@ -103,6 +91,24 @@ class Tiktok::MessageService attributes end + def build_message + conversation.messages.build( + content: message_content, + account_id: channel.inbox.account_id, + inbox_id: channel.inbox.id, + message_type: incoming_message? ? :incoming : :outgoing, + content_attributes: message_content_attributes, + source_id: tt_message_id, + created_at: tt_message_time, + updated_at: tt_message_time + ) + end + + def apply_message_direction(message) + message.sender = contact_inbox.contact if incoming_message? && !outgoing_echo + message.status = :delivered if outgoing_message? + end + def text_message? tt_message_type == 'text' end @@ -180,12 +186,4 @@ class Tiktok::MessageService def outgoing_message? !incoming_message? end - - def track_downloaded_file(attachment_file) - @downloaded_files << attachment_file - end - - def close_downloaded_files - Array(@downloaded_files).each(&:close!) - end end diff --git a/app/services/twilio/attachment_handling.rb b/app/services/twilio/attachment_handling.rb new file mode 100644 index 000000000..62a43c3af --- /dev/null +++ b/app/services/twilio/attachment_handling.rb @@ -0,0 +1,69 @@ +module Twilio::AttachmentHandling + private + + def attach_files + num_media = params[:NumMedia].to_i + return if num_media.zero? + + num_media.times do |index| + media_url = params[:"MediaUrl#{index}"] + attach_single_file(media_url) if media_url.present? + end + end + + def attach_single_file(media_url) + download_attachment_file(media_url) do |attachment_file| + track_downloaded_file(attachment_file) + @message.attachments.new( + account_id: @message.account_id, + file_type: file_type(attachment_file.content_type), + file: { + io: attachment_file.tempfile, + filename: attachment_file.original_filename, + content_type: attachment_file.content_type + } + ) + end + end + + def download_attachment_file(media_url, &) + download_with_auth(media_url, &) + rescue SafeFetch::Error => e + handle_download_attachment_error(e, media_url, &) + end + + def download_with_auth(media_url, &) + SafeFetch.fetch( + media_url, + http_basic_authentication: attachment_auth_credentials, + allowed_content_types: Attachment::ACCEPTABLE_FILE_TYPES, + & + ) + end + + def attachment_auth_credentials + return [twilio_channel.api_key_sid, twilio_channel.auth_token] if twilio_channel.api_key_sid.present? + + [twilio_channel.account_sid, twilio_channel.auth_token] + end + + def handle_download_attachment_error(error, media_url, &) + Rails.logger.info "Error downloading attachment from Twilio: #{error.message}: Retrying without auth" + SafeFetch.fetch(media_url, allowed_content_types: Attachment::ACCEPTABLE_FILE_TYPES, &) + rescue SafeFetch::Error => e + Rails.logger.info "Error downloading attachment from Twilio: #{e.message}: Skipping" + end + + def location_message? + params[:MessageType] == 'location' && params[:Latitude].present? && params[:Longitude].present? + end + + def attach_location + @message.attachments.new( + account_id: @message.account_id, + file_type: :location, + coordinates_lat: params[:Latitude].to_f, + coordinates_long: params[:Longitude].to_f + ) + end +end diff --git a/app/services/twilio/incoming_message_service.rb b/app/services/twilio/incoming_message_service.rb index 61c16d599..f5c98f2dd 100644 --- a/app/services/twilio/incoming_message_service.rb +++ b/app/services/twilio/incoming_message_service.rb @@ -1,28 +1,28 @@ -# rubocop:disable Metrics/ClassLength class Twilio::IncomingMessageService include ::FileTypeHelper + include ::DownloadedFileTracking + include ::Twilio::AttachmentHandling pattr_initialize [:params!] def perform - @downloaded_files = [] - return if twilio_channel.blank? + with_downloaded_files do + return if twilio_channel.blank? - set_contact - set_conversation - @message = @conversation.messages.build( - content: message_body, - account_id: @inbox.account_id, - inbox_id: @inbox.id, - message_type: :incoming, - sender: @contact, - source_id: params[:SmsSid] - ) - attach_files - attach_location if location_message? - @message.save! - ensure - close_downloaded_files + set_contact + set_conversation + @message = @conversation.messages.build( + content: message_body, + account_id: @inbox.account_id, + inbox_id: @inbox.id, + message_type: :incoming, + sender: @contact, + source_id: params[:SmsSid] + ) + attach_files + attach_location if location_message? + @message.save! + end end private @@ -136,79 +136,6 @@ class Twilio::IncomingMessageService end end - def attach_files - num_media = params[:NumMedia].to_i - return if num_media.zero? - - num_media.times do |i| - media_url = params[:"MediaUrl#{i}"] - attach_single_file(media_url) if media_url.present? - end - end - - def attach_single_file(media_url) - download_attachment_file(media_url) do |attachment_file| - track_downloaded_file(attachment_file) - @message.attachments.new( - account_id: @message.account_id, - file_type: file_type(attachment_file.content_type), - file: { - io: attachment_file.tempfile, - filename: attachment_file.original_filename, - content_type: attachment_file.content_type - } - ) - end - end - - def download_attachment_file(media_url, &) - download_with_auth(media_url, &) - rescue SafeFetch::Error => e - handle_download_attachment_error(e, media_url, &) - end - - def download_with_auth(media_url, &) - auth_credentials = if twilio_channel.api_key_sid.present? - # When using api_key_sid, the auth token should be the api_secret_key - [twilio_channel.api_key_sid, twilio_channel.auth_token] - else - # When using account_sid, the auth token is the account's auth token - [twilio_channel.account_sid, twilio_channel.auth_token] - end - - SafeFetch.fetch( - media_url, - http_basic_authentication: auth_credentials, - allowed_content_types: Attachment::ACCEPTABLE_FILE_TYPES, - & - ) - end - - def handle_download_attachment_error(error, media_url, &) - Rails.logger.info "Error downloading attachment from Twilio: #{error.message}: Retrying without auth" - SafeFetch.fetch( - media_url, - allowed_content_types: Attachment::ACCEPTABLE_FILE_TYPES, - & - ) - rescue SafeFetch::Error => e - Rails.logger.info "Error downloading attachment from Twilio: #{e.message}: Skipping" - nil - end - - def location_message? - params[:MessageType] == 'location' && params[:Latitude].present? && params[:Longitude].present? - end - - def attach_location - @message.attachments.new( - account_id: @message.account_id, - file_type: :location, - coordinates_lat: params[:Latitude].to_f, - coordinates_long: params[:Longitude].to_f - ) - end - def update_contact_name_if_needed return if params[:ProfileName].blank? return if @contact.name == params[:ProfileName] @@ -222,13 +149,4 @@ class Twilio::IncomingMessageService def contact_name_matches_phone_number? @contact.name == phone_number || @contact.name == formatted_phone_number end - - def track_downloaded_file(attachment_file) - @downloaded_files << attachment_file - end - - def close_downloaded_files - Array(@downloaded_files).each(&:close!) - end end -# rubocop:enable Metrics/ClassLength diff --git a/app/services/whatsapp/incoming_message_base_service.rb b/app/services/whatsapp/incoming_message_base_service.rb index b03ddc09f..8eb6b868e 100644 --- a/app/services/whatsapp/incoming_message_base_service.rb +++ b/app/services/whatsapp/incoming_message_base_service.rb @@ -2,6 +2,7 @@ # https://docs.360dialog.com/whatsapp-api/whatsapp-api/media # https://developers.facebook.com/docs/whatsapp/api/media/ class Whatsapp::IncomingMessageBaseService + include ::DownloadedFileTracking include ::Whatsapp::IncomingMessageServiceHelpers pattr_initialize [:inbox!, :params!, :outgoing_echo] @@ -17,36 +18,32 @@ class Whatsapp::IncomingMessageBaseService end # Returns messages array for both regular messages and echo events - def messages_data - @processed_params&.dig(:messages) || @processed_params&.dig(:message_echoes) - end + def messages_data = @processed_params&.dig(:messages) || @processed_params&.dig(:message_echoes) private def process_messages - @downloaded_files = [] + with_downloaded_files do + # We don't support reactions & ephemeral message now, we need to skip processing the message + # if the webhook event is a reaction or an ephermal message or an unsupported message. + return if unprocessable_message_type?(message_type) - # We don't support reactions & ephemeral message now, we need to skip processing the message - # if the webhook event is a reaction or an ephermal message or an unsupported message. - return if unprocessable_message_type?(message_type) + # Multiple webhook events can be received for the same message due to + # misconfigurations in the Meta business manager account. + # We use an atomic Redis SET NX to prevent concurrent workers from both + # processing the same message simultaneously. + return if find_message_by_source_id(messages_data.first[:id]) + return unless lock_message_source_id! - # Multiple webhook events can be received for the same message due to - # misconfigurations in the Meta business manager account. - # We use an atomic Redis SET NX to prevent concurrent workers from both - # processing the same message simultaneously. - return if find_message_by_source_id(messages_data.first[:id]) - return unless lock_message_source_id! + set_contact + return unless @contact + return if @contact.blocked? && !outgoing_echo - set_contact - return unless @contact - return if @contact.blocked? && !outgoing_echo - - ActiveRecord::Base.transaction do - set_conversation - create_messages + ActiveRecord::Base.transaction do + set_conversation + create_messages + end end - ensure - close_downloaded_files end def process_statuses @@ -234,12 +231,4 @@ class Whatsapp::IncomingMessageBaseService formatted_phone_number = TelephoneNumber.parse(phone_number).international_number @contact.name == phone_number || @contact.name == formatted_phone_number end - - def track_downloaded_file(attachment_file) - @downloaded_files << attachment_file - end - - def close_downloaded_files - Array(@downloaded_files).each(&:close!) - end end diff --git a/lib/safe_fetch.rb b/lib/safe_fetch.rb index a7d96ac14..49c1ded96 100644 --- a/lib/safe_fetch.rb +++ b/lib/safe_fetch.rb @@ -1,6 +1,5 @@ require 'ssrf_filter' -# rubocop:disable Metrics/ModuleLength module SafeFetch DEFAULT_ALLOWED_CONTENT_TYPE_PREFIXES = %w[image/ video/ audio/].freeze DEFAULT_ALLOWED_CONTENT_TYPES = %w[ @@ -45,149 +44,21 @@ module SafeFetch class FileTooLargeError < Error; end class UnsupportedContentTypeError < Error; end class UnsupportedMethodError < Error; end +end - # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists - def self.fetch(url, - method: :get, - body: nil, - max_bytes: nil, - open_timeout: DEFAULT_OPEN_TIMEOUT, - read_timeout: DEFAULT_READ_TIMEOUT, - headers: nil, - http_basic_authentication: nil, - allowed_content_type_prefixes: DEFAULT_ALLOWED_CONTENT_TYPE_PREFIXES, - allowed_content_types: DEFAULT_ALLOWED_CONTENT_TYPES, - validate_content_type: true) +require_relative 'safe_fetch/request_options' +require_relative 'safe_fetch/fetcher' + +module SafeFetch + def self.fetch(url, **, &) raise ArgumentError, 'block required' unless block_given? - effective_max_bytes = max_bytes || default_max_bytes - uri = parse_and_validate_url!(url) - filename = filename_for(uri) - tempfile = Tempfile.new('chatwoot-safe-fetch', binmode: true) - - response = stream_to_tempfile( - url, - method, - body, - tempfile, - effective_max_bytes, - open_timeout, - read_timeout, - headers, - http_basic_authentication, - allowed_content_type_prefixes, - allowed_content_types, - validate_content_type - ) - raise HttpError, "#{response.code} #{response.message}" unless response.is_a?(Net::HTTPSuccess) - - tempfile.rewind - yield Result.new( - tempfile: duplicate_tempfile(tempfile), - filename: filename, - content_type: normalize_content_type(response['content-type']) - ) + Fetcher.new(RequestOptions.new(url: url, **)).fetch(&) rescue SsrfFilter::InvalidUriScheme, URI::InvalidURIError => e raise InvalidUrlError, e.message rescue SsrfFilter::Error, Resolv::ResolvError => e raise UnsafeUrlError, e.message rescue Net::OpenTimeout, Net::ReadTimeout, SocketError, OpenSSL::SSL::SSLError => e raise FetchError, e.message - ensure - tempfile&.close! - end - # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists - - class << self - private - - # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists - def stream_to_tempfile(url, method, body, tempfile, max_bytes, open_timeout, read_timeout, headers, http_basic_authentication, - allowed_content_type_prefixes, allowed_content_types, validate_content_type) - response = nil - bytes_written = 0 - - http_method = normalize_method(method) - - SsrfFilter.public_send( - http_method, - url, - headers: headers, - body: body, - request_proc: request_proc(http_basic_authentication), - sensitive_headers: sensitive_headers(headers), - http_options: { open_timeout: open_timeout, read_timeout: read_timeout } - ) do |res| - response = res - next unless res.is_a?(Net::HTTPSuccess) - - if validate_content_type && !allowed_content_type?(res['content-type'], allowed_content_type_prefixes, allowed_content_types) - raise UnsupportedContentTypeError, "content-type not allowed: #{res['content-type']}" - end - - res.read_body do |chunk| - bytes_written += chunk.bytesize - raise FileTooLargeError, "exceeded #{max_bytes} bytes" if bytes_written > max_bytes - - tempfile.write(chunk) - end - end - - response - end - # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists - - def filename_for(uri) - File.basename(uri.path).presence || "download-#{Time.current.to_i}-#{SecureRandom.hex(4)}" - end - - def duplicate_tempfile(tempfile) - duplicated = tempfile.dup - duplicated.rewind - duplicated - end - - def default_max_bytes - limit_mb = GlobalConfigService.load('MAXIMUM_FILE_UPLOAD_SIZE', DEFAULT_MAX_BYTES_FALLBACK_MB).to_i - limit_mb = DEFAULT_MAX_BYTES_FALLBACK_MB if limit_mb <= 0 - limit_mb.megabytes - end - - def request_proc(http_basic_authentication) - return if http_basic_authentication.blank? - - proc { |request| request.basic_auth(*http_basic_authentication) } - end - - def sensitive_headers(headers) - DEFAULT_SENSITIVE_HEADERS | Array(headers).map { |header, _| header.to_s } - end - - def parse_and_validate_url!(url) - uri = URI.parse(url) - raise InvalidUrlError, 'scheme must be http or https' unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS) - raise InvalidUrlError, 'missing host' if uri.host.blank? - - uri - end - - def normalize_method(method) - http_method = method.to_s.downcase.to_sym - return http_method if SsrfFilter::VERB_MAP.key?(http_method) - - raise UnsupportedMethodError, "unsupported method: #{method}" - end - - def normalize_content_type(value) - value.to_s.split(';').first&.strip&.downcase - end - - def allowed_content_type?(value, prefixes, content_types) - mime = normalize_content_type(value) - return false if mime.blank? - - Array(prefixes).any? { |prefix| mime.start_with?(prefix) } || Array(content_types).include?(mime) - end end end -# rubocop:enable Metrics/ModuleLength diff --git a/lib/safe_fetch/fetcher.rb b/lib/safe_fetch/fetcher.rb new file mode 100644 index 000000000..88290a53c --- /dev/null +++ b/lib/safe_fetch/fetcher.rb @@ -0,0 +1,80 @@ +class SafeFetch::Fetcher + def initialize(options) + @options = options + end + + def fetch + with_tempfile do |tempfile| + response = stream_response(tempfile) + raise SafeFetch::HttpError, "#{response.code} #{response.message}" unless response.is_a?(Net::HTTPSuccess) + + yield SafeFetch::Result.new( + tempfile: duplicate_tempfile(tempfile), + filename: options.filename, + content_type: normalize_content_type(response['content-type']) + ) + end + end + + private + + attr_reader :options + + def with_tempfile + tempfile = Tempfile.new('chatwoot-safe-fetch', binmode: true) + yield tempfile + ensure + tempfile&.close! + end + + def stream_response(tempfile) + response = nil + bytes_written = 0 + + SsrfFilter.public_send(options.method, options.url, **options.request_options) do |res| + response = res + next unless res.is_a?(Net::HTTPSuccess) + + validate_content_type!(res['content-type']) + bytes_written = write_response_body(res, tempfile, bytes_written) + end + + response + end + + def validate_content_type!(content_type) + return unless options.validate_content_type? + return if allowed_content_type?(content_type) + + raise SafeFetch::UnsupportedContentTypeError, "content-type not allowed: #{content_type}" + end + + def write_response_body(response, tempfile, bytes_written) + response.read_body do |chunk| + bytes_written += chunk.bytesize + raise SafeFetch::FileTooLargeError, "exceeded #{options.effective_max_bytes} bytes" if bytes_written > options.effective_max_bytes + + tempfile.write(chunk) + end + + bytes_written + end + + def allowed_content_type?(value) + mime = normalize_content_type(value) + return false if mime.blank? + + options.allowed_content_type_prefixes.any? { |prefix| mime.start_with?(prefix) } || + options.allowed_content_types.include?(mime) + end + + def duplicate_tempfile(tempfile) + duplicated = tempfile.dup + duplicated.rewind + duplicated + end + + def normalize_content_type(value) + value.to_s.split(';').first&.strip&.downcase + end +end diff --git a/lib/safe_fetch/request_options.rb b/lib/safe_fetch/request_options.rb new file mode 100644 index 000000000..58e2accae --- /dev/null +++ b/lib/safe_fetch/request_options.rb @@ -0,0 +1,88 @@ +class SafeFetch::RequestOptions + DEFAULTS = { + method: :get, + body: nil, + max_bytes: nil, + open_timeout: SafeFetch::DEFAULT_OPEN_TIMEOUT, + read_timeout: SafeFetch::DEFAULT_READ_TIMEOUT, + headers: nil, + http_basic_authentication: nil, + allowed_content_type_prefixes: SafeFetch::DEFAULT_ALLOWED_CONTENT_TYPE_PREFIXES, + allowed_content_types: SafeFetch::DEFAULT_ALLOWED_CONTENT_TYPES, + validate_content_type: true + }.freeze + + attr_reader :allowed_content_type_prefixes, :allowed_content_types, :body, :headers, + :http_basic_authentication, :method, :open_timeout, :read_timeout, :uri, :url + + def initialize(url:, **options) + config = DEFAULTS.merge(options) + @url = url + @uri = parse_and_validate_url!(url) + @method = normalize_method(config[:method]) + @body = config[:body] + @max_bytes = config[:max_bytes] + @open_timeout = config[:open_timeout] + @read_timeout = config[:read_timeout] + @headers = config[:headers] + @http_basic_authentication = config[:http_basic_authentication] + @allowed_content_type_prefixes = Array(config[:allowed_content_type_prefixes]) + @allowed_content_types = Array(config[:allowed_content_types]) + @validate_content_type = config[:validate_content_type] + end + + def effective_max_bytes + @effective_max_bytes ||= @max_bytes || default_max_bytes + end + + def filename + @filename ||= File.basename(uri.path).presence || "download-#{Time.current.to_i}-#{SecureRandom.hex(4)}" + end + + def request_options + { + headers: headers, + body: body, + request_proc: request_proc, + sensitive_headers: sensitive_headers, + http_options: { open_timeout: open_timeout, read_timeout: read_timeout } + } + end + + def validate_content_type? + @validate_content_type + end + + private + + def default_max_bytes + limit_mb = GlobalConfigService.load('MAXIMUM_FILE_UPLOAD_SIZE', SafeFetch::DEFAULT_MAX_BYTES_FALLBACK_MB).to_i + limit_mb = SafeFetch::DEFAULT_MAX_BYTES_FALLBACK_MB if limit_mb <= 0 + limit_mb.megabytes + end + + def parse_and_validate_url!(value) + parsed_uri = URI.parse(value) + raise SafeFetch::InvalidUrlError, 'scheme must be http or https' unless parsed_uri.is_a?(URI::HTTP) || parsed_uri.is_a?(URI::HTTPS) + raise SafeFetch::InvalidUrlError, 'missing host' if parsed_uri.host.blank? + + parsed_uri + end + + def normalize_method(value) + http_method = value.to_s.downcase.to_sym + return http_method if SsrfFilter::VERB_MAP.key?(http_method) + + raise SafeFetch::UnsupportedMethodError, "unsupported method: #{value}" + end + + def request_proc + return if http_basic_authentication.blank? + + proc { |request| request.basic_auth(*http_basic_authentication) } + end + + def sensitive_headers + SafeFetch::DEFAULT_SENSITIVE_HEADERS | Array(headers).map { |header, _| header.to_s } + end +end