refactor: split connection timeout handling into modular concerns

- Extract query inspection logic to Database::QueryInspection
- Extract query logging to Database::QueryLogging
- Move connection diagnostics to Database::ConnectionDiagnostics
- Fix all rubocop issues by improving code organization

app/controllers/concerns/database/connection_diagnostics.rb

@@ -0,0 +1,51 @@
+module Database::ConnectionDiagnostics
+  extend ActiveSupport::Concern
+  include Database::QueryInspection
+  include Database::QueryLogging
+
+  private
+
+  def log_connection_pool_stats(connection_pool)
+    {
+      pool_size: connection_pool.size,
+      active_connections: connection_pool.connections.count(&:in_use?),
+      total_connections: connection_pool.connections.count,
+      waiting_threads: connection_pool.num_waiting_in_queue,
+      checkout_timeout: connection_pool.checkout_timeout
+    }
+  end
+
+  def connection_status_sql
+    <<~SQL.squish
+      SELECT count(*) as connection_count, state#{' '}
+      FROM pg_stat_activity#{' '}
+      GROUP BY state;
+    SQL
+  end
+
+  def fetch_connection_diagnostics
+    {
+      active_queries: fetch_active_queries,
+      locked_queries: fetch_locked_queries,
+      connection_status: ActiveRecord::Base.connection.execute(connection_status_sql).to_a
+    }
+  rescue StandardError => e
+    Rails.logger.error "Error fetching active query data: #{e.message}"
+    { active_queries: [], locked_queries: [] }
+  end
+
+  def caller_info(exception)
+    {
+      process_id: Process.pid,
+      thread_id: Thread.current.object_id,
+      backtrace: exception.backtrace&.first(15) || []
+    }
+  end
+
+  def log_timeout_error(exception, connection_info)
+    Rails.logger.error "ActiveRecord::ConnectionTimeoutError: #{exception.message}"
+    Rails.logger.error "Connection Pool Stats: #{connection_info.except(:active_queries, :locked_queries).inspect}"
+    log_active_queries(connection_info[:active_queries])
+    log_locked_queries(connection_info[:locked_queries])
+  end
+end

app/controllers/concerns/database/query_inspection.rb

@@ -0,0 +1,91 @@
+module Database::QueryInspection
+  extend ActiveSupport::Concern
+
+  private
+
+  def active_queries_sql
+    <<~SQL.squish
+      SELECT pid,#{' '}
+             now() - pg_stat_activity.query_start AS duration,
+             query,
+             state,
+             wait_event_type,
+             wait_event,
+             backend_type,
+             application_name,
+             client_addr,
+             usename
+      FROM pg_stat_activity#{' '}
+      WHERE state <> 'idle'
+        AND query NOT ILIKE '%pg_stat_activity%'
+      ORDER BY duration DESC;
+    SQL
+  end
+
+  def process_active_query_row(row)
+    {
+      pid: row['pid'],
+      duration: row['duration'].to_s,
+      state: row['state'],
+      query: row['query'],
+      wait_event_type: row['wait_event_type'],
+      wait_event: row['wait_event'],
+      backend_type: row['backend_type'],
+      application_name: row['application_name'],
+      client_addr: row['client_addr'],
+      username: row['usename']
+    }
+  end
+
+  def fetch_active_queries
+    query_data = ActiveRecord::Base.connection.execute(active_queries_sql)
+    query_data.map { |row| process_active_query_row(row) }
+  end
+
+  def locked_queries_sql
+    <<~SQL.squish
+      SELECT blocked_locks.pid AS blocked_pid,
+             blocked_activity.usename AS blocked_user,
+             blocking_locks.pid AS blocking_pid,
+             blocking_activity.usename AS blocking_user,
+             blocked_activity.query AS blocked_statement,
+             blocking_activity.query AS blocking_statement,
+             now() - blocking_activity.query_start AS blocking_duration
+      FROM pg_catalog.pg_locks blocked_locks
+      JOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid
+      JOIN pg_catalog.pg_locks blocking_locks#{' '}
+          ON blocking_locks.locktype = blocked_locks.locktype
+          AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE
+          AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
+          AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
+          AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
+          AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
+          AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
+          AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
+          AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
+          AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
+          AND blocking_locks.pid != blocked_locks.pid
+      JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid
+      WHERE NOT blocked_locks.granted;
+    SQL
+  end
+
+  def process_lock_row(row)
+    {
+      blocked_pid: row['blocked_pid'],
+      blocked_user: row['blocked_user'],
+      blocking_pid: row['blocking_pid'],
+      blocking_user: row['blocking_user'],
+      blocked_statement: row['blocked_statement'],
+      blocking_statement: row['blocking_statement'],
+      blocking_duration: row['blocking_duration'].to_s
+    }
+  end
+
+  def fetch_locked_queries
+    lock_data = ActiveRecord::Base.connection.execute(locked_queries_sql)
+    return [] if lock_data.count.zero?
+
+    lock_data.map { |row| process_lock_row(row) }
+  end
+end

app/controllers/concerns/database/query_logging.rb

@@ -0,0 +1,33 @@
+module Database::QueryLogging
+  extend ActiveSupport::Concern
+
+  private
+
+  def log_query_details(query_info, index)
+    Rails.logger.error "Query ##{index + 1} [PID: #{query_info[:pid]}] [Duration: #{query_info[:duration]}] [State: #{query_info[:state]}]:"
+    Rails.logger.error "App: #{query_info[:application_name]} User: #{query_info[:username]} Client: #{query_info[:client_addr]}"
+    Rails.logger.error "Waiting: #{query_info[:wait_event_type]} / #{query_info[:wait_event]}" if query_info[:wait_event_type].present?
+    Rails.logger.error query_info[:query]
+  end
+
+  def log_active_queries(active_queries)
+    if active_queries.any?
+      Rails.logger.error "Active Database Queries (#{active_queries.count}):"
+      active_queries.each_with_index { |query_info, index| log_query_details(query_info, index) }
+    else
+      Rails.logger.error 'No active queries found or unable to retrieve query information'
+    end
+  end
+
+  def log_locked_queries(locked_queries)
+    return if locked_queries.blank?
+
+    Rails.logger.error "Locked Queries (#{locked_queries.count}):"
+    locked_queries.each_with_index do |lock_info, index|
+      Rails.logger.error "Lock ##{index + 1}: PID #{lock_info[:blocked_pid]} blocked by PID #{lock_info[:blocking_pid]}"
+      Rails.logger.error "Duration: #{lock_info[:blocking_duration]}"
+      Rails.logger.error "Blocked query: #{lock_info[:blocked_statement]}"
+      Rails.logger.error "Blocking query: #{lock_info[:blocking_statement]}"
+    end
+  end
+end

app/controllers/concerns/request_exception_handler.rb

@@ -1,8 +1,10 @@
 module RequestExceptionHandler
   extend ActiveSupport::Concern
+  include Database::ConnectionDiagnostics
 
   included do
     rescue_from ActiveRecord::RecordInvalid, with: :render_record_invalid
+    rescue_from ActiveRecord::ConnectionTimeoutError, with: :handle_connection_timeout
   end
 
   private
@@ -18,11 +20,36 @@ module RequestExceptionHandler
   rescue ActionController::ParameterMissing => e
     log_handled_error(e)
     render_could_not_create_error(e.message)
+  rescue ActiveRecord::ConnectionTimeoutError => e
+    handle_connection_timeout(e)
   ensure
     # to address the thread variable leak issues in Puma/Thin webserver
     Current.reset
   end
 
+  def handle_connection_timeout(exception)
+    connection_pool = ActiveRecord::Base.connection_pool
+    connection_info = log_connection_pool_stats(connection_pool)
+
+    # Gather diagnostic info
+    diagnostics = fetch_connection_diagnostics
+    connection_info.merge!(diagnostics)
+    connection_info[:caller_info] = caller_info(exception)
+
+    # Log error details
+    log_timeout_error(exception, connection_info)
+
+    # Report to exception tracker
+    ChatwootExceptionTracker.new(
+      exception,
+      user: Current.user,
+      account: Current.account,
+      additional_context: { connection_info: connection_info }
+    ).capture_exception
+
+    render_service_unavailable('Database connection timeout. Please try again later.')
+  end
+
   def render_unauthorized(message)
     render json: { error: message }, status: :unauthorized
   end
@@ -43,6 +70,10 @@ module RequestExceptionHandler
     render json: { error: message }, status: :internal_server_error
   end
 
+  def render_service_unavailable(message)
+    render json: { error: message }, status: :service_unavailable
+  end
+
   def render_record_invalid(exception)
     log_handled_error(exception)
     render json: {

lib/chatwoot_exception_tracker.rb

@@ -5,10 +5,11 @@
 ############
 
 class ChatwootExceptionTracker
-  def initialize(exception, user: nil, account: nil)
+  def initialize(exception, user: nil, account: nil, additional_context: {})
     @exception = exception
     @user = user
     @account = account
+    @additional_context = additional_context
   end
 
   def capture_exception
@@ -26,6 +27,12 @@ class ChatwootExceptionTracker
       end
 
       scope.set_user(id: @user.id, email: @user.email) if @user.is_a?(User)
+
+      # Add additional context if provided
+      @additional_context.each do |context_name, context_data|
+        scope.set_context(context_name.to_s, context_data)
+      end
+
       Sentry.capture_exception(@exception)
     end
   end