From f57cb830f4a98771a6de806a0a27bea716d1893b Mon Sep 17 00:00:00 2001 From: Pranav Date: Mon, 18 Aug 2025 20:37:18 -0700 Subject: [PATCH] llm tracer --- Gemfile | 3 + Gemfile.lock | 21 +++++++ .../initializers/captain_instrumentation.rb | 8 +++ enterprise/app/helpers/captain/chat_helper.rb | 61 +++++++++++++------ .../services/captain/copilot/chat_service.rb | 23 ++++--- 5 files changed, 90 insertions(+), 26 deletions(-) create mode 100644 config/initializers/captain_instrumentation.rb diff --git a/Gemfile b/Gemfile index a3b3c0ae8..534170f3e 100644 --- a/Gemfile +++ b/Gemfile @@ -186,6 +186,9 @@ gem 'ruby_llm-schema' gem 'shopify_api' +gem 'opentelemetry-sdk' +gem 'opentelemetry-exporter-otlp' + ### Gems required only in specific deployment environments ### ############################################################## diff --git a/Gemfile.lock b/Gemfile.lock index be258f524..232b8e4c8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -564,6 +564,25 @@ GEM actionpack (>= 4.2) omniauth (~> 2.0) openssl (3.2.0) + opentelemetry-api (1.6.0) + opentelemetry-common (0.22.0) + opentelemetry-api (~> 1.0) + opentelemetry-exporter-otlp (0.26.3) + google-protobuf (~> 3.14) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-sdk (~> 1.2) + opentelemetry-semantic_conventions + opentelemetry-registry (0.4.0) + opentelemetry-api (~> 1.1) + opentelemetry-sdk (1.3.2) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-registry (~> 0.2) + opentelemetry-semantic_conventions + opentelemetry-semantic_conventions (1.11.0) + opentelemetry-api (~> 1.0) orm_adapter (0.5.0) os (1.1.4) ostruct (0.6.1) @@ -994,6 +1013,8 @@ DEPENDENCIES omniauth-google-oauth2 (>= 1.1.3) omniauth-oauth2 omniauth-rails_csrf_protection (~> 1.0, >= 1.0.2) + opentelemetry-exporter-otlp + opentelemetry-sdk pg pg_search pgvector diff --git a/config/initializers/captain_instrumentation.rb b/config/initializers/captain_instrumentation.rb new file mode 100644 index 000000000..c6314b38c --- /dev/null +++ b/config/initializers/captain_instrumentation.rb @@ -0,0 +1,8 @@ +require 'opentelemetry/sdk' +require 'opentelemetry/exporter/otlp' + +OpenTelemetry::SDK.configure do |c| + c.service_name = 'ChatwootAppCaptain' +end + +CaptainTracer = OpenTelemetry.tracer_provider.tracer('CaptainTracer') diff --git a/enterprise/app/helpers/captain/chat_helper.rb b/enterprise/app/helpers/captain/chat_helper.rb index f90b8d07e..b44958306 100644 --- a/enterprise/app/helpers/captain/chat_helper.rb +++ b/enterprise/app/helpers/captain/chat_helper.rb @@ -1,18 +1,35 @@ module Captain::ChatHelper def request_chat_completion - log_chat_completion_request + CaptainTracer.in_span('llm_chat_completion') do |span| + # span.set_attribute('model', @model) + # span.set_attribute('messages', @messages) + # span.set_attribute('tools', @tool_registry&.registered_tools || []) + span.set_attribute('input', @messages.last[:content]) + span.set_attribute('temperature', @assistant&.config&.[]('temperature').to_f || 1) - response = @client.chat( - parameters: { - model: @model, - messages: @messages, - tools: @tool_registry&.registered_tools || [], - response_format: { type: 'json_object' }, - temperature: @assistant&.config&.[]('temperature').to_f || 1 - } - ) + log_chat_completion_request - handle_response(response) + response = @client.chat( + parameters: { + model: @model, + messages: @messages, + tools: @tool_registry&.registered_tools || [], + response_format: { type: 'json_object' }, + temperature: @assistant&.config&.[]('temperature').to_f || 1 + } + ) + + span.set_attribute('model', response['model']) + span.set_attribute('llm.usage.cached_tokens', response.dig('usage', 'prompt_tokens_details', 'cached_tokens')) + # span.set_attribute('llm.usage.input_tokens', response.dig('usage', 'input_tokens')) + span.set_attribute('llm.usage.output_tokens', response.dig('usage', 'output_tokens')) + span.set_attribute('llm.usage.total_tokens', response.dig('usage', 'total_tokens')) + span.set_attribute('llm.usage.completion_tokens', response.dig('usage', 'completion_tokens')) + span.set_attribute('llm.completions.0.role', response['role']) + span.set_attribute('llm.completions.0.content', response['content']&.first&.[]('text') || 'empty-response') + + handle_response(response) + end rescue StandardError => e Rails.logger.error "#{self.class.name} Assistant: #{@assistant.id}, Error in chat completion: #{e}" raise e @@ -41,14 +58,22 @@ module Captain::ChatHelper end def process_tool_call(tool_call) - arguments = JSON.parse(tool_call['function']['arguments']) - function_name = tool_call['function']['name'] - tool_call_id = tool_call['id'] + CaptainTracer.in_span('llm_tool_call') do |span| + arguments = JSON.parse(tool_call['function']['arguments']) + function_name = tool_call['function']['name'] + tool_call_id = tool_call['id'] + span.set_attribute('function_name', function_name) + if arguments.present? + arguments.each do |key, value| + span.set_attribute("arguments.#{key}", value) + end + end - if @tool_registry.respond_to?(function_name) - execute_tool(function_name, arguments, tool_call_id) - else - process_invalid_tool_call(function_name, tool_call_id) + if @tool_registry.respond_to?(function_name) + execute_tool(function_name, arguments, tool_call_id) + else + process_invalid_tool_call(function_name, tool_call_id) + end end end diff --git a/enterprise/app/services/captain/copilot/chat_service.rb b/enterprise/app/services/captain/copilot/chat_service.rb index ca7057c86..eb5d72ca2 100644 --- a/enterprise/app/services/captain/copilot/chat_service.rb +++ b/enterprise/app/services/captain/copilot/chat_service.rb @@ -20,16 +20,23 @@ class Captain::Copilot::ChatService < Llm::BaseOpenAiService end def generate_response(input) - @messages << { role: 'user', content: input } if input.present? - response = request_chat_completion + CaptainTracer.in_span('copilot_generate_response') do |span| + span.set_attribute('assistant_id', @assistant.id) + span.set_attribute('account_id', @account.id) + span.set_attribute('user_id', @user.id) if @user.present? + span.set_attribute('copilot_thread_id', @copilot_thread.id) if @copilot_thread.present? - Rails.logger.debug { "#{self.class.name} Assistant: #{@assistant.id}, Received response #{response}" } - Rails.logger.info( - "#{self.class.name} Assistant: #{@assistant.id}, Incrementing response usage for account #{@account.id}" - ) - @account.increment_response_usage + @messages << { role: 'user', content: input } if input.present? + response = request_chat_completion - response + Rails.logger.debug { "#{self.class.name} Assistant: #{@assistant.id}, Received response #{response}" } + Rails.logger.info( + "#{self.class.name} Assistant: #{@assistant.id}, Incrementing response usage for account #{@account.id}" + ) + @account.increment_response_usage + + response + end end private