llm tracer

Gemfile

@@ -186,6 +186,9 @@ gem 'ruby_llm-schema'
 
 gem 'shopify_api'
 
+gem 'opentelemetry-sdk'
+gem 'opentelemetry-exporter-otlp'
+
 ### Gems required only in specific deployment environments ###
 ##############################################################
 

Gemfile.lock

@@ -564,6 +564,25 @@ GEM
       actionpack (>= 4.2)
       omniauth (~> 2.0)
     openssl (3.2.0)
+    opentelemetry-api (1.6.0)
+    opentelemetry-common (0.22.0)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-exporter-otlp (0.26.3)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos-types (~> 1.3)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-sdk (~> 1.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-registry (0.4.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-sdk (1.3.2)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.11.0)
+      opentelemetry-api (~> 1.0)
     orm_adapter (0.5.0)
     os (1.1.4)
     ostruct (0.6.1)
@@ -994,6 +1013,8 @@ DEPENDENCIES
   omniauth-google-oauth2 (>= 1.1.3)
   omniauth-oauth2
   omniauth-rails_csrf_protection (~> 1.0, >= 1.0.2)
+  opentelemetry-exporter-otlp
+  opentelemetry-sdk
   pg
   pg_search
   pgvector

config/initializers/captain_instrumentation.rb

@@ -0,0 +1,8 @@
+require 'opentelemetry/sdk'
+require 'opentelemetry/exporter/otlp'
+
+OpenTelemetry::SDK.configure do |c|
+  c.service_name = 'ChatwootAppCaptain'
+end
+
+CaptainTracer = OpenTelemetry.tracer_provider.tracer('CaptainTracer')

enterprise/app/helpers/captain/chat_helper.rb

@@ -1,18 +1,35 @@
 module Captain::ChatHelper
   def request_chat_completion
-    log_chat_completion_request
+    CaptainTracer.in_span('llm_chat_completion') do |span|
+      # span.set_attribute('model', @model)
+      # span.set_attribute('messages', @messages)
+      # span.set_attribute('tools', @tool_registry&.registered_tools || [])
+      span.set_attribute('input', @messages.last[:content])
+      span.set_attribute('temperature', @assistant&.config&.[]('temperature').to_f || 1)
 
-    response = @client.chat(
-      parameters: {
-        model: @model,
-        messages: @messages,
-        tools: @tool_registry&.registered_tools || [],
-        response_format: { type: 'json_object' },
-        temperature: @assistant&.config&.[]('temperature').to_f || 1
-      }
-    )
+      log_chat_completion_request
 
-    handle_response(response)
+      response = @client.chat(
+        parameters: {
+          model: @model,
+          messages: @messages,
+          tools: @tool_registry&.registered_tools || [],
+          response_format: { type: 'json_object' },
+          temperature: @assistant&.config&.[]('temperature').to_f || 1
+        }
+      )
+
+      span.set_attribute('model', response['model'])
+      span.set_attribute('llm.usage.cached_tokens', response.dig('usage', 'prompt_tokens_details', 'cached_tokens'))
+      # span.set_attribute('llm.usage.input_tokens', response.dig('usage', 'input_tokens'))
+      span.set_attribute('llm.usage.output_tokens', response.dig('usage', 'output_tokens'))
+      span.set_attribute('llm.usage.total_tokens', response.dig('usage', 'total_tokens'))
+      span.set_attribute('llm.usage.completion_tokens', response.dig('usage', 'completion_tokens'))
+      span.set_attribute('llm.completions.0.role', response['role'])
+      span.set_attribute('llm.completions.0.content', response['content']&.first&.[]('text') || 'empty-response')
+
+      handle_response(response)
+    end
   rescue StandardError => e
     Rails.logger.error "#{self.class.name} Assistant: #{@assistant.id}, Error in chat completion: #{e}"
     raise e
@@ -41,14 +58,22 @@ module Captain::ChatHelper
   end
 
   def process_tool_call(tool_call)
-    arguments = JSON.parse(tool_call['function']['arguments'])
-    function_name = tool_call['function']['name']
-    tool_call_id = tool_call['id']
+    CaptainTracer.in_span('llm_tool_call') do |span|
+      arguments = JSON.parse(tool_call['function']['arguments'])
+      function_name = tool_call['function']['name']
+      tool_call_id = tool_call['id']
+      span.set_attribute('function_name', function_name)
+      if arguments.present?
+        arguments.each do |key, value|
+          span.set_attribute("arguments.#{key}", value)
+        end
+      end
 
-    if @tool_registry.respond_to?(function_name)
-      execute_tool(function_name, arguments, tool_call_id)
-    else
-      process_invalid_tool_call(function_name, tool_call_id)
+      if @tool_registry.respond_to?(function_name)
+        execute_tool(function_name, arguments, tool_call_id)
+      else
+        process_invalid_tool_call(function_name, tool_call_id)
+      end
     end
   end
 

enterprise/app/services/captain/copilot/chat_service.rb

@@ -20,16 +20,23 @@ class Captain::Copilot::ChatService < Llm::BaseOpenAiService
   end
 
   def generate_response(input)
-    @messages << { role: 'user', content: input } if input.present?
-    response = request_chat_completion
+    CaptainTracer.in_span('copilot_generate_response') do |span|
+      span.set_attribute('assistant_id', @assistant.id)
+      span.set_attribute('account_id', @account.id)
+      span.set_attribute('user_id', @user.id) if @user.present?
+      span.set_attribute('copilot_thread_id', @copilot_thread.id) if @copilot_thread.present?
 
-    Rails.logger.debug { "#{self.class.name} Assistant: #{@assistant.id}, Received response #{response}" }
-    Rails.logger.info(
-      "#{self.class.name} Assistant: #{@assistant.id}, Incrementing response usage for account #{@account.id}"
-    )
-    @account.increment_response_usage
+      @messages << { role: 'user', content: input } if input.present?
+      response = request_chat_completion
 
-    response
+      Rails.logger.debug { "#{self.class.name} Assistant: #{@assistant.id}, Received response #{response}" }
+      Rails.logger.info(
+        "#{self.class.name} Assistant: #{@assistant.id}, Incrementing response usage for account #{@account.id}"
+      )
+      @account.increment_response_usage
+
+      response
+    end
   end
 
   private