feat(captain): group conversation FAQ signals

enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb

@@ -0,0 +1,74 @@
+class Captain::Llm::ConversationFaqPromptsService
+  class << self
+    def generator(language = 'english')
+      <<~PROMPT
+        You create high-quality FAQ candidates from resolved support conversations.
+        Only generate an FAQ when the conversation contains durable, reusable knowledge that would help many future customers.
+
+        ## Source rules
+        - The conversation history contains only customer messages and human support agent messages.
+        - Base every FAQ strictly on information stated in the human support agent messages. Do not infer, generalize, or add external knowledge.
+        - A human support agent must state every fact used in the FAQ answer. Customer messages cannot supply missing answer facts.
+        - The human support agent must provide the final answer. If the agent only greets, asks clarifying questions, asks for contact details, promises to check, shares an attachment, or transfers the conversation, return: `{"faqs":[]}`.
+        - For each FAQ, first identify the exact human support agent message that fully answers it. If no single human agent message gives a complete public answer, remove that FAQ.
+
+        ## Decision gate
+        Return `{"faqs":[]}` unless every generated FAQ can pass all of these checks:
+        1. The answer is fully stated by a human support agent, not by the customer.
+        2. The answer is a public, durable rule or procedure, not a private account action, manual review, troubleshooting session, quote, file, link, or follow-up.
+        3. The answer can be written without private identifiers, customer-specific facts, direct URLs, attachments, invoices, screenshots, or support-ticket steps.
+        4. The question would still make sense in a help center if the original conversation, customer, and agent did not exist.
+        Do not rescue a rejected conversation by rewriting it as a generic support question.
+
+        ## Return no FAQ for
+        - Spam, scams, advertisements, SEO/link-building pitches, adult/gambling/financial promotions, gibberish, abusive content, or conversations unrelated to the business being supported.
+        - Account-specific, order-specific, payment-specific, subscription-specific, login/access, verification, delivery, certificate, or troubleshooting issues, even if they could be rewritten as a general support question.
+        - Conversations that mainly hand off to a human, ask the customer to wait, request private identifiers or contact details, collect screenshots, attachments, or documents, or tell the customer to contact support for case review.
+        - Temporary workarounds, one-off exceptions, unclear answers, unresolved problems, wrong-service conversations, complaints, greetings, or abandoned conversations.
+        - Internal support workflow details, chat session rules, escalation mechanics, ticket-routing instructions, or "someone will get back to you" messages.
+        - Answers that are just a direct/private link, attachment, file, invoice, one-off quote or estimate, account-specific URL, or instructions to open a support ticket.
+        - Questions whose useful answer is "contact support", "wait for the team", "share your details", "we will check", or "this needs manual review".
+        - Questions about whether support can help with a private issue, third-party service, transaction, payment, delivery, or account problem.
+        - Pricing, policy, availability, roadmap, deadline, or legal claims unless the human support agent gives a clear and stable answer in the conversation.
+        - Questions already answered only by asking the customer for more information.
+
+        ## FAQ quality rules
+        - Prefer returning no FAQ over a weak or narrow FAQ.
+        - A good candidate teaches a generally reusable product, service, policy, setup, or process rule that another customer could use without contacting support.
+        - Generate at most one FAQ unless the human agent clearly answered multiple distinct, reusable questions.
+        - Do not create duplicate or overlapping FAQs in the same response.
+        - Questions must be general enough for a help center, not personalized to the current customer.
+        - Remove customer names, order numbers, invoice numbers, IDs, private URLs, phone numbers, emails, screenshots, attachments, and other personal or transaction-specific details.
+        - Answers must be complete, self-contained, and supported by the human agent's messages.
+
+        ## Examples
+        - Customer mentions a price or procedure, then the human agent only greets or says they will check: return `{"faqs":[]}`.
+        - Human agent shares only a private link, file, invoice, quote, screenshot, or attachment: return `{"faqs":[]}`.
+        - Human agent clearly states a public rule, such as which purchases are allowed for a program or service: generate one general FAQ.
+
+        Generate the FAQs only in the #{language}, use no other language.
+        If no suitable reusable FAQ is available, return: `{"faqs":[]}`.
+
+        Return only valid JSON in this exact structure:
+        ```json
+        { "faqs": [ { "question": "", "answer": "" } ] }
+        ```
+      PROMPT
+    end
+
+    def equivalence_classifier
+      <<~PROMPT
+        Decide whether the candidate and existing entries represent the same reusable FAQ.
+
+        Return `same_faq` as true only when both questions have the same user intent and both answers give the same substantive guidance.
+        Wording, grammar, level of detail, and examples may differ. Return false when either entry adds, removes, contradicts, or changes a condition,
+        policy, procedure, audience, product, plan, time frame, or outcome. Related FAQs are not the same FAQ. When uncertain, return false.
+
+        Return only valid JSON in this exact structure:
+        ```json
+        { "same_faq": true }
+        ```
+      PROMPT
+    end
+  end
+end

enterprise/app/services/captain/llm/conversation_faq_service.rb

@@ -2,6 +2,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
   include Integrations::LlmInstrumentation
 
   DISTANCE_THRESHOLD = 0.3
+  MATCH_LIMIT = 5
   LLM_FEATURE = 'conversation_faq_generation'.freeze
 
   def initialize(assistant, conversation)
@@ -9,24 +10,18 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
     @assistant = assistant
     @conversation = conversation
     @content = conversation_faq_content
+    @embedding_service = Captain::Llm::EmbeddingService.new(account_id: conversation.account_id)
   end
 
-  # Generates and deduplicates FAQs from conversation content
-  # Skips processing if there was no human interaction
   def generate_and_deduplicate
     return [] if no_human_interaction?
 
-    new_faqs = generate
-    return [] if new_faqs.empty?
-
-    duplicate_faqs, unique_faqs = find_and_separate_duplicates(new_faqs)
-    save_new_faqs(unique_faqs)
-    log_duplicate_faqs(duplicate_faqs) if Rails.env.development?
+    generate.map { |faq| route_candidate(faq) }
   end
 
   private
 
-  attr_reader :content, :conversation, :assistant
+  attr_reader :content, :conversation, :assistant, :embedding_service
 
   def conversation_faq_content
     [
@@ -51,11 +46,11 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
   def format_conversation_faq_message(message)
     return unless faq_source_message?(message)
 
-    content = message.content_for_llm
-    return if content.blank?
+    message_content = message.content_for_llm
+    return if message_content.blank?
 
     sender = human_support_reply?(message) ? 'Support Agent' : 'User'
-    "#{sender}: #{content}\n"
+    "#{sender}: #{message_content}\n"
   end
 
   def faq_source_message?(message)
@@ -76,92 +71,134 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
     conversation.first_reply_created_at.nil?
   end
 
-  def find_and_separate_duplicates(faqs)
-    duplicate_faqs = []
-    unique_faqs = []
+  def route_candidate(faq)
+    embedding = embedding_service.get_embedding(candidate_text(faq))
 
-    faqs.each do |faq|
-      combined_text = "#{faq['question']}: #{faq['answer']}"
-      embedding = Captain::Llm::EmbeddingService.new(account_id: @conversation.account_id).get_embedding(combined_text)
-      similar_faqs = find_similar_faqs(embedding)
-
-      if similar_faqs.any?
-        duplicate_faqs << { faq: faq, similar_faqs: similar_faqs }
-      else
-        unique_faqs << faq
-      end
-    end
-
-    [duplicate_faqs, unique_faqs]
-  end
-
-  def find_similar_faqs(embedding)
-    similar_faqs = assistant
-                   .responses
-                   .nearest_neighbors(:embedding, embedding, distance: 'cosine')
-    Rails.logger.debug(similar_faqs.map { |faq| [faq.question, faq.neighbor_distance] })
-    similar_faqs.select { |record| record.neighbor_distance < DISTANCE_THRESHOLD }
-  end
-
-  def save_new_faqs(faqs)
-    faqs.map do |faq|
-      assistant.responses.create!(
-        question: faq['question'],
-        answer: faq['answer'],
-        status: 'pending',
-        documentable: conversation
+    if matching_record(assistant.responses.approved, faq, embedding)
+      return Captain::FaqObservation.create!(
+        conversation: conversation,
+        generated_question: faq.fetch('question'),
+        generated_answer: faq.fetch('answer'),
+        status: :discarded
       )
     end
+
+    suggestion = matching_record(assistant.faq_suggestions.open, faq, embedding)
+    suggestion ||= assistant.faq_suggestions.create!(
+      question: faq.fetch('question'),
+      answer: faq.fetch('answer'),
+      embedding: embedding
+    )
+
+    attach_observation(suggestion, faq)
   end
 
-  def log_duplicate_faqs(duplicate_faqs)
-    return if duplicate_faqs.empty?
+  def matching_record(relation, faq, embedding)
+    likely_matches(relation, embedding).find { |record| same_faq?(faq, record) }
+  end
 
-    Rails.logger.info "Found #{duplicate_faqs.length} duplicate FAQs:"
-    duplicate_faqs.each do |duplicate|
-      Rails.logger.info(
-        "Q: #{duplicate[:faq]['question']}\n" \
-        "A: #{duplicate[:faq]['answer']}\n\n" \
-        "Similar existing FAQs: #{duplicate[:similar_faqs].map { |f| "Q: #{f.question} A: #{f.answer}" }.join(', ')}"
-      )
+  def likely_matches(relation, embedding)
+    return [] unless relation.exists?
+
+    relation
+      .nearest_neighbors(:embedding, embedding, distance: 'cosine')
+      .limit(MATCH_LIMIT)
+      .select { |record| record.neighbor_distance < DISTANCE_THRESHOLD }
+  end
+
+  def same_faq?(candidate, existing_record)
+    comparison = {
+      candidate: candidate.slice('question', 'answer'),
+      existing: { question: existing_record.question, answer: existing_record.answer }
+    }
+    prompt = Captain::Llm::ConversationFaqPromptsService.equivalence_classifier
+    response = instrument_llm_call(equivalence_instrumentation_params(prompt, comparison)) do
+      chat
+        .with_params(response_format: { type: 'json_object' })
+        .with_instructions(prompt)
+        .ask(comparison.to_json)
     end
+
+    response_content = sanitize_json_response(response.content)
+    return false if response_content.blank?
+
+    JSON.parse(response_content).fetch('same_faq', false) == true
+  rescue JSON::ParserError, RubyLLM::Error => e
+    Rails.logger.error "FAQ equivalence classification failed: #{e.message}"
+    false
+  end
+
+  def attach_observation(suggestion, faq)
+    suggestion.with_lock do
+      existing_observation = suggestion.observations.find_by(conversation: conversation)
+      next existing_observation if existing_observation
+
+      observation = suggestion.observations.create!(
+        conversation: conversation,
+        generated_question: faq.fetch('question'),
+        generated_answer: faq.fetch('answer'),
+        status: :attached
+      )
+      suggestion.update!(source_count: suggestion.source_count + 1)
+      observation
+    end
+  end
+
+  def candidate_text(faq)
+    "#{faq.fetch('question')}: #{faq.fetch('answer')}"
   end
 
   def generate
-    response = instrument_llm_call(instrumentation_params) do
+    response = instrument_llm_call(generation_instrumentation_params) do
       chat
         .with_params(response_format: { type: 'json_object' })
         .with_instructions(system_prompt)
-        .ask(@content)
+        .ask(content)
     end
-    parse_response(response.content)
+    parse_generation_response(response.content)
   rescue RubyLLM::Error => e
     Rails.logger.error "LLM API Error: #{e.message}"
     []
   end
 
-  def instrumentation_params
+  def generation_instrumentation_params
     {
       span_name: 'llm.captain.conversation_faq',
-      model: @model,
-      temperature: @temperature,
-      account_id: @conversation.account_id,
-      conversation_id: @conversation.display_id,
+      model: model,
+      temperature: temperature,
+      account_id: conversation.account_id,
+      conversation_id: conversation.display_id,
       feature_name: 'conversation_faq',
       messages: [
         { role: 'system', content: system_prompt },
-        { role: 'user', content: @content }
+        { role: 'user', content: content }
       ],
-      metadata: { assistant_id: @assistant.id }
+      metadata: { assistant_id: assistant.id }
+    }
+  end
+
+  def equivalence_instrumentation_params(prompt, comparison)
+    {
+      span_name: 'llm.captain.faq_equivalence',
+      model: model,
+      temperature: temperature,
+      account_id: conversation.account_id,
+      conversation_id: conversation.display_id,
+      feature_name: 'conversation_faq_deduplication',
+      messages: [
+        { role: 'system', content: prompt },
+        { role: 'user', content: comparison.to_json }
+      ],
+      metadata: { assistant_id: assistant.id }
     }
   end
 
   def system_prompt
-    account_language = @conversation.account.locale_english_name
-    Captain::Llm::SystemPromptsService.conversation_faq_generator(account_language)
+    account_language = conversation.account.locale_english_name
+    Captain::Llm::ConversationFaqPromptsService.generator(account_language)
   end
 
-  def parse_response(response)
+  def parse_generation_response(response)
     return [] if response.nil?
 
     JSON.parse(sanitize_json_response(response)).fetch('faqs', [])

enterprise/app/services/captain/llm/system_prompts_service.rb

@@ -51,62 +51,6 @@ class Captain::Llm::SystemPromptsService
       PROMPT
     end
 
-    def conversation_faq_generator(language = 'english')
-      <<~SYSTEM_PROMPT_MESSAGE
-        You create high-quality FAQ candidates from resolved support conversations.
-        Only generate an FAQ when the conversation contains durable, reusable knowledge that would help many future customers.
-
-        ## Source rules
-        - The conversation history contains only customer messages and human support agent messages.
-        - Base every FAQ strictly on information stated in the human support agent messages. Do not infer, generalize, or add external knowledge.
-        - A human support agent must state every fact used in the FAQ answer. Customer messages cannot supply missing answer facts.
-        - The human support agent must provide the final answer. If the agent only greets, asks clarifying questions, asks for contact details, promises to check, shares an attachment, or transfers the conversation, return: `{"faqs":[]}`.
-        - For each FAQ, first identify the exact human support agent message that fully answers it. If no single human agent message gives a complete public answer, remove that FAQ.
-
-        ## Decision gate
-        Return `{"faqs":[]}` unless every generated FAQ can pass all of these checks:
-        1. The answer is fully stated by a human support agent, not by the customer.
-        2. The answer is a public, durable rule or procedure, not a private account action, manual review, troubleshooting session, quote, file, link, or follow-up.
-        3. The answer can be written without private identifiers, customer-specific facts, direct URLs, attachments, invoices, screenshots, or support-ticket steps.
-        4. The question would still make sense in a help center if the original conversation, customer, and agent did not exist.
-        Do not rescue a rejected conversation by rewriting it as a generic support question.
-
-        ## Return no FAQ for
-        - Spam, scams, advertisements, SEO/link-building pitches, adult/gambling/financial promotions, gibberish, abusive content, or conversations unrelated to the business being supported.
-        - Account-specific, order-specific, payment-specific, subscription-specific, login/access, verification, delivery, certificate, or troubleshooting issues, even if they could be rewritten as a general support question.
-        - Conversations that mainly hand off to a human, ask the customer to wait, request private identifiers or contact details, collect screenshots, attachments, or documents, or tell the customer to contact support for case review.
-        - Temporary workarounds, one-off exceptions, unclear answers, unresolved problems, wrong-service conversations, complaints, greetings, or abandoned conversations.
-        - Internal support workflow details, chat session rules, escalation mechanics, ticket-routing instructions, or "someone will get back to you" messages.
-        - Answers that are just a direct/private link, attachment, file, invoice, one-off quote or estimate, account-specific URL, or instructions to open a support ticket.
-        - Questions whose useful answer is "contact support", "wait for the team", "share your details", "we will check", or "this needs manual review".
-        - Questions about whether support can help with a private issue, third-party service, transaction, payment, delivery, or account problem.
-        - Pricing, policy, availability, roadmap, deadline, or legal claims unless the human support agent gives a clear and stable answer in the conversation.
-        - Questions already answered only by asking the customer for more information.
-
-        ## FAQ quality rules
-        - Prefer returning no FAQ over a weak or narrow FAQ.
-        - A good candidate teaches a generally reusable product, service, policy, setup, or process rule that another customer could use without contacting support.
-        - Generate at most one FAQ unless the human agent clearly answered multiple distinct, reusable questions.
-        - Do not create duplicate or overlapping FAQs in the same response.
-        - Questions must be general enough for a help center, not personalized to the current customer.
-        - Remove customer names, order numbers, invoice numbers, IDs, private URLs, phone numbers, emails, screenshots, attachments, and other personal or transaction-specific details.
-        - Answers must be complete, self-contained, and supported by the human agent's messages.
-
-        ## Examples
-        - Customer mentions a price or procedure, then the human agent only greets or says they will check: return `{"faqs":[]}`.
-        - Human agent shares only a private link, file, invoice, quote, screenshot, or attachment: return `{"faqs":[]}`.
-        - Human agent clearly states a public rule, such as which purchases are allowed for a program or service: generate one general FAQ.
-
-        Generate the FAQs only in the #{language}, use no other language.
-        If no suitable reusable FAQ is available, return: `{"faqs":[]}`.
-
-        Return only valid JSON in this exact structure:
-        ```json
-        { "faqs": [ { "question": "", "answer": "" } ] }
-        ```
-      SYSTEM_PROMPT_MESSAGE
-    end
-
     def notes_generator(language = 'english')
       <<~SYSTEM_PROMPT_MESSAGE
         You are a note taker looking to convert the conversation with a contact into actionable notes for the CRM.

spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb

@@ -2,7 +2,7 @@ require 'rails_helper'
 
 RSpec.describe Captain::Llm::ConversationFaqService do
   let(:captain_assistant) { create(:captain_assistant) }
-  let(:conversation) { create(:conversation, first_reply_created_at: Time.zone.now) }
+  let(:conversation) { create(:conversation, account: captain_assistant.account, first_reply_created_at: Time.zone.now) }
   let(:service) { described_class.new(captain_assistant, conversation) }
   let(:embedding_service) { instance_double(Captain::Llm::EmbeddingService) }
   let(:mock_chat) { instance_double(RubyLLM::Chat) }
@@ -15,6 +15,8 @@ RSpec.describe Captain::Llm::ConversationFaqService do
   let(:mock_response) do
     instance_double(RubyLLM::Message, content: { faqs: sample_faqs }.to_json)
   end
+  let(:embedding_one) { [1.0] + Array.new(1535, 0.0) }
+  let(:embedding_two) { [0.0, 1.0] + Array.new(1534, 0.0) }
 
   before do
     create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
@@ -29,8 +31,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
   describe '#generate_and_deduplicate' do
     context 'when successful' do
       before do
-        allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
-        allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([])
+        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one, embedding_two)
       end
 
       it 'uses the conversation FAQ generation feature model' do
@@ -136,19 +137,24 @@ RSpec.describe Captain::Llm::ConversationFaqService do
         service.generate_and_deduplicate
       end
 
-      it 'creates new FAQs for valid conversation content' do
+      it 'creates suggestions instead of trusted FAQs for valid conversation content' do
         expect do
           service.generate_and_deduplicate
-        end.to change(captain_assistant.responses, :count).by(2)
+        end.to change(captain_assistant.faq_suggestions, :count).by(2)
+        expect(Captain::FaqObservation.count).to eq(2)
+        expect(captain_assistant.responses.count).to be_zero
       end
 
-      it 'saves FAQs with pending status linked to conversation' do
+      it 'saves open suggestions with one attached source each' do
         service.generate_and_deduplicate
         expect(
-          captain_assistant.responses.pluck(:question, :answer, :status, :documentable_id)
+          captain_assistant.faq_suggestions.pluck(:question, :answer, :status, :source_count)
         ).to contain_exactly(
-          ['What is the purpose?', 'To help users.', 'pending', conversation.id],
-          ['How does it work?', 'Through AI.', 'pending', conversation.id]
+          ['What is the purpose?', 'To help users.', 'open', 1],
+          ['How does it work?', 'Through AI.', 'open', 1]
+        )
+        expect(Captain::FaqObservation.attached.pluck(:conversation_id)).to contain_exactly(
+          conversation.id, conversation.id
         )
       end
     end
@@ -168,26 +174,49 @@ RSpec.describe Captain::Llm::ConversationFaqService do
 
     context 'when finding duplicates' do
       let(:existing_response) do
-        create(:captain_assistant_response, assistant: captain_assistant, question: 'Similar question', answer: 'Similar answer')
-      end
-      let(:similar_neighbor) do
-        OpenStruct.new(
-          id: 1,
-          question: existing_response.question,
-          answer: existing_response.answer,
-          neighbor_distance: 0.1
-        )
+        create(:captain_assistant_response, assistant: captain_assistant, account: captain_assistant.account,
+                                            question: 'Similar question', answer: 'Similar answer', embedding: embedding_one)
       end
+      let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
 
       before do
-        allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
-        allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([similar_neighbor])
+        existing_response
+        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
+        allow(mock_chat).to receive(:ask) do |input|
+          input.start_with?('{') ? equivalence_response : mock_response
+        end
       end
 
-      it 'filters out duplicate FAQs based on embedding similarity' do
+      it 'discards candidates the LLM confirms are covered by an approved FAQ' do
         expect do
           service.generate_and_deduplicate
-        end.not_to change(captain_assistant.responses, :count)
+        end.to change(Captain::FaqObservation.discarded, :count).by(2)
+        expect(captain_assistant.faq_suggestions.count).to be_zero
+      end
+    end
+
+    context 'when an open suggestion is the same FAQ' do
+      let(:sample_faqs) { [{ 'question' => 'How can I use the feature?', 'answer' => 'Enable it in settings.' }] }
+      let(:existing_suggestion) do
+        captain_assistant.faq_suggestions.create!(question: 'How do I enable the feature?', answer: 'Turn it on in settings.',
+                                                  embedding: embedding_one, source_count: 1)
+      end
+      let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
+
+      before do
+        existing_suggestion
+        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
+        allow(mock_chat).to receive(:ask) do |input|
+          input.start_with?('{') ? equivalence_response : mock_response
+        end
+      end
+
+      it 'attaches the observation and increments the source count' do
+        expect do
+          service.generate_and_deduplicate
+        end.to change(existing_suggestion.observations, :count).by(1)
+        expect(existing_suggestion.reload.source_count).to eq(2)
+        expect(captain_assistant.faq_suggestions.count).to eq(1)
       end
     end
 
@@ -236,17 +265,17 @@ RSpec.describe Captain::Llm::ConversationFaqService do
   describe 'language handling' do
     context 'when conversation has different language' do
       let(:account) { create(:account, locale: 'fr') }
+      let(:captain_assistant) { create(:captain_assistant, account: account) }
       let(:conversation) do
         create(:conversation, account: account, first_reply_created_at: Time.zone.now)
       end
 
       before do
-        allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
-        allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([])
+        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one, embedding_two)
       end
 
       it 'uses account language for system prompt' do
-        expect(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator)
+        expect(Captain::Llm::ConversationFaqPromptsService).to receive(:generator)
           .with('french')
           .at_least(:once)
           .and_call_original