fix(captain): revalidate FAQ matches after edits

enterprise/app/services/captain/llm/conversation_faq_service.rb

@@ -1,6 +1,8 @@
 class Captain::Llm::ConversationFaqService < Llm::BaseAiService
   include Integrations::LlmInstrumentation
 
+  class SuggestionChangedError < StandardError; end
+
   DISTANCE_THRESHOLD = 0.3
   MATCH_LIMIT = 5
   LLM_FEATURE = 'conversation_faq_generation'.freeze
@@ -45,6 +47,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
     return discard_observation(faq) if matching_record(dismissed_suggestions_for_language, faq, embedding)
 
     suggestion = matching_record(open_suggestions_for_language, faq, embedding)
+    matched_content = suggestion&.slice('question', 'answer')
     suggestion ||= assistant.faq_suggestions.create!(
       question: faq.fetch('question'),
       answer: faq.fetch('answer'),
@@ -52,7 +55,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
       language: faq_language
     )
 
-    attach_observation(suggestion, faq)
+    attach_observation(suggestion, faq, matched_content)
   end
 
   def matching_record(relation, faq, embedding)
@@ -96,9 +99,10 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
     raise
   end
 
-  def attach_observation(suggestion, faq)
+  def attach_observation(suggestion, faq, matched_content)
     suggestion.with_lock do
       next unless suggestion.open?
+      raise SuggestionChangedError if matched_content && suggestion.slice('question', 'answer') != matched_content
 
       existing_observation = suggestion.observations.find_by(conversation: conversation)
       next existing_observation if existing_observation

spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb

@@ -340,6 +340,23 @@ RSpec.describe Captain::Llm::ConversationFaqService do
         expect(existing_suggestion.reload.source_count).to eq(2)
         expect(captain_assistant.faq_suggestions.count).to eq(1)
       end
+
+      it 'does not attach the observation when the suggestion changes after classification' do
+        allow(mock_chat).to receive(:ask) do |input|
+          if input.start_with?('{')
+            existing_suggestion.update!(question: 'Edited after classification started')
+            match_response
+          else
+            mock_response
+          end
+        end
+
+        expect do
+          service.generate_suggestions
+        end.to raise_error(described_class::SuggestionChangedError)
+        expect(existing_suggestion.observations.count).to eq(1)
+        expect(existing_suggestion.reload.source_count).to eq(1)
+      end
     end
 
     context 'when a similar open suggestion uses another language' do