From 102f19fe417ff68e49ec60077ff07c2355ec5811 Mon Sep 17 00:00:00 2001 From: Aakash Bakhle <48802744+aakashb95@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:31:08 +0530 Subject: [PATCH 1/2] feat(captain): add FAQ suggestion data model (1/3) (#14977) Resolved conversations need a separate suggestion layer so repeated FAQ signals can be grouped without creating untrusted knowledge entries. This PR adds the persistence foundation only; it introduces no user-facing behavior by itself. ## Closes - [CW-7495](https://linear.app/chatwoot/issue/CW-7495/backend-llm-changes-to-make-conversation-faqs-as-signalssuggestions) (stacked PR 1/3; the issue is complete after the full stack lands) ## What changed - Added `captain_faq_suggestions` with question, answer, embedding, source count, and review status. - Added `captain_faq_observations` to retain conversation-level signals. - Added Captain assistant, account, and conversation associations. - Added vector and lookup indexes for semantic grouping. ## How to test This layer has no standalone UI behavior. Apply the migration and confirm Captain assistants can persist open FAQ suggestions with attached conversation observations. --- ...13184351_create_captain_faq_suggestions.rb | 48 +++++++++++++++++ db/schema.rb | 35 ++++++++++++- enterprise/app/models/captain/assistant.rb | 1 + .../app/models/captain/faq_observation.rb | 42 +++++++++++++++ .../app/models/captain/faq_suggestion.rb | 51 +++++++++++++++++++ .../app/models/enterprise/concerns/account.rb | 2 + .../enterprise/concerns/conversation.rb | 1 + 7 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20260713184351_create_captain_faq_suggestions.rb create mode 100644 enterprise/app/models/captain/faq_observation.rb create mode 100644 enterprise/app/models/captain/faq_suggestion.rb diff --git a/db/migrate/20260713184351_create_captain_faq_suggestions.rb b/db/migrate/20260713184351_create_captain_faq_suggestions.rb new file mode 100644 index 000000000..6bc03f387 --- /dev/null +++ b/db/migrate/20260713184351_create_captain_faq_suggestions.rb @@ -0,0 +1,48 @@ +class CreateCaptainFaqSuggestions < ActiveRecord::Migration[7.1] + def change + create_faq_suggestions + create_faq_observations + end + + private + + def create_faq_suggestions + create_table :captain_faq_suggestions do |t| + t.string :question, null: false + t.text :answer, null: false + t.vector :embedding, limit: 1536 + t.references :assistant, null: false, index: true + t.references :account, null: false, index: true + t.string :language, null: false, default: 'en' + t.integer :source_count, null: false, default: 0 + t.integer :status, null: false, default: 0 + + t.timestamps + end + + add_index :captain_faq_suggestions, [:account_id, :assistant_id, :status, :language], + name: 'idx_cap_faq_suggestions_on_account_assistant_status_language' + add_index :captain_faq_suggestions, :embedding, using: :ivfflat, + name: 'vector_idx_captain_faq_suggestions_embedding', + opclass: :vector_cosine_ops + end + + def create_faq_observations + create_table :captain_faq_observations do |t| + t.references :account, null: false, index: true + t.references :conversation, null: false, index: true + t.references :faq_suggestion, index: true + t.string :generated_question, null: false + t.text :generated_answer, null: false + t.string :language, null: false, default: 'en' + t.integer :status, null: false, default: 0 + + t.timestamps + end + + add_index :captain_faq_observations, [:conversation_id, :faq_suggestion_id], + unique: true, + where: 'faq_suggestion_id IS NOT NULL', + name: 'idx_captain_faq_observations_on_conversation_and_suggestion' + end +end diff --git a/db/schema.rb b/db/schema.rb index f02b613d9..43e7135b9 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2026_07_10_000000) do +ActiveRecord::Schema[7.1].define(version: 2026_07_13_184351) do # These extensions should be enabled to support this database enable_extension "pg_stat_statements" enable_extension "pg_trgm" @@ -417,6 +417,39 @@ ActiveRecord::Schema[7.1].define(version: 2026_07_10_000000) do t.index ["status"], name: "index_captain_documents_on_status" end + create_table "captain_faq_observations", force: :cascade do |t| + t.bigint "account_id", null: false + t.bigint "conversation_id", null: false + t.bigint "faq_suggestion_id" + t.string "generated_question", null: false + t.text "generated_answer", null: false + t.string "language", default: "en", null: false + t.integer "status", default: 0, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_captain_faq_observations_on_account_id" + t.index ["conversation_id", "faq_suggestion_id"], name: "idx_captain_faq_observations_on_conversation_and_suggestion", unique: true, where: "(faq_suggestion_id IS NOT NULL)" + t.index ["conversation_id"], name: "index_captain_faq_observations_on_conversation_id" + t.index ["faq_suggestion_id"], name: "index_captain_faq_observations_on_faq_suggestion_id" + end + + create_table "captain_faq_suggestions", force: :cascade do |t| + t.string "question", null: false + t.text "answer", null: false + t.vector "embedding", limit: 1536 + t.bigint "assistant_id", null: false + t.bigint "account_id", null: false + t.string "language", default: "en", null: false + t.integer "source_count", default: 0, null: false + t.integer "status", default: 0, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_captain_faq_suggestions_on_account_id" + t.index ["account_id", "assistant_id", "status", "language"], name: "idx_cap_faq_suggestions_on_account_assistant_status_language" + t.index ["assistant_id"], name: "index_captain_faq_suggestions_on_assistant_id" + t.index ["embedding"], name: "vector_idx_captain_faq_suggestions_embedding", opclass: :vector_cosine_ops, using: :ivfflat + end + create_table "captain_inboxes", force: :cascade do |t| t.bigint "captain_assistant_id", null: false t.bigint "inbox_id", null: false diff --git a/enterprise/app/models/captain/assistant.rb b/enterprise/app/models/captain/assistant.rb index b3134e2f2..bf4691e2c 100644 --- a/enterprise/app/models/captain/assistant.rb +++ b/enterprise/app/models/captain/assistant.rb @@ -28,6 +28,7 @@ class Captain::Assistant < ApplicationRecord belongs_to :account has_many :documents, class_name: 'Captain::Document', dependent: :destroy_async has_many :responses, class_name: 'Captain::AssistantResponse', dependent: :destroy_async + has_many :faq_suggestions, class_name: 'Captain::FaqSuggestion', dependent: :destroy_async has_many :captain_inboxes, class_name: 'CaptainInbox', foreign_key: :captain_assistant_id, diff --git a/enterprise/app/models/captain/faq_observation.rb b/enterprise/app/models/captain/faq_observation.rb new file mode 100644 index 000000000..15c5e1284 --- /dev/null +++ b/enterprise/app/models/captain/faq_observation.rb @@ -0,0 +1,42 @@ +# == Schema Information +# +# Table name: captain_faq_observations +# +# id :bigint not null, primary key +# generated_answer :text not null +# generated_question :string not null +# language :string default("en"), not null +# status :integer default("attached"), not null +# created_at :datetime not null +# updated_at :datetime not null +# account_id :bigint not null +# conversation_id :bigint not null +# faq_suggestion_id :bigint +# +class Captain::FaqObservation < ApplicationRecord + self.table_name = 'captain_faq_observations' + + belongs_to :account + belongs_to :conversation, class_name: '::Conversation' + belongs_to :faq_suggestion, class_name: 'Captain::FaqSuggestion', optional: true, inverse_of: :observations + + enum status: { attached: 0, discarded: 1 } + + validates :generated_question, :generated_answer, :language, presence: true + validates :faq_suggestion, presence: true, if: :attached? + validate :faq_suggestion_belongs_to_account + + before_validation :ensure_account + + private + + def ensure_account + self.account = conversation&.account + end + + def faq_suggestion_belongs_to_account + return if faq_suggestion.blank? || faq_suggestion.account_id == account_id + + errors.add(:faq_suggestion, :invalid) + end +end diff --git a/enterprise/app/models/captain/faq_suggestion.rb b/enterprise/app/models/captain/faq_suggestion.rb new file mode 100644 index 000000000..047d5e1fe --- /dev/null +++ b/enterprise/app/models/captain/faq_suggestion.rb @@ -0,0 +1,51 @@ +# == Schema Information +# +# Table name: captain_faq_suggestions +# +# id :bigint not null, primary key +# answer :text not null +# embedding :vector(1536) +# language :string default("en"), not null +# question :string not null +# source_count :integer default(0), not null +# status :integer default("open"), not null +# created_at :datetime not null +# updated_at :datetime not null +# account_id :bigint not null +# assistant_id :bigint not null +# +class Captain::FaqSuggestion < ApplicationRecord + self.table_name = 'captain_faq_suggestions' + + belongs_to :assistant, class_name: 'Captain::Assistant' + belongs_to :account + has_many :observations, + class_name: 'Captain::FaqObservation', + dependent: :delete_all, + inverse_of: :faq_suggestion + has_neighbors :embedding, normalize: true + + enum status: { open: 0, approved: 1, dismissed: 2 } + + validates :question, :answer, :language, presence: true + + before_validation :ensure_account + after_commit :update_embedding, on: [:create, :update] + + scope :ordered, -> { order(source_count: :desc, updated_at: :desc) } + scope :by_language, ->(language) { where(language: language) } + + private + + def ensure_account + self.account = assistant&.account + end + + def update_embedding + return unless open? + return unless saved_change_to_question? || saved_change_to_answer? || embedding.nil? + return if previously_new_record? && embedding.present? + + Captain::Llm::UpdateEmbeddingJob.perform_later(self, "#{question}: #{answer}") + end +end diff --git a/enterprise/app/models/enterprise/concerns/account.rb b/enterprise/app/models/enterprise/concerns/account.rb index 1f5376940..427b1e1af 100644 --- a/enterprise/app/models/enterprise/concerns/account.rb +++ b/enterprise/app/models/enterprise/concerns/account.rb @@ -11,6 +11,8 @@ module Enterprise::Concerns::Account has_many :captain_assistants, dependent: :destroy_async, class_name: 'Captain::Assistant' has_many :captain_assistant_responses, dependent: :destroy_async, class_name: 'Captain::AssistantResponse' + has_many :captain_faq_observations, dependent: :destroy_async, class_name: 'Captain::FaqObservation' + has_many :captain_faq_suggestions, dependent: :destroy_async, class_name: 'Captain::FaqSuggestion' has_many :captain_documents, dependent: :destroy_async, class_name: 'Captain::Document' has_many :captain_custom_tools, dependent: :destroy_async, class_name: 'Captain::CustomTool' has_many :captain_agent_sessions, dependent: :destroy_async, class_name: 'Captain::AgentSession' diff --git a/enterprise/app/models/enterprise/concerns/conversation.rb b/enterprise/app/models/enterprise/concerns/conversation.rb index a075704d1..c247e01e8 100644 --- a/enterprise/app/models/enterprise/concerns/conversation.rb +++ b/enterprise/app/models/enterprise/concerns/conversation.rb @@ -7,6 +7,7 @@ module Enterprise::Concerns::Conversation has_many :sla_events, dependent: :destroy_async has_many :calls, dependent: :destroy_async has_many :captain_responses, class_name: 'Captain::AssistantResponse', dependent: :nullify, as: :documentable + has_many :captain_faq_observations, class_name: 'Captain::FaqObservation', dependent: :delete_all scope :with_sla_applicable_contact, -> { left_joins(:contact).where(contacts: { blocked: [false, nil] }) } before_validation :validate_sla_policy, if: -> { sla_policy_id_changed? } From cf1fdff04b250044862e5863cc33e4429800dac4 Mon Sep 17 00:00:00 2001 From: aakashb95 Date: Tue, 14 Jul 2026 23:13:03 +0530 Subject: [PATCH 2/2] fix(captain): filter FAQ signals with business context --- .../llm/conversation_faq_prompts_service.rb | 1 + .../captain/llm/conversation_faq_service.rb | 29 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb b/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb index 0265c39a3..dd0df4a6c 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb @@ -6,6 +6,7 @@ class Captain::Llm::ConversationFaqPromptsService Only generate an FAQ when the conversation contains durable, reusable knowledge that would help many future customers. ## Source rules + - The input starts with trusted business context. Use it to reject conversations about other businesses or topics, but never use it as the source of an FAQ answer. - The conversation history contains only customer messages and human support agent messages. - Base every FAQ strictly on information stated in the human support agent messages. Do not infer, generalize, or add external knowledge. - A human support agent must state every fact used in the FAQ answer. Customer messages cannot supply missing answer facts. diff --git a/enterprise/app/services/captain/llm/conversation_faq_service.rb b/enterprise/app/services/captain/llm/conversation_faq_service.rb index aae67cd0d..e6bb15891 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_service.rb @@ -25,6 +25,8 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService def conversation_faq_content [ + 'Business Context:', + JSON.pretty_generate(business_context), "Conversation ID: ##{conversation.display_id}", "Channel: #{conversation.inbox.channel.name}", 'Message History:', @@ -74,9 +76,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService def route_candidate(faq) embedding = embedding_service.get_embedding(candidate_text(faq)) - if matching_record(approved_faqs_for_language, faq, embedding) - return discard_observation(faq) - end + return discard_observation(faq) if matching_record(approved_faqs_for_language, faq, embedding) suggestion = matching_record(open_suggestions_for_language, faq, embedding) suggestion ||= assistant.faq_suggestions.create!( @@ -96,10 +96,13 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService def likely_matches(relation, embedding) return [] unless relation.exists? - relation - .nearest_neighbors(:embedding, embedding, distance: 'cosine') - .limit(MATCH_LIMIT) - .select { |record| record.neighbor_distance < DISTANCE_THRESHOLD } + ApplicationRecord.transaction do + ApplicationRecord.connection.execute("SET LOCAL ivfflat.iterative_scan = 'relaxed_order'") + relation + .nearest_neighbors(:embedding, embedding, distance: 'cosine') + .limit(MATCH_LIMIT) + .select { |record| record.neighbor_distance < DISTANCE_THRESHOLD } + end end def same_faq?(candidate, existing_record) @@ -136,7 +139,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService language: faq_language, status: :attached ) - suggestion.update!(source_count: suggestion.observations.attached.count) + suggestion.update!(source_count: suggestion.source_count + 1) observation end end @@ -214,6 +217,16 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService Captain::Llm::ConversationFaqPromptsService.generator(language_name(faq_language)) end + def business_context + { + product_name: assistant.config['product_name'], + assistant_description: assistant.description, + instructions: assistant.config['instructions'], + response_guidelines: assistant.response_guidelines, + guardrails: assistant.guardrails + }.compact + end + def faq_language @faq_language ||= normalize_language(conversation.language.presence || conversation.account.locale.presence || I18n.default_locale.to_s) end