Aakash BakhleandGitHub
d57354c8b5
Tightens the resolved-conversation FAQ generator so it only proposes durable, reusable FAQ candidates supported by human support-agent messages. The implementation now sends a conversation-FAQ-specific transcript to the LLM: customer messages plus real human support-agent messages only, excluding bot, private, activity, and template messages. ## Closes - https://linear.app/chatwoot/issue/CW-7494/tighten-conversation-faq-generation-prompt ## What changed - Added a human-only transcript builder in `ConversationFaqService` instead of using the generic `conversation.to_llm_text` output. - Excluded bot/agent-bot messages before the LLM call, which removes the main bot-line leakage class deterministically. - Preserved native-channel human replies where outgoing messages are stored as `external_echo` without a `User` sender. - Kept a prompt decision gate requiring each FAQ to be backed by a complete public human-agent answer. - Added generic no-FAQ classes for spam, wrong-service conversations, private account/payment/order/certificate/troubleshooting cases, support workflow mechanics, and direct-link/file/quote outputs. - Added a separate `conversation_faq_generation` model route defaulting to `gpt-5.2`, while keeping `document_faq_generation` on its existing `gpt-4.1-mini` default. Conversation FAQ generation passes that feature default ahead of the legacy global `CAPTAIN_OPEN_AI_MODEL` setting unless an account-level override is configured. - Kept the prompt domain-neutral so it can still generate reusable product, service, policy, setup, and process FAQs outside SaaS contexts. ## Sampling notes - Production Langfuse traces showed `llm.captain.conversation_faq` calls using `gpt-4.1` in the sampled account set. - Locally, `Llm::FeatureRouter.resolve(feature: 'conversation_faq_generation')` now resolves to `gpt-5.2`. - Reviewed recent production `llm.captain.conversation_faq` traces across 13+ accounts in compact form. - Replayed 20 full traces across 10 accounts/domains, including education, hosting, retail/auto, APIs, logistics, tax/fiscal workflows, and Chatwoot account 1. - Explicit `gpt-5.2` replay with human-only conversation history returned no FAQ for 15/20 traces. - A comparison replay with `gpt-4.1-mini` returned no FAQ for only 7/20 traces, bringing back several private/order/payment/support-workflow cases. - Remaining non-empty `gpt-5.2` outputs are now mostly borderline/possibly useful human-agent-derived FAQs rather than obvious bot-sourced answers. ## How to test - Resolve conversations where the answer came only from the bot; no pending FAQ should be generated. - Resolve spam, unrelated, wrong-service, or private payment/order/account conversations; no pending FAQ should be generated. - Resolve conversations that require account/order/payment/login/private verification or a human handoff; no pending FAQ should be generated. - Resolve a conversation where a human agent gives a stable, reusable help-center answer; the generated pending FAQ should be general and self-contained.
176 lines
3.8 KiB
YAML
176 lines
3.8 KiB
YAML
providers:
|
|
openai:
|
|
display_name: 'OpenAI'
|
|
anthropic:
|
|
display_name: 'Anthropic'
|
|
gemini:
|
|
display_name: 'Gemini'
|
|
|
|
models:
|
|
gpt-4.1:
|
|
provider: openai
|
|
display_name: 'GPT-4.1'
|
|
credit_multiplier: 3
|
|
gpt-4.1-mini:
|
|
provider: openai
|
|
display_name: 'GPT-4.1 Mini'
|
|
credit_multiplier: 1
|
|
gpt-4.1-nano:
|
|
provider: openai
|
|
display_name: 'GPT-4.1 Nano'
|
|
credit_multiplier: 1
|
|
gpt-5.1:
|
|
provider: openai
|
|
display_name: 'GPT-5.1'
|
|
credit_multiplier: 2
|
|
gpt-5-mini:
|
|
provider: openai
|
|
display_name: 'GPT-5 Mini'
|
|
credit_multiplier: 1
|
|
gpt-5-nano:
|
|
provider: openai
|
|
display_name: 'GPT-5 Nano'
|
|
credit_multiplier: 1
|
|
gpt-5.2:
|
|
provider: openai
|
|
display_name: 'GPT-5.2'
|
|
credit_multiplier: 3
|
|
claude-haiku-4.5:
|
|
provider: anthropic
|
|
display_name: 'Claude Haiku 4.5'
|
|
coming_soon: true
|
|
credit_multiplier: 2
|
|
claude-sonnet-4.5:
|
|
provider: anthropic
|
|
display_name: 'Claude Sonnet 4.5'
|
|
coming_soon: true
|
|
credit_multiplier: 3
|
|
gemini-3-flash:
|
|
provider: gemini
|
|
display_name: 'Gemini 3 Flash'
|
|
coming_soon: true
|
|
credit_multiplier: 1
|
|
gemini-3-pro:
|
|
provider: gemini
|
|
display_name: 'Gemini 3 Pro'
|
|
coming_soon: true
|
|
credit_multiplier: 3
|
|
whisper-1:
|
|
provider: openai
|
|
display_name: 'Whisper'
|
|
credit_multiplier: 1
|
|
gpt-4o-mini-transcribe:
|
|
provider: openai
|
|
display_name: 'GPT-4o Mini Transcribe'
|
|
credit_multiplier: 1
|
|
text-embedding-3-small:
|
|
provider: openai
|
|
display_name: 'Text Embedding 3 Small'
|
|
credit_multiplier: 1
|
|
|
|
features:
|
|
editor:
|
|
models:
|
|
[
|
|
gpt-4.1-mini,
|
|
gpt-4.1-nano,
|
|
gpt-5-mini,
|
|
gpt-4.1,
|
|
gpt-5.1,
|
|
gpt-5.2,
|
|
claude-haiku-4.5,
|
|
gemini-3-flash,
|
|
gemini-3-pro,
|
|
]
|
|
default: gpt-4.1-mini
|
|
assistant:
|
|
models:
|
|
[
|
|
gpt-4.1-mini,
|
|
gpt-5-mini,
|
|
gpt-4.1,
|
|
gpt-5.1,
|
|
gpt-5.2,
|
|
claude-haiku-4.5,
|
|
claude-sonnet-4.5,
|
|
gemini-3-flash,
|
|
gemini-3-pro,
|
|
]
|
|
default: gpt-4.1
|
|
copilot:
|
|
models:
|
|
[
|
|
gpt-4.1-mini,
|
|
gpt-5-mini,
|
|
gpt-4.1,
|
|
gpt-5.1,
|
|
gpt-5.2,
|
|
claude-haiku-4.5,
|
|
claude-sonnet-4.5,
|
|
gemini-3-flash,
|
|
gemini-3-pro,
|
|
]
|
|
default: gpt-4.1
|
|
label_suggestion:
|
|
models:
|
|
[gpt-4.1-nano, gpt-4.1-mini, gpt-5-mini, gemini-3-flash, claude-haiku-4.5]
|
|
default: gpt-4.1-mini
|
|
document_faq_generation:
|
|
models:
|
|
[
|
|
gpt-4.1-mini,
|
|
gpt-5-mini,
|
|
gpt-4.1,
|
|
gpt-5.1,
|
|
gpt-5.2,
|
|
claude-haiku-4.5,
|
|
claude-sonnet-4.5,
|
|
gemini-3-flash,
|
|
gemini-3-pro,
|
|
]
|
|
default: gpt-4.1-mini
|
|
conversation_faq_generation:
|
|
models:
|
|
[
|
|
gpt-4.1-mini,
|
|
gpt-5-mini,
|
|
gpt-4.1,
|
|
gpt-5.1,
|
|
gpt-5.2,
|
|
claude-haiku-4.5,
|
|
claude-sonnet-4.5,
|
|
gemini-3-flash,
|
|
gemini-3-pro,
|
|
]
|
|
default: gpt-5.2
|
|
pdf_faq_generation:
|
|
models: [gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, gpt-5.2]
|
|
default: gpt-4.1-mini
|
|
help_center_article_generation:
|
|
models:
|
|
[
|
|
gpt-4.1-mini,
|
|
gpt-5-mini,
|
|
gpt-4.1,
|
|
gpt-5.1,
|
|
gpt-5.2,
|
|
claude-haiku-4.5,
|
|
claude-sonnet-4.5,
|
|
gemini-3-flash,
|
|
gemini-3-pro,
|
|
]
|
|
default: gpt-5.2
|
|
onboarding_content_generation:
|
|
models:
|
|
[gpt-4.1, gpt-4.1-mini, gpt-5-mini, gpt-5.1, gpt-5.2]
|
|
default: gpt-4.1
|
|
help_center_query_translation:
|
|
models: [gpt-4.1-nano, gpt-4.1-mini, gpt-5-mini]
|
|
default: gpt-4.1-nano
|
|
audio_transcription:
|
|
models: [gpt-4o-mini-transcribe, whisper-1]
|
|
default: gpt-4o-mini-transcribe
|
|
help_center_search:
|
|
models: [text-embedding-3-small]
|
|
default: text-embedding-3-small
|