akadmin/chatwoot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: Add a review step for FAQs generated from conversations before using it (#10693)

Browse Source 
This PR introduces a review step for generated FAQs, allowing a human to
validate and approve them before use in customer interactions. While
hallucinations are minimal, this step ensures accurate and reliable FAQs
for Captain to use during LLM calls when responding to customers.

- Added a status field for the FAQ
- Allow the filter on the UI.
<img width="1072" alt="Screenshot 2025-01-15 at 6 39 26 PM"
src="https://github.com/user-attachments/assets/81dfc038-31e9-40e6-8a09-586ebc4e8384"
/>
This commit is contained in:
Pranav
2025-01-16 09:54:34 +05:30
committed by GitHub
parent e3b5b30666
commit 6096932f76
15 changed files with 216 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/javascript/dashboard/api/captain/response.js
+2 -1
View File
@@ -6,13 +6,14 @@ class CaptainResponses extends ApiClient {
super('captain/assistant_responses', { accountScoped: true });
}
get({ page = 1, searchKey, assistantId, documentId } = {}) {
get({ page = 1, searchKey, assistantId, documentId, status } = {}) {
return axios.get(this.url, {
params: {
page,
searchKey,
assistant_id: assistantId,
document_id: documentId,
status,
},
});
}
app/javascript/dashboard/components-next/captain/assistant/ResponseCard.story.vue
+2
View File
@@ -9,6 +9,7 @@ const responses = [
created_at: 1736283330,
id: 87,
question: 'Why is my Messenger in Chatwoot deactivated?',
status: 'pending',
assistant: {
account_id: 1,
config: {
@@ -148,6 +149,7 @@ const responses = [
:id="response.id"
:question="response.question"
:answer="response.answer"
:status="response.status"
:assistant="response.assistant"
:created-at="response.created_at"
/>
app/javascript/dashboard/components-next/captain/assistant/ResponseCard.vue
+29
View File
@@ -25,6 +25,10 @@ const props = defineProps({
type: Boolean,
default: false,
},
status: {
type: String,
default: 'approved',
},
assistant: {
type: Object,
default: () => ({}),
@@ -45,7 +49,22 @@ const { t } = useI18n();
const [showActionsDropdown, toggleDropdown] = useToggle();
const statusAction = computed(() => {
if (props.status === 'pending') {
return [
{
label: t('CAPTAIN.RESPONSES.OPTIONS.APPROVE'),
value: 'approve',
action: 'approve',
icon: 'i-lucide-circle-check-big',
},
];
}
return [];
});
const menuItems = computed(() => [
...statusAction.value,
{
label: t('CAPTAIN.RESPONSES.OPTIONS.EDIT_RESPONSE'),
value: 'edit',
@@ -107,6 +126,16 @@ const handleAssistantAction = ({ action, value }) => {
<i class="i-woot-captain" />
{{ assistant?.name || '' }}
</span>
<div
v-if="status !== 'approved'"
class="shrink-0 text-sm text-n-slate-11 line-clamp-1 inline-flex items-center gap-1 ml-3"
>
<i
class="i-ph-stack text-base"
:title="t('CAPTAIN.RESPONSES.STATUS.TITLE')"
/>
{{ t(`CAPTAIN.RESPONSES.STATUS.${status.toUpperCase()}`) }}
</div>
<div
class="shrink-0 text-sm text-n-slate-11 line-clamp-1 inline-flex items-center gap-1 ml-3"
>
app/javascript/dashboard/components-next/captain/pageComponents/document/RelatedResponses.vue
+1
View File
@@ -57,6 +57,7 @@ defineExpose({ dialogRef });
:id="response.id"
:key="response.id"
:question="response.question"
:status="response.status"
:answer="response.answer"
:assistant="response.assistant"
:created-at="response.created_at"
app/javascript/dashboard/i18n/locale/en/integrations.json
+17 -5
View File
@@ -347,7 +347,7 @@
},
"FEATURES": {
"TITLE": "Features",
"ALLOW_CONVERSATION_FAQS": "Generate responses from resolved conversations",
"ALLOW_CONVERSATION_FAQS": "Generate FAQs from resolved conversations",
"ALLOW_MEMORIES": "Capture key details as memories from customer interactions."
}
},
@@ -366,8 +366,8 @@
"HEADER": "Documents",
"ADD_NEW": "Create a new document",
"RELATED_RESPONSES": {
"TITLE": "Related Responses",
"DESCRIPTION": "These responses are generated directly from the document."
"TITLE": "Related FAQs",
"DESCRIPTION": "These FAQs are generated directly from the document."
},
"FORM_DESCRIPTION": "Enter the URL of the document to add it as a knowledge source and choose the assistant to associate it with.",
"CREATE": {
@@ -410,6 +410,17 @@
"SUCCESS_MESSAGE": "FAQ deleted successfully",
"ERROR_MESSAGE": "There was an error deleting the FAQ, please try again."
},
"FILTER" :{
"ASSISTANT": "Assistant: {selected}",
"STATUS": "Status: {selected}",
"ALL_ASSISTANTS": "All"
},
"STATUS": {
"TITLE": "Status",
"PENDING": "Pending",
"APPROVED": "Approved",
"ALL": "All"
},
"FORM_DESCRIPTION": "Add a question and its corresponding answer to the knowledge base and select the assistant it should be associated with.",
"CREATE": {
"TITLE": "Add an FAQ",
@@ -437,10 +448,11 @@
"EDIT": {
"TITLE": "Update the FAQ",
"SUCCESS_MESSAGE": "The FAQ has been successfully updated",
"ERROR_MESSAGE": "There was an error updating the FAQ, please try again."
"ERROR_MESSAGE": "There was an error updating the FAQ, please try again",
"APPROVE_SUCCESS_MESSAGE": "The FAQ was marked as approved"
},
"OPTIONS": {
"APPROVE": "Mark as approved",
"EDIT_RESPONSE": "Edit FAQ",
"DELETE_RESPONSE": "Delete FAQ"
}
app/javascript/dashboard/routes/dashboard/captain/responses/Index.vue
+133 -3
View File
@@ -1,6 +1,11 @@
<script setup>
import { computed, onMounted, ref, nextTick } from 'vue';
import { useMapGetter, useStore } from 'dashboard/composables/store';
import { useAlert } from 'dashboard/composables';
import { useI18n } from 'vue-i18n';
import { OnClickOutside } from '@vueuse/components';
import Button from 'dashboard/components-next/button/Button.vue';
import DropdownMenu from 'dashboard/components-next/dropdown-menu/DropdownMenu.vue';
import DeleteDialog from 'dashboard/components-next/captain/pageComponents/DeleteDialog.vue';
import PageLayout from 'dashboard/components-next/captain/PageLayout.vue';
@@ -16,13 +21,75 @@ const isFetching = computed(() => uiFlags.value.fetchingList);
const selectedResponse = ref(null);
const deleteDialog = ref(null);
const selectedStatus = ref('all');
const selectedAssistant = ref('all');
const dialogType = ref('');
const { t } = useI18n();
const createDialog = ref(null);
const isStatusFilterOpen = ref(false);
const isAssistantFilterOpen = ref(false);
const statusOptions = computed(() =>
['all', 'pending', 'approved'].map(key => ({
label: t(`CAPTAIN.RESPONSES.STATUS.${key.toUpperCase()}`),
value: key,
action: 'filter',
}))
);
const selectedStatusLabel = computed(() => {
const status = statusOptions.value.find(
option => option.value === selectedStatus.value
);
return t('CAPTAIN.RESPONSES.FILTER.STATUS', {
selected: status ? status.label : '',
});
});
const assistants = useMapGetter('captainAssistants/getRecords');
const assistantOptions = computed(() => [
{
label: t(`CAPTAIN.RESPONSES.FILTER.ALL_ASSISTANTS`),
value: 'all',
action: 'filter',
},
...assistants.value.map(assistant => ({
value: assistant.id,
label: assistant.name,
action: 'filter',
})),
]);
const selectedAssistantLabel = computed(() => {
const assistant = assistantOptions.value.find(
option => option.value === selectedAssistant.value
);
return t('CAPTAIN.RESPONSES.FILTER.ASSISTANT', {
selected: assistant ? assistant.label : '',
});
});
const handleDelete = () => {
deleteDialog.value.dialogRef.open();
};
const createDialog = ref(null);
const handleAccept = async () => {
try {
await store.dispatch('captainResponses/update', {
id: selectedResponse.value.id,
status: 'approved',
});
useAlert(t(`CAPTAIN.RESPONSES.EDIT.APPROVE_SUCCESS_MESSAGE`));
} catch (error) {
const errorMessage =
error?.message || t(`CAPTAIN.RESPONSES.EDIT.ERROR_MESSAGE`);
useAlert(errorMessage);
} finally {
selectedResponse.value = null;
}
};
const handleCreate = () => {
dialogType.value = 'create';
@@ -43,6 +110,9 @@ const handleAction = ({ action, id }) => {
if (action === 'edit') {
handleEdit();
}
if (action === 'approve') {
handleAccept();
}
});
};
@@ -52,11 +122,30 @@ const handleCreateClose = () => {
};
const fetchResponses = (page = 1) => {
store.dispatch('captainResponses/get', { page });
const filterParams = {};
if (selectedStatus.value !== 'all') {
filterParams.status = selectedStatus.value;
}
if (selectedAssistant.value !== 'all') {
filterParams.assistantId = selectedAssistant.value;
}
store.dispatch('captainResponses/get', { page, ...filterParams });
};
const onPageChange = page => fetchResponses(page);
const handleStatusFilterChange = ({ value }) => {
selectedStatus.value = value;
isStatusFilterOpen.value = false;
fetchResponses();
};
const handleAssistantFilterChange = ({ value }) => {
selectedAssistant.value = value;
isAssistantFilterOpen.value = false;
fetchResponses();
};
onMounted(() => {
store.dispatch('captainAssistants/get');
fetchResponses();
@@ -73,12 +162,52 @@ onMounted(() => {
@update:current-page="onPageChange"
@click="handleCreate"
>
<div v-if="!isFetching" class="mb-4 -mt-3 flex gap-3">
<OnClickOutside @trigger="isStatusFilterOpen = false">
<Button
:label="selectedStatusLabel"
icon="i-lucide-chevron-down"
size="sm"
color="slate"
trailing-icon
class="max-w-48"
@click="isStatusFilterOpen = !isStatusFilterOpen"
/>
<DropdownMenu
v-if="isStatusFilterOpen"
:menu-items="statusOptions"
class="mt-2"
@action="handleStatusFilterChange"
/>
</OnClickOutside>
<OnClickOutside @trigger="isAssistantFilterOpen = false">
<Button
:label="selectedAssistantLabel"
icon="i-lucide-chevron-down"
size="sm"
color="slate"
trailing-icon
class="max-w-48"
@click="isAssistantFilterOpen = !isAssistantFilterOpen"
/>
<DropdownMenu
v-if="isAssistantFilterOpen"
:menu-items="assistantOptions"
class="mt-2"
@action="handleAssistantFilterChange"
/>
</OnClickOutside>
</div>
<div
v-if="isFetching"
class="flex items-center justify-center py-10 text-n-slate-11"
>
<Spinner />
</div>
<div v-else-if="responses.length" class="flex flex-col gap-4">
<ResponseCard
v-for="response in responses"
@@ -87,6 +216,7 @@ onMounted(() => {
:question="response.question"
:answer="response.answer"
:assistant="response.assistant"
:status="response.status"
:created-at="response.created_at"
:updated-at="response.updated_at"
@action="handleAction"
db/migrate/20250116000103_add_status_to_captain_assistant_responses.rb
+6
View File
@@ -0,0 +1,6 @@
class AddStatusToCaptainAssistantResponses < ActiveRecord::Migration[7.0]
def change
add_column :captain_assistant_responses, :status, :integer, default: 1, null: false
add_index :captain_assistant_responses, :status
end
end
db/schema.rb
+3 -1
View File
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.0].define(version: 2025_01_08_211541) do
ActiveRecord::Schema[7.0].define(version: 2025_01_16_000103) do
# These extensions should be enabled to support this database
enable_extension "pg_stat_statements"
enable_extension "pg_trgm"
@@ -254,10 +254,12 @@ ActiveRecord::Schema[7.0].define(version: 2025_01_08_211541) do
t.bigint "account_id", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.integer "status", default: 1, null: false
t.index ["account_id"], name: "index_captain_assistant_responses_on_account_id"
t.index ["assistant_id"], name: "index_captain_assistant_responses_on_assistant_id"
t.index ["document_id"], name: "index_captain_assistant_responses_on_document_id"
t.index ["embedding"], name: "vector_idx_knowledge_entries_embedding", using: :ivfflat
t.index ["status"], name: "index_captain_assistant_responses_on_status"
end
create_table "captain_assistants", force: :cascade do |t|
enterprise/app/controllers/api/v1/accounts/captain/assistant_responses_controller.rb
+4 -2
View File
@@ -13,6 +13,7 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
base_query = @responses
base_query = base_query.where(assistant_id: permitted_params[:assistant_id]) if permitted_params[:assistant_id].present?
base_query = base_query.where(document_id: permitted_params[:document_id]) if permitted_params[:document_id].present?
base_query = base_query.where(status: permitted_params[:status]) if permitted_params[:status].present?
@responses_count = base_query.count
@@ -54,7 +55,7 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
end
def permitted_params
params.permit(:id, :assistant_id, :page, :document_id, :account_id)
params.permit(:id, :assistant_id, :page, :document_id, :account_id, :status)
end
def response_params
@@ -62,7 +63,8 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
:question,
:answer,
:document_id,
:assistant_id
:assistant_id,
:status
)
end
end
enterprise/app/models/captain/assistant_response.rb
+9
View File
@@ -6,6 +6,7 @@
# answer :text not null
# embedding :vector(1536)
# question :string not null
# status :integer default("approved"), not null
# created_at :datetime not null
# updated_at :datetime not null
# account_id :bigint not null
@@ -17,6 +18,7 @@
# index_captain_assistant_responses_on_account_id (account_id)
# index_captain_assistant_responses_on_assistant_id (assistant_id)
# index_captain_assistant_responses_on_document_id (document_id)
# index_captain_assistant_responses_on_status (status)
# vector_idx_knowledge_entries_embedding (embedding) USING ivfflat
#
class Captain::AssistantResponse < ApplicationRecord
@@ -31,6 +33,7 @@ class Captain::AssistantResponse < ApplicationRecord
validates :answer, presence: true
before_validation :ensure_account
before_validation :ensure_status
after_commit :update_response_embedding
scope :ordered, -> { order(created_at: :desc) }
@@ -38,6 +41,8 @@ class Captain::AssistantResponse < ApplicationRecord
scope :by_assistant, ->(assistant_id) { where(assistant_id: assistant_id) }
scope :with_document, ->(document_id) { where(document_id: document_id) }
enum status: { pending: 0, approved: 1 }
def self.search(query)
embedding = Captain::Llm::EmbeddingService.new.get_embedding(query)
nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(5)
@@ -45,6 +50,10 @@ class Captain::AssistantResponse < ApplicationRecord
private
def ensure_status
self.status ||= :approved
end
def ensure_account
self.account = assistant&.account
end
enterprise/app/services/captain/copilot/chat_service.rb
+1
View File
@@ -66,6 +66,7 @@ class Captain::Copilot::ChatService
assistant = Captain::Assistant.find(memory[:assistant_id])
assistant
.responses
.approved
.search(inputs['search_query'])
.map do |response|
"\n\nQuestion: #{response[:question]}\nAnswer: #{response[:answer]}"
enterprise/app/services/captain/llm/assistant_chat_service.rb
+1
View File
@@ -84,6 +84,7 @@ class Captain::Llm::AssistantChatService < Captain::Llm::BaseOpenAiService
def fetch_documentation(query)
@assistant
.responses
.approved
.search(query)
.map { |response| format_response(response) }.join
end
enterprise/app/services/captain/llm/conversation_faq_service.rb
+1 -1
View File
@@ -50,7 +50,7 @@ class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService
def save_new_faqs(faqs)
faqs.map do |faq|
@assistant.responses.create!(question: faq['question'], answer: faq['answer'])
@assistant.responses.create!(question: faq['question'], answer: faq['answer'], status: 'pending')
end
end
enterprise/app/views/api/v1/models/captain/_assistant_response.json.jbuilder
+1
View File
@@ -14,3 +14,4 @@ end
json.id resource.id
json.question resource.question
json.updated_at resource.updated_at.to_i
json.status resource.status
spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb
+6 -3
View File
@@ -48,9 +48,12 @@ RSpec.describe Captain::Llm::ConversationFaqService do
it 'saves the correct FAQ content' do
service.generate_and_deduplicate
expect(captain_assistant.responses.pluck(:question,
:answer)).to contain_exactly(['What is the purpose?', 'To help users.'],
['How does it work?', 'Through AI.'])
expect(
captain_assistant.responses.pluck(:question, :answer, :status)
).to contain_exactly(
['What is the purpose?', 'To help users.', 'pending'],
['How does it work?', 'Through AI.', 'pending']
)
end
end