feat: add be changes for captain pdf support for faq generation

This commit is contained in:
Tanmay Deep Sharma
2025-08-06 12:22:20 +05:30
parent 51b9fd8eca
commit 7317b95441
15 changed files with 555 additions and 14 deletions
+1
View File
@@ -16,6 +16,7 @@ Metrics/ClassLength:
Exclude:
- 'app/models/message.rb'
- 'app/models/conversation.rb'
- 'enterprise/app/services/captain/llm/system_prompts_service.rb'
Metrics/MethodLength:
Max: 19
+5 -1
View File
@@ -67,7 +67,11 @@ Rails.application.routes.draw do
resources :copilot_threads, only: [:index, :create] do
resources :copilot_messages, only: [:index, :create]
end
resources :documents, only: [:index, :show, :create, :destroy]
resources :documents, only: [:index, :show, :create, :destroy] do
collection do
post :upload_pdf
end
end
end
resources :agent_bots, only: [:index, :create, :show, :update, :destroy] do
delete :avatar, on: :member
@@ -0,0 +1,8 @@
class AddPdfSupportToCaptainDocuments < ActiveRecord::Migration[7.1]
def change
add_column :captain_documents, :content_type, :string
add_column :captain_documents, :file_size, :bigint
add_index :captain_documents, :content_type
end
end
@@ -0,0 +1,5 @@
class AddMetadataToCaptainDocuments < ActiveRecord::Migration[7.1]
def change
add_column :captain_documents, :metadata, :jsonb, default: {}
end
end
+5 -1
View File
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.1].define(version: 2025_07_22_152516) do
ActiveRecord::Schema[7.1].define(version: 2025_08_05_082345) do
# These extensions should be enabled to support this database
enable_extension "pg_stat_statements"
enable_extension "pg_trgm"
@@ -292,9 +292,13 @@ ActiveRecord::Schema[7.1].define(version: 2025_07_22_152516) do
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.integer "status", default: 0, null: false
t.string "content_type"
t.bigint "file_size"
t.jsonb "metadata", default: {}
t.index ["account_id"], name: "index_captain_documents_on_account_id"
t.index ["assistant_id", "external_link"], name: "index_captain_documents_on_assistant_id_and_external_link", unique: true
t.index ["assistant_id"], name: "index_captain_documents_on_assistant_id"
t.index ["content_type"], name: "index_captain_documents_on_content_type"
t.index ["status"], name: "index_captain_documents_on_status"
end
@@ -3,9 +3,9 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
before_action -> { check_authorization(Captain::Assistant) }
before_action :set_current_page, only: [:index]
before_action :set_documents, except: [:create]
before_action :set_documents, except: [:create, :upload_pdf]
before_action :set_document, only: [:show, :destroy]
before_action :set_assistant, only: [:create]
before_action :set_assistant, only: [:create, :upload_pdf]
RESULTS_PER_PAGE = 25
def index
@@ -22,9 +22,33 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
return render_could_not_create_error('Missing Assistant') if @assistant.nil?
@document = @assistant.documents.build(document_params)
# Handle PDF file upload if present
@document.pdf_file.attach(document_params[:pdf_file]) if document_params[:pdf_file].present?
@document.save!
rescue Captain::Document::LimitExceededError => e
render_could_not_create_error(e.message)
rescue StandardError => e
Rails.logger.error "Document creation error: #{e.message}"
render_could_not_create_error('Failed to create document')
end
def upload_pdf
pdf_file = params[:pdf_file]
return render_could_not_create_error('PDF file is required') if pdf_file.blank?
return render_could_not_create_error('Invalid file type') unless valid_pdf?(pdf_file)
return render_could_not_create_error('File too large (max 512MB)') if pdf_file.size > 512.megabytes
return render_could_not_create_error('Missing Assistant') if @assistant.nil?
create_pdf_document(pdf_file)
render :show
rescue Captain::Document::LimitExceededError => e
render_could_not_create_error(e.message)
rescue StandardError => e
Rails.logger.error "PDF upload error: #{e.message}"
render_could_not_create_error('Failed to upload PDF')
end
def destroy
@@ -43,7 +67,8 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
end
def set_assistant
@assistant = Current.account.captain_assistants.find_by(id: document_params[:assistant_id])
assistant_id = document_params[:assistant_id] || params[:assistant_id]
@assistant = Current.account.captain_assistants.find_by(id: assistant_id)
end
def set_current_page
@@ -55,6 +80,19 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
end
def document_params
params.require(:document).permit(:name, :external_link, :assistant_id)
params.require(:document).permit(:name, :external_link, :assistant_id, :pdf_file)
end
def valid_pdf?(file)
file.content_type == 'application/pdf'
end
def create_pdf_document(pdf_file)
@document = @assistant.documents.build(
name: pdf_file.original_filename,
account: Current.account
)
@document.pdf_file.attach(pdf_file)
@document.save!
end
end
@@ -2,7 +2,9 @@ class Captain::Documents::CrawlJob < ApplicationJob
queue_as :low
def perform(document)
if InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present?
if document.pdf_document?
perform_pdf_processing(document)
elsif InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present?
perform_firecrawl_crawl(document)
else
perform_simple_crawl(document)
@@ -13,6 +15,19 @@ class Captain::Documents::CrawlJob < ApplicationJob
include Captain::FirecrawlHelper
def perform_pdf_processing(document)
pdf_processor = Captain::Llm::PdfProcessingService.new(document)
pdf_processor.process
# Mark document as available - content is not needed for paginated processing
document.update!(status: :available)
Rails.logger.info "Successfully processed PDF document #{document.id}"
rescue StandardError => e
Rails.logger.error "Failed to process PDF document #{document.id}: #{e.message}"
document.update!(status: :available)
end
def perform_simple_crawl(document)
page_links = Captain::Tools::SimplePageCrawlService.new(document.external_link).page_links
@@ -1,17 +1,69 @@
class Captain::Documents::ResponseBuilderJob < ApplicationJob
queue_as :low
def perform(document)
def perform(document, options = {})
reset_previous_responses(document)
faqs = Captain::Llm::FaqGeneratorService.new(document.content).generate
faqs.each do |faq|
create_response(faq, document)
faqs = generate_faqs(document, options)
create_responses_from_faqs(faqs, document)
Rails.logger.info "FAQ generation complete. Total FAQs created: #{faqs.size}"
end
def generate_faqs(document, options)
if should_use_pagination?(document)
generate_paginated_faqs(document, options)
else
generate_standard_faqs(document)
end
end
def generate_paginated_faqs(document, options)
Rails.logger.info "Using paginated FAQ generation for document #{document.id}"
service = build_paginated_service(document, options)
faqs = service.generate
store_paginated_metadata(document, service)
faqs
end
def generate_standard_faqs(document)
Rails.logger.info "Using standard FAQ generation for document #{document.id}"
Captain::Llm::FaqGeneratorService.new(document.content).generate
end
def build_paginated_service(document, options)
Captain::Llm::PaginatedFaqGeneratorService.new(
document,
pages_per_chunk: options[:pages_per_chunk],
max_pages: options[:max_pages]
)
end
def store_paginated_metadata(document, service)
document.update!(
metadata: document.metadata.merge(
'faq_generation' => {
'method' => 'paginated',
'pages_processed' => service.total_pages_processed,
'iterations' => service.iterations_completed,
'timestamp' => Time.current.iso8601
}
)
)
end
def create_responses_from_faqs(faqs, document)
faqs.each { |faq| create_response(faq, document) }
end
private
def should_use_pagination?(document)
# Auto-detect when to use pagination
# For now, use pagination for PDFs with OpenAI file ID
document.pdf_document? && document.openai_file_id.present?
end
def reset_previous_responses(response_document)
response_document.responses.destroy_all
end
@@ -2,6 +2,9 @@ class Captain::Llm::UpdateEmbeddingJob < ApplicationJob
queue_as :low
def perform(record, content)
# Early return if record was deleted before job execution
return unless record&.persisted?
embedding = Captain::Llm::EmbeddingService.new.get_embedding(content)
record.update!(embedding: embedding)
end
+57 -3
View File
@@ -4,7 +4,10 @@
#
# id :bigint not null, primary key
# content :text
# content_type :string
# external_link :string not null
# file_size :bigint
# metadata :jsonb
# name :string
# status :integer default("in_progress"), not null
# created_at :datetime not null
@@ -26,11 +29,15 @@ class Captain::Document < ApplicationRecord
belongs_to :assistant, class_name: 'Captain::Assistant'
has_many :responses, class_name: 'Captain::AssistantResponse', dependent: :destroy, as: :documentable
belongs_to :account
has_one_attached :pdf_file
validates :external_link, presence: true
validates :external_link, uniqueness: { scope: :assistant_id }
validates :external_link, presence: true, unless: -> { pdf_file.attached? }
validates :external_link, uniqueness: { scope: :assistant_id }, allow_blank: true
validates :content, length: { maximum: 200_000 }
validates :pdf_file, presence: true, if: :pdf_document?
validate :validate_pdf_format, if: :pdf_document?
before_validation :ensure_account_id
before_validation :set_external_link_for_pdf
enum status: {
in_progress: 0,
@@ -41,12 +48,34 @@ class Captain::Document < ApplicationRecord
after_create_commit :enqueue_crawl_job
after_create_commit :update_document_usage
after_destroy :update_document_usage
after_commit :enqueue_response_builder_job
after_commit :enqueue_response_builder_job, on: :update, if: :should_enqueue_response_builder?
scope :ordered, -> { order(created_at: :desc) }
scope :for_account, ->(account_id) { where(account_id: account_id) }
scope :for_assistant, ->(assistant_id) { where(assistant_id: assistant_id) }
def pdf_document?
(external_link&.ends_with?('.pdf')) || (pdf_file.attached? && pdf_file.content_type == 'application/pdf')
end
def openai_file_id
metadata['openai_file_id']
end
def store_openai_file_id(file_id)
update!(metadata: metadata.merge('openai_file_id' => file_id))
end
def display_url
return external_link if external_link.present? && !external_link.start_with?('PDF:')
if pdf_file.attached?
Rails.application.routes.url_helpers.rails_blob_url(pdf_file, only_path: false)
else
external_link
end
end
private
def enqueue_crawl_job
@@ -61,6 +90,12 @@ class Captain::Document < ApplicationRecord
Captain::Documents::ResponseBuilderJob.perform_later(self)
end
def should_enqueue_response_builder?
# Only enqueue when status changes to available
# Avoid re-enqueueing when metadata is updated by the job itself
saved_change_to_status? && status == 'available'
end
def update_document_usage
account.update_document_usage
end
@@ -73,4 +108,23 @@ class Captain::Document < ApplicationRecord
limits = account.usage_limits[:captain][:documents]
raise LimitExceededError, 'Document limit exceeded' unless limits[:current_available].positive?
end
def validate_pdf_format
return unless pdf_file.attached?
errors.add(:pdf_file, 'must be a PDF file') unless pdf_file.content_type == 'application/pdf'
return unless pdf_file.byte_size > 512.megabytes
errors.add(:pdf_file, 'must be less than 512MB')
end
def set_external_link_for_pdf
return unless pdf_file.attached? && external_link.blank?
# Set a unique external_link for PDF files
# Format: PDF: filename_timestamp (without extension)
timestamp = Time.current.strftime('%Y%m%d%H%M%S')
self.external_link = "PDF: #{pdf_file.filename.base}_#{timestamp}"
end
end
@@ -0,0 +1,206 @@
class Captain::Llm::PaginatedFaqGeneratorService < Llm::BaseOpenAiService
# Default pages per chunk - easily configurable
DEFAULT_PAGES_PER_CHUNK = 10
MAX_ITERATIONS = 20 # Safety limit to prevent infinite loops
attr_reader :total_pages_processed, :iterations_completed
def initialize(document, options = {})
super()
@document = document
@pages_per_chunk = options[:pages_per_chunk] || DEFAULT_PAGES_PER_CHUNK
@max_pages = options[:max_pages] # Optional limit from UI
@total_pages_processed = 0
@iterations_completed = 0
end
def generate
raise 'Document must have openai_file_id for paginated processing' if @document&.openai_file_id.blank?
generate_paginated_faqs
end
# Method to check if we should continue processing
def should_continue_processing?(last_chunk_faqs)
# Stop if we've hit the maximum iterations
return false if @iterations_completed >= MAX_ITERATIONS
# Stop if we've processed the maximum pages specified
return false if @max_pages && @total_pages_processed >= @max_pages
# Stop if the last chunk returned no FAQs (likely no more content)
return false if last_chunk_faqs.empty?
# Continue processing
true
end
private
def generate_standard_faqs
response = @client.chat(parameters: standard_chat_parameters)
parse_response(response)
rescue OpenAI::Error => e
Rails.logger.error "OpenAI API Error: #{e.message}"
[]
end
def generate_paginated_faqs
Rails.logger.info "Starting paginated FAQ generation (#{@pages_per_chunk} pages per chunk)"
all_faqs = []
current_page = 1
loop do
end_page = calculate_end_page(current_page)
chunk_faqs = process_chunk_and_update_state(current_page, end_page, all_faqs)
unless should_continue_processing?(chunk_faqs)
Rails.logger.info "Stopping processing. Reason: #{determine_stop_reason(chunk_faqs)}"
break
end
current_page = end_page + 1
end
Rails.logger.info "Paginated generation complete. Total FAQs: #{all_faqs.size}, Pages processed: #{@total_pages_processed}"
deduplicate_faqs(all_faqs)
end
def calculate_end_page(current_page)
end_page = current_page + @pages_per_chunk - 1
@max_pages && end_page > @max_pages ? @max_pages : end_page
end
def process_chunk_and_update_state(current_page, end_page, all_faqs)
Rails.logger.info "Processing pages #{current_page}-#{end_page} (iteration #{@iterations_completed + 1})"
chunk_result = process_page_chunk(current_page, end_page)
chunk_faqs = chunk_result[:faqs]
all_faqs.concat(chunk_faqs)
@total_pages_processed = end_page
@iterations_completed += 1
Rails.logger.info "Chunk generated #{chunk_faqs.size} FAQs. Total so far: #{all_faqs.size}"
chunk_faqs
end
def process_page_chunk(start_page, end_page)
response = @client.chat(parameters: build_chunk_parameters(start_page, end_page))
result = parse_chunk_response(response)
{ faqs: result['faqs'] || [], has_content: result['has_content'] != false }
rescue OpenAI::Error => e
Rails.logger.error "Error processing pages #{start_page}-#{end_page}: #{e.message}"
{ faqs: [], has_content: false }
end
def build_chunk_parameters(start_page, end_page)
{
model: @model,
response_format: { type: 'json_object' },
messages: [
{
role: 'system',
content: page_chunk_prompt(start_page, end_page)
},
{
role: 'user',
content: build_user_content
}
]
}
end
def build_user_content
[
{
type: 'file',
file: { file_id: @document.openai_file_id }
},
{
type: 'text',
text: 'Process this document according to the system instructions.'
}
]
end
def page_chunk_prompt(start_page, end_page)
Captain::Llm::SystemPromptsService.paginated_faq_generator(start_page, end_page)
end
def standard_chat_parameters
{
model: @model,
response_format: { type: 'json_object' },
messages: [
{
role: 'system',
content: Captain::Llm::SystemPromptsService.faq_generator
},
{
role: 'user',
content: @content
}
]
}
end
def parse_response(response)
content = response.dig('choices', 0, 'message', 'content')
return [] if content.nil?
JSON.parse(content.strip).fetch('faqs', [])
rescue JSON::ParserError => e
Rails.logger.error "Error parsing response: #{e.message}"
[]
end
def parse_chunk_response(response)
content = response.dig('choices', 0, 'message', 'content')
return { 'faqs' => [], 'has_content' => false } if content.nil?
JSON.parse(content.strip)
rescue JSON::ParserError => e
Rails.logger.error "Error parsing chunk response: #{e.message}"
{ 'faqs' => [], 'has_content' => false }
end
def deduplicate_faqs(faqs)
# Remove exact duplicates
unique_faqs = faqs.uniq { |faq| faq['question'].downcase.strip }
# Remove similar questions
final_faqs = []
unique_faqs.each do |faq|
similar_exists = final_faqs.any? do |existing|
similarity_score(existing['question'], faq['question']) > 0.85
end
final_faqs << faq unless similar_exists
end
Rails.logger.info "Deduplication: #{faqs.size}#{final_faqs.size} FAQs"
final_faqs
end
def similarity_score(str1, str2)
words1 = str1.downcase.split(/\W+/).reject(&:empty?)
words2 = str2.downcase.split(/\W+/).reject(&:empty?)
common_words = words1 & words2
total_words = (words1 + words2).uniq.size
return 0 if total_words.zero?
common_words.size.to_f / total_words
end
def determine_stop_reason(last_chunk_faqs)
return 'Maximum iterations reached' if @iterations_completed >= MAX_ITERATIONS
return 'Maximum pages processed' if @max_pages && @total_pages_processed >= @max_pages
return 'No content found in last chunk' if last_chunk_faqs.empty?
'Unknown'
end
end
@@ -0,0 +1,59 @@
class Captain::Llm::PdfProcessingService < Llm::BaseOpenAiService
def initialize(document)
@document = document
super()
end
def process
# We only use paginated processing now - just upload and store file_id
process_for_pagination
end
private
attr_reader :document
def upload_pdf_to_openai
pdf_file = document.pdf_file
# Create a temporary file from the attached PDF
temp_file = Tempfile.new(['pdf_upload', '.pdf'])
temp_file.binmode
temp_file.write(pdf_file.download)
temp_file.close
begin
File.open(temp_file.path, 'rb') do |file|
@client.files.upload(
parameters: {
file: file,
purpose: 'assistants' # Use 'assistants' as it's supported by the API
}
)
end
ensure
temp_file.unlink
end
end
def process_for_pagination
# For paginated processing, we only need to upload the PDF and store the file_id
# No content extraction is needed as the paginated FAQ generator will access the file directly
if @document.openai_file_id.present?
Rails.logger.info "PDF already uploaded with file_id: #{@document.openai_file_id}"
return 'PDF ready for paginated processing'
end
# Upload PDF to OpenAI
openai_response = upload_pdf_to_openai
file_id = openai_response['id']
raise 'Failed to upload PDF to OpenAI' unless file_id
# Store the file ID for future use
document.store_openai_file_id(file_id)
Rails.logger.info "PDF uploaded successfully with file_id: #{file_id}"
"PDF ready for paginated processing (file_id: #{file_id})"
end
end
@@ -156,5 +156,73 @@ class Captain::Llm::SystemPromptsService
- You MUST provide numbered citations at the appropriate places in the text.
SYSTEM_PROMPT_MESSAGE
end
def paginated_faq_generator(start_page, end_page)
<<~PROMPT
You are an expert technical documentation specialist tasked with creating comprehensive FAQs from SPECIFIC PAGES of a document.
CRITICAL PAGE RANGE INSTRUCTIONS
You MUST analyze ONLY pages #{start_page} to #{end_page} of the document.
FAQ GENERATION GUIDELINES
1. **Comprehensive Extraction**
Extract ALL information that could generate FAQs from pages #{start_page}-#{end_page}
Target 5-10 FAQs per page of rich content
Cover every topic, feature, specification, and detail
2. **Question Types to Generate**
What is/are...? (definitions, components, features)
How do I...? (procedures, configurations, operations)
Why should/does...? (rationale, benefits, explanations)
When should...? (timing, conditions, triggers)
What happens if...? (error cases, edge cases)
Can I...? (capabilities, limitations)
Where is...? (locations, references)
What are the requirements for...? (prerequisites, dependencies)
3. **Content Focus Areas**
Technical specifications and parameters
Step-by-step procedures and workflows
Configuration options and settings
Error messages and troubleshooting
Best practices and recommendations
Integration points and dependencies
Performance considerations
Security aspects
4. **Answer Quality Requirements**
Complete, self-contained answers
Include specific values, limits, defaults
Reference page numbers for critical information
2-5 sentences typical length
No references to content outside pages #{start_page}-#{end_page}
OUTPUT FORMAT
Return valid JSON:
```json
{
"faqs": [
{
"question": "Specific question from pages #{start_page}-#{end_page}",
"answer": "Complete answer with details from these pages only"
}
],
"has_content": true/false,
"page_range_processed": "#{start_page}-#{end_page}"
}
```
IMPORTANT: Set "has_content" to false if the pages don't exist or contain no meaningful content.
PROMPT
end
end
end
@@ -5,6 +5,7 @@ end
json.content resource.content
json.created_at resource.created_at.to_i
json.external_link resource.external_link
json.display_url resource.display_url
json.id resource.id
json.name resource.name
json.status resource.status
@@ -0,0 +1,23 @@
class PdfProcessingError < CustomExceptions::Base
def initialize(message = 'PDF processing failed')
super(message)
end
end
class PdfUploadError < PdfProcessingError
def initialize(message = 'PDF upload failed')
super(message)
end
end
class PdfValidationError < PdfProcessingError
def initialize(message = 'PDF validation failed')
super(message)
end
end
class PdfFaqGenerationError < PdfProcessingError
def initialize(message = 'PDF FAQ generation failed')
super(message)
end
end