feat: add be changes for captain pdf support for faq generation
This commit is contained in:
@@ -16,6 +16,7 @@ Metrics/ClassLength:
|
||||
Exclude:
|
||||
- 'app/models/message.rb'
|
||||
- 'app/models/conversation.rb'
|
||||
- 'enterprise/app/services/captain/llm/system_prompts_service.rb'
|
||||
|
||||
Metrics/MethodLength:
|
||||
Max: 19
|
||||
|
||||
+5
-1
@@ -67,7 +67,11 @@ Rails.application.routes.draw do
|
||||
resources :copilot_threads, only: [:index, :create] do
|
||||
resources :copilot_messages, only: [:index, :create]
|
||||
end
|
||||
resources :documents, only: [:index, :show, :create, :destroy]
|
||||
resources :documents, only: [:index, :show, :create, :destroy] do
|
||||
collection do
|
||||
post :upload_pdf
|
||||
end
|
||||
end
|
||||
end
|
||||
resources :agent_bots, only: [:index, :create, :show, :update, :destroy] do
|
||||
delete :avatar, on: :member
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
class AddPdfSupportToCaptainDocuments < ActiveRecord::Migration[7.1]
|
||||
def change
|
||||
add_column :captain_documents, :content_type, :string
|
||||
add_column :captain_documents, :file_size, :bigint
|
||||
|
||||
add_index :captain_documents, :content_type
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,5 @@
|
||||
class AddMetadataToCaptainDocuments < ActiveRecord::Migration[7.1]
|
||||
def change
|
||||
add_column :captain_documents, :metadata, :jsonb, default: {}
|
||||
end
|
||||
end
|
||||
+5
-1
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[7.1].define(version: 2025_07_22_152516) do
|
||||
ActiveRecord::Schema[7.1].define(version: 2025_08_05_082345) do
|
||||
# These extensions should be enabled to support this database
|
||||
enable_extension "pg_stat_statements"
|
||||
enable_extension "pg_trgm"
|
||||
@@ -292,9 +292,13 @@ ActiveRecord::Schema[7.1].define(version: 2025_07_22_152516) do
|
||||
t.datetime "created_at", null: false
|
||||
t.datetime "updated_at", null: false
|
||||
t.integer "status", default: 0, null: false
|
||||
t.string "content_type"
|
||||
t.bigint "file_size"
|
||||
t.jsonb "metadata", default: {}
|
||||
t.index ["account_id"], name: "index_captain_documents_on_account_id"
|
||||
t.index ["assistant_id", "external_link"], name: "index_captain_documents_on_assistant_id_and_external_link", unique: true
|
||||
t.index ["assistant_id"], name: "index_captain_documents_on_assistant_id"
|
||||
t.index ["content_type"], name: "index_captain_documents_on_content_type"
|
||||
t.index ["status"], name: "index_captain_documents_on_status"
|
||||
end
|
||||
|
||||
|
||||
@@ -3,9 +3,9 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
|
||||
before_action -> { check_authorization(Captain::Assistant) }
|
||||
|
||||
before_action :set_current_page, only: [:index]
|
||||
before_action :set_documents, except: [:create]
|
||||
before_action :set_documents, except: [:create, :upload_pdf]
|
||||
before_action :set_document, only: [:show, :destroy]
|
||||
before_action :set_assistant, only: [:create]
|
||||
before_action :set_assistant, only: [:create, :upload_pdf]
|
||||
RESULTS_PER_PAGE = 25
|
||||
|
||||
def index
|
||||
@@ -22,9 +22,33 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
|
||||
return render_could_not_create_error('Missing Assistant') if @assistant.nil?
|
||||
|
||||
@document = @assistant.documents.build(document_params)
|
||||
|
||||
# Handle PDF file upload if present
|
||||
@document.pdf_file.attach(document_params[:pdf_file]) if document_params[:pdf_file].present?
|
||||
|
||||
@document.save!
|
||||
rescue Captain::Document::LimitExceededError => e
|
||||
render_could_not_create_error(e.message)
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "Document creation error: #{e.message}"
|
||||
render_could_not_create_error('Failed to create document')
|
||||
end
|
||||
|
||||
def upload_pdf
|
||||
pdf_file = params[:pdf_file]
|
||||
|
||||
return render_could_not_create_error('PDF file is required') if pdf_file.blank?
|
||||
return render_could_not_create_error('Invalid file type') unless valid_pdf?(pdf_file)
|
||||
return render_could_not_create_error('File too large (max 512MB)') if pdf_file.size > 512.megabytes
|
||||
return render_could_not_create_error('Missing Assistant') if @assistant.nil?
|
||||
|
||||
create_pdf_document(pdf_file)
|
||||
render :show
|
||||
rescue Captain::Document::LimitExceededError => e
|
||||
render_could_not_create_error(e.message)
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "PDF upload error: #{e.message}"
|
||||
render_could_not_create_error('Failed to upload PDF')
|
||||
end
|
||||
|
||||
def destroy
|
||||
@@ -43,7 +67,8 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
|
||||
end
|
||||
|
||||
def set_assistant
|
||||
@assistant = Current.account.captain_assistants.find_by(id: document_params[:assistant_id])
|
||||
assistant_id = document_params[:assistant_id] || params[:assistant_id]
|
||||
@assistant = Current.account.captain_assistants.find_by(id: assistant_id)
|
||||
end
|
||||
|
||||
def set_current_page
|
||||
@@ -55,6 +80,19 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
|
||||
end
|
||||
|
||||
def document_params
|
||||
params.require(:document).permit(:name, :external_link, :assistant_id)
|
||||
params.require(:document).permit(:name, :external_link, :assistant_id, :pdf_file)
|
||||
end
|
||||
|
||||
def valid_pdf?(file)
|
||||
file.content_type == 'application/pdf'
|
||||
end
|
||||
|
||||
def create_pdf_document(pdf_file)
|
||||
@document = @assistant.documents.build(
|
||||
name: pdf_file.original_filename,
|
||||
account: Current.account
|
||||
)
|
||||
@document.pdf_file.attach(pdf_file)
|
||||
@document.save!
|
||||
end
|
||||
end
|
||||
|
||||
@@ -2,7 +2,9 @@ class Captain::Documents::CrawlJob < ApplicationJob
|
||||
queue_as :low
|
||||
|
||||
def perform(document)
|
||||
if InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present?
|
||||
if document.pdf_document?
|
||||
perform_pdf_processing(document)
|
||||
elsif InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present?
|
||||
perform_firecrawl_crawl(document)
|
||||
else
|
||||
perform_simple_crawl(document)
|
||||
@@ -13,6 +15,19 @@ class Captain::Documents::CrawlJob < ApplicationJob
|
||||
|
||||
include Captain::FirecrawlHelper
|
||||
|
||||
def perform_pdf_processing(document)
|
||||
pdf_processor = Captain::Llm::PdfProcessingService.new(document)
|
||||
pdf_processor.process
|
||||
|
||||
# Mark document as available - content is not needed for paginated processing
|
||||
document.update!(status: :available)
|
||||
|
||||
Rails.logger.info "Successfully processed PDF document #{document.id}"
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "Failed to process PDF document #{document.id}: #{e.message}"
|
||||
document.update!(status: :available)
|
||||
end
|
||||
|
||||
def perform_simple_crawl(document)
|
||||
page_links = Captain::Tools::SimplePageCrawlService.new(document.external_link).page_links
|
||||
|
||||
|
||||
@@ -1,17 +1,69 @@
|
||||
class Captain::Documents::ResponseBuilderJob < ApplicationJob
|
||||
queue_as :low
|
||||
|
||||
def perform(document)
|
||||
def perform(document, options = {})
|
||||
reset_previous_responses(document)
|
||||
|
||||
faqs = Captain::Llm::FaqGeneratorService.new(document.content).generate
|
||||
faqs.each do |faq|
|
||||
create_response(faq, document)
|
||||
faqs = generate_faqs(document, options)
|
||||
create_responses_from_faqs(faqs, document)
|
||||
|
||||
Rails.logger.info "FAQ generation complete. Total FAQs created: #{faqs.size}"
|
||||
end
|
||||
|
||||
def generate_faqs(document, options)
|
||||
if should_use_pagination?(document)
|
||||
generate_paginated_faqs(document, options)
|
||||
else
|
||||
generate_standard_faqs(document)
|
||||
end
|
||||
end
|
||||
|
||||
def generate_paginated_faqs(document, options)
|
||||
Rails.logger.info "Using paginated FAQ generation for document #{document.id}"
|
||||
service = build_paginated_service(document, options)
|
||||
faqs = service.generate
|
||||
store_paginated_metadata(document, service)
|
||||
faqs
|
||||
end
|
||||
|
||||
def generate_standard_faqs(document)
|
||||
Rails.logger.info "Using standard FAQ generation for document #{document.id}"
|
||||
Captain::Llm::FaqGeneratorService.new(document.content).generate
|
||||
end
|
||||
|
||||
def build_paginated_service(document, options)
|
||||
Captain::Llm::PaginatedFaqGeneratorService.new(
|
||||
document,
|
||||
pages_per_chunk: options[:pages_per_chunk],
|
||||
max_pages: options[:max_pages]
|
||||
)
|
||||
end
|
||||
|
||||
def store_paginated_metadata(document, service)
|
||||
document.update!(
|
||||
metadata: document.metadata.merge(
|
||||
'faq_generation' => {
|
||||
'method' => 'paginated',
|
||||
'pages_processed' => service.total_pages_processed,
|
||||
'iterations' => service.iterations_completed,
|
||||
'timestamp' => Time.current.iso8601
|
||||
}
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def create_responses_from_faqs(faqs, document)
|
||||
faqs.each { |faq| create_response(faq, document) }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def should_use_pagination?(document)
|
||||
# Auto-detect when to use pagination
|
||||
# For now, use pagination for PDFs with OpenAI file ID
|
||||
document.pdf_document? && document.openai_file_id.present?
|
||||
end
|
||||
|
||||
def reset_previous_responses(response_document)
|
||||
response_document.responses.destroy_all
|
||||
end
|
||||
|
||||
@@ -2,6 +2,9 @@ class Captain::Llm::UpdateEmbeddingJob < ApplicationJob
|
||||
queue_as :low
|
||||
|
||||
def perform(record, content)
|
||||
# Early return if record was deleted before job execution
|
||||
return unless record&.persisted?
|
||||
|
||||
embedding = Captain::Llm::EmbeddingService.new.get_embedding(content)
|
||||
record.update!(embedding: embedding)
|
||||
end
|
||||
|
||||
@@ -4,7 +4,10 @@
|
||||
#
|
||||
# id :bigint not null, primary key
|
||||
# content :text
|
||||
# content_type :string
|
||||
# external_link :string not null
|
||||
# file_size :bigint
|
||||
# metadata :jsonb
|
||||
# name :string
|
||||
# status :integer default("in_progress"), not null
|
||||
# created_at :datetime not null
|
||||
@@ -26,11 +29,15 @@ class Captain::Document < ApplicationRecord
|
||||
belongs_to :assistant, class_name: 'Captain::Assistant'
|
||||
has_many :responses, class_name: 'Captain::AssistantResponse', dependent: :destroy, as: :documentable
|
||||
belongs_to :account
|
||||
has_one_attached :pdf_file
|
||||
|
||||
validates :external_link, presence: true
|
||||
validates :external_link, uniqueness: { scope: :assistant_id }
|
||||
validates :external_link, presence: true, unless: -> { pdf_file.attached? }
|
||||
validates :external_link, uniqueness: { scope: :assistant_id }, allow_blank: true
|
||||
validates :content, length: { maximum: 200_000 }
|
||||
validates :pdf_file, presence: true, if: :pdf_document?
|
||||
validate :validate_pdf_format, if: :pdf_document?
|
||||
before_validation :ensure_account_id
|
||||
before_validation :set_external_link_for_pdf
|
||||
|
||||
enum status: {
|
||||
in_progress: 0,
|
||||
@@ -41,12 +48,34 @@ class Captain::Document < ApplicationRecord
|
||||
after_create_commit :enqueue_crawl_job
|
||||
after_create_commit :update_document_usage
|
||||
after_destroy :update_document_usage
|
||||
after_commit :enqueue_response_builder_job
|
||||
after_commit :enqueue_response_builder_job, on: :update, if: :should_enqueue_response_builder?
|
||||
scope :ordered, -> { order(created_at: :desc) }
|
||||
|
||||
scope :for_account, ->(account_id) { where(account_id: account_id) }
|
||||
scope :for_assistant, ->(assistant_id) { where(assistant_id: assistant_id) }
|
||||
|
||||
def pdf_document?
|
||||
(external_link&.ends_with?('.pdf')) || (pdf_file.attached? && pdf_file.content_type == 'application/pdf')
|
||||
end
|
||||
|
||||
def openai_file_id
|
||||
metadata['openai_file_id']
|
||||
end
|
||||
|
||||
def store_openai_file_id(file_id)
|
||||
update!(metadata: metadata.merge('openai_file_id' => file_id))
|
||||
end
|
||||
|
||||
def display_url
|
||||
return external_link if external_link.present? && !external_link.start_with?('PDF:')
|
||||
|
||||
if pdf_file.attached?
|
||||
Rails.application.routes.url_helpers.rails_blob_url(pdf_file, only_path: false)
|
||||
else
|
||||
external_link
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def enqueue_crawl_job
|
||||
@@ -61,6 +90,12 @@ class Captain::Document < ApplicationRecord
|
||||
Captain::Documents::ResponseBuilderJob.perform_later(self)
|
||||
end
|
||||
|
||||
def should_enqueue_response_builder?
|
||||
# Only enqueue when status changes to available
|
||||
# Avoid re-enqueueing when metadata is updated by the job itself
|
||||
saved_change_to_status? && status == 'available'
|
||||
end
|
||||
|
||||
def update_document_usage
|
||||
account.update_document_usage
|
||||
end
|
||||
@@ -73,4 +108,23 @@ class Captain::Document < ApplicationRecord
|
||||
limits = account.usage_limits[:captain][:documents]
|
||||
raise LimitExceededError, 'Document limit exceeded' unless limits[:current_available].positive?
|
||||
end
|
||||
|
||||
def validate_pdf_format
|
||||
return unless pdf_file.attached?
|
||||
|
||||
errors.add(:pdf_file, 'must be a PDF file') unless pdf_file.content_type == 'application/pdf'
|
||||
|
||||
return unless pdf_file.byte_size > 512.megabytes
|
||||
|
||||
errors.add(:pdf_file, 'must be less than 512MB')
|
||||
end
|
||||
|
||||
def set_external_link_for_pdf
|
||||
return unless pdf_file.attached? && external_link.blank?
|
||||
|
||||
# Set a unique external_link for PDF files
|
||||
# Format: PDF: filename_timestamp (without extension)
|
||||
timestamp = Time.current.strftime('%Y%m%d%H%M%S')
|
||||
self.external_link = "PDF: #{pdf_file.filename.base}_#{timestamp}"
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
class Captain::Llm::PaginatedFaqGeneratorService < Llm::BaseOpenAiService
|
||||
# Default pages per chunk - easily configurable
|
||||
DEFAULT_PAGES_PER_CHUNK = 10
|
||||
MAX_ITERATIONS = 20 # Safety limit to prevent infinite loops
|
||||
|
||||
attr_reader :total_pages_processed, :iterations_completed
|
||||
|
||||
def initialize(document, options = {})
|
||||
super()
|
||||
@document = document
|
||||
@pages_per_chunk = options[:pages_per_chunk] || DEFAULT_PAGES_PER_CHUNK
|
||||
@max_pages = options[:max_pages] # Optional limit from UI
|
||||
@total_pages_processed = 0
|
||||
@iterations_completed = 0
|
||||
end
|
||||
|
||||
def generate
|
||||
raise 'Document must have openai_file_id for paginated processing' if @document&.openai_file_id.blank?
|
||||
|
||||
generate_paginated_faqs
|
||||
end
|
||||
|
||||
# Method to check if we should continue processing
|
||||
def should_continue_processing?(last_chunk_faqs)
|
||||
# Stop if we've hit the maximum iterations
|
||||
return false if @iterations_completed >= MAX_ITERATIONS
|
||||
|
||||
# Stop if we've processed the maximum pages specified
|
||||
return false if @max_pages && @total_pages_processed >= @max_pages
|
||||
|
||||
# Stop if the last chunk returned no FAQs (likely no more content)
|
||||
return false if last_chunk_faqs.empty?
|
||||
|
||||
# Continue processing
|
||||
true
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def generate_standard_faqs
|
||||
response = @client.chat(parameters: standard_chat_parameters)
|
||||
parse_response(response)
|
||||
rescue OpenAI::Error => e
|
||||
Rails.logger.error "OpenAI API Error: #{e.message}"
|
||||
[]
|
||||
end
|
||||
|
||||
def generate_paginated_faqs
|
||||
Rails.logger.info "Starting paginated FAQ generation (#{@pages_per_chunk} pages per chunk)"
|
||||
|
||||
all_faqs = []
|
||||
current_page = 1
|
||||
|
||||
loop do
|
||||
end_page = calculate_end_page(current_page)
|
||||
chunk_faqs = process_chunk_and_update_state(current_page, end_page, all_faqs)
|
||||
|
||||
unless should_continue_processing?(chunk_faqs)
|
||||
Rails.logger.info "Stopping processing. Reason: #{determine_stop_reason(chunk_faqs)}"
|
||||
break
|
||||
end
|
||||
|
||||
current_page = end_page + 1
|
||||
end
|
||||
|
||||
Rails.logger.info "Paginated generation complete. Total FAQs: #{all_faqs.size}, Pages processed: #{@total_pages_processed}"
|
||||
deduplicate_faqs(all_faqs)
|
||||
end
|
||||
|
||||
def calculate_end_page(current_page)
|
||||
end_page = current_page + @pages_per_chunk - 1
|
||||
@max_pages && end_page > @max_pages ? @max_pages : end_page
|
||||
end
|
||||
|
||||
def process_chunk_and_update_state(current_page, end_page, all_faqs)
|
||||
Rails.logger.info "Processing pages #{current_page}-#{end_page} (iteration #{@iterations_completed + 1})"
|
||||
|
||||
chunk_result = process_page_chunk(current_page, end_page)
|
||||
chunk_faqs = chunk_result[:faqs]
|
||||
|
||||
all_faqs.concat(chunk_faqs)
|
||||
@total_pages_processed = end_page
|
||||
@iterations_completed += 1
|
||||
|
||||
Rails.logger.info "Chunk generated #{chunk_faqs.size} FAQs. Total so far: #{all_faqs.size}"
|
||||
chunk_faqs
|
||||
end
|
||||
|
||||
def process_page_chunk(start_page, end_page)
|
||||
response = @client.chat(parameters: build_chunk_parameters(start_page, end_page))
|
||||
result = parse_chunk_response(response)
|
||||
{ faqs: result['faqs'] || [], has_content: result['has_content'] != false }
|
||||
rescue OpenAI::Error => e
|
||||
Rails.logger.error "Error processing pages #{start_page}-#{end_page}: #{e.message}"
|
||||
{ faqs: [], has_content: false }
|
||||
end
|
||||
|
||||
def build_chunk_parameters(start_page, end_page)
|
||||
{
|
||||
model: @model,
|
||||
response_format: { type: 'json_object' },
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: page_chunk_prompt(start_page, end_page)
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: build_user_content
|
||||
}
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
def build_user_content
|
||||
[
|
||||
{
|
||||
type: 'file',
|
||||
file: { file_id: @document.openai_file_id }
|
||||
},
|
||||
{
|
||||
type: 'text',
|
||||
text: 'Process this document according to the system instructions.'
|
||||
}
|
||||
]
|
||||
end
|
||||
|
||||
def page_chunk_prompt(start_page, end_page)
|
||||
Captain::Llm::SystemPromptsService.paginated_faq_generator(start_page, end_page)
|
||||
end
|
||||
|
||||
def standard_chat_parameters
|
||||
{
|
||||
model: @model,
|
||||
response_format: { type: 'json_object' },
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: Captain::Llm::SystemPromptsService.faq_generator
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: @content
|
||||
}
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
def parse_response(response)
|
||||
content = response.dig('choices', 0, 'message', 'content')
|
||||
return [] if content.nil?
|
||||
|
||||
JSON.parse(content.strip).fetch('faqs', [])
|
||||
rescue JSON::ParserError => e
|
||||
Rails.logger.error "Error parsing response: #{e.message}"
|
||||
[]
|
||||
end
|
||||
|
||||
def parse_chunk_response(response)
|
||||
content = response.dig('choices', 0, 'message', 'content')
|
||||
return { 'faqs' => [], 'has_content' => false } if content.nil?
|
||||
|
||||
JSON.parse(content.strip)
|
||||
rescue JSON::ParserError => e
|
||||
Rails.logger.error "Error parsing chunk response: #{e.message}"
|
||||
{ 'faqs' => [], 'has_content' => false }
|
||||
end
|
||||
|
||||
def deduplicate_faqs(faqs)
|
||||
# Remove exact duplicates
|
||||
unique_faqs = faqs.uniq { |faq| faq['question'].downcase.strip }
|
||||
|
||||
# Remove similar questions
|
||||
final_faqs = []
|
||||
unique_faqs.each do |faq|
|
||||
similar_exists = final_faqs.any? do |existing|
|
||||
similarity_score(existing['question'], faq['question']) > 0.85
|
||||
end
|
||||
|
||||
final_faqs << faq unless similar_exists
|
||||
end
|
||||
|
||||
Rails.logger.info "Deduplication: #{faqs.size} → #{final_faqs.size} FAQs"
|
||||
final_faqs
|
||||
end
|
||||
|
||||
def similarity_score(str1, str2)
|
||||
words1 = str1.downcase.split(/\W+/).reject(&:empty?)
|
||||
words2 = str2.downcase.split(/\W+/).reject(&:empty?)
|
||||
|
||||
common_words = words1 & words2
|
||||
total_words = (words1 + words2).uniq.size
|
||||
|
||||
return 0 if total_words.zero?
|
||||
|
||||
common_words.size.to_f / total_words
|
||||
end
|
||||
|
||||
def determine_stop_reason(last_chunk_faqs)
|
||||
return 'Maximum iterations reached' if @iterations_completed >= MAX_ITERATIONS
|
||||
return 'Maximum pages processed' if @max_pages && @total_pages_processed >= @max_pages
|
||||
return 'No content found in last chunk' if last_chunk_faqs.empty?
|
||||
|
||||
'Unknown'
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,59 @@
|
||||
class Captain::Llm::PdfProcessingService < Llm::BaseOpenAiService
|
||||
def initialize(document)
|
||||
@document = document
|
||||
super()
|
||||
end
|
||||
|
||||
def process
|
||||
# We only use paginated processing now - just upload and store file_id
|
||||
process_for_pagination
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :document
|
||||
|
||||
def upload_pdf_to_openai
|
||||
pdf_file = document.pdf_file
|
||||
|
||||
# Create a temporary file from the attached PDF
|
||||
temp_file = Tempfile.new(['pdf_upload', '.pdf'])
|
||||
temp_file.binmode
|
||||
temp_file.write(pdf_file.download)
|
||||
temp_file.close
|
||||
|
||||
begin
|
||||
File.open(temp_file.path, 'rb') do |file|
|
||||
@client.files.upload(
|
||||
parameters: {
|
||||
file: file,
|
||||
purpose: 'assistants' # Use 'assistants' as it's supported by the API
|
||||
}
|
||||
)
|
||||
end
|
||||
ensure
|
||||
temp_file.unlink
|
||||
end
|
||||
end
|
||||
|
||||
def process_for_pagination
|
||||
# For paginated processing, we only need to upload the PDF and store the file_id
|
||||
# No content extraction is needed as the paginated FAQ generator will access the file directly
|
||||
if @document.openai_file_id.present?
|
||||
Rails.logger.info "PDF already uploaded with file_id: #{@document.openai_file_id}"
|
||||
return 'PDF ready for paginated processing'
|
||||
end
|
||||
|
||||
# Upload PDF to OpenAI
|
||||
openai_response = upload_pdf_to_openai
|
||||
file_id = openai_response['id']
|
||||
|
||||
raise 'Failed to upload PDF to OpenAI' unless file_id
|
||||
|
||||
# Store the file ID for future use
|
||||
document.store_openai_file_id(file_id)
|
||||
|
||||
Rails.logger.info "PDF uploaded successfully with file_id: #{file_id}"
|
||||
"PDF ready for paginated processing (file_id: #{file_id})"
|
||||
end
|
||||
end
|
||||
@@ -156,5 +156,73 @@ class Captain::Llm::SystemPromptsService
|
||||
- You MUST provide numbered citations at the appropriate places in the text.
|
||||
SYSTEM_PROMPT_MESSAGE
|
||||
end
|
||||
|
||||
def paginated_faq_generator(start_page, end_page)
|
||||
<<~PROMPT
|
||||
You are an expert technical documentation specialist tasked with creating comprehensive FAQs from SPECIFIC PAGES of a document.
|
||||
|
||||
════════════════════════════════════════════════════════
|
||||
CRITICAL PAGE RANGE INSTRUCTIONS
|
||||
════════════════════════════════════════════════════════
|
||||
|
||||
You MUST analyze ONLY pages #{start_page} to #{end_page} of the document.
|
||||
|
||||
════════════════════════════════════════════════════════
|
||||
FAQ GENERATION GUIDELINES
|
||||
════════════════════════════════════════════════════════
|
||||
|
||||
1. **Comprehensive Extraction**
|
||||
• Extract ALL information that could generate FAQs from pages #{start_page}-#{end_page}
|
||||
• Target 5-10 FAQs per page of rich content
|
||||
• Cover every topic, feature, specification, and detail
|
||||
|
||||
2. **Question Types to Generate**
|
||||
• What is/are...? (definitions, components, features)
|
||||
• How do I...? (procedures, configurations, operations)
|
||||
• Why should/does...? (rationale, benefits, explanations)
|
||||
• When should...? (timing, conditions, triggers)
|
||||
• What happens if...? (error cases, edge cases)
|
||||
• Can I...? (capabilities, limitations)
|
||||
• Where is...? (locations, references)
|
||||
• What are the requirements for...? (prerequisites, dependencies)
|
||||
|
||||
3. **Content Focus Areas**
|
||||
• Technical specifications and parameters
|
||||
• Step-by-step procedures and workflows
|
||||
• Configuration options and settings
|
||||
• Error messages and troubleshooting
|
||||
• Best practices and recommendations
|
||||
• Integration points and dependencies
|
||||
• Performance considerations
|
||||
• Security aspects
|
||||
|
||||
4. **Answer Quality Requirements**
|
||||
• Complete, self-contained answers
|
||||
• Include specific values, limits, defaults
|
||||
• Reference page numbers for critical information
|
||||
• 2-5 sentences typical length
|
||||
• No references to content outside pages #{start_page}-#{end_page}
|
||||
|
||||
════════════════════════════════════════════════════════
|
||||
OUTPUT FORMAT
|
||||
════════════════════════════════════════════════════════
|
||||
|
||||
Return valid JSON:
|
||||
```json
|
||||
{
|
||||
"faqs": [
|
||||
{
|
||||
"question": "Specific question from pages #{start_page}-#{end_page}",
|
||||
"answer": "Complete answer with details from these pages only"
|
||||
}
|
||||
],
|
||||
"has_content": true/false,
|
||||
"page_range_processed": "#{start_page}-#{end_page}"
|
||||
}
|
||||
```
|
||||
|
||||
IMPORTANT: Set "has_content" to false if the pages don't exist or contain no meaningful content.
|
||||
PROMPT
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -5,6 +5,7 @@ end
|
||||
json.content resource.content
|
||||
json.created_at resource.created_at.to_i
|
||||
json.external_link resource.external_link
|
||||
json.display_url resource.display_url
|
||||
json.id resource.id
|
||||
json.name resource.name
|
||||
json.status resource.status
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
class PdfProcessingError < CustomExceptions::Base
|
||||
def initialize(message = 'PDF processing failed')
|
||||
super(message)
|
||||
end
|
||||
end
|
||||
|
||||
class PdfUploadError < PdfProcessingError
|
||||
def initialize(message = 'PDF upload failed')
|
||||
super(message)
|
||||
end
|
||||
end
|
||||
|
||||
class PdfValidationError < PdfProcessingError
|
||||
def initialize(message = 'PDF validation failed')
|
||||
super(message)
|
||||
end
|
||||
end
|
||||
|
||||
class PdfFaqGenerationError < PdfProcessingError
|
||||
def initialize(message = 'PDF FAQ generation failed')
|
||||
super(message)
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user