diff --git a/Gemfile.lock b/Gemfile.lock index d9908f5e1..3494b4489 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -275,14 +275,16 @@ GEM fugit (1.11.1) et-orbi (~> 1, >= 1.2.11) raabro (~> 1.4) - gapic-common (0.20.0) + gapic-common (1.0.0) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) - google-protobuf (~> 3.14) - googleapis-common-protos (>= 1.3.12, < 2.a) - googleapis-common-protos-types (>= 1.3.1, < 2.a) - googleauth (~> 1.0) - grpc (~> 1.36) + google-cloud-env (~> 2.2) + google-logging-utils (~> 0.1) + google-protobuf (>= 3.25, < 5.a) + googleapis-common-protos (~> 1.6) + googleapis-common-protos-types (~> 1.15) + googleauth (~> 1.12) + grpc (~> 1.66) geocoder (1.8.1) gli (2.22.2) ostruct @@ -327,6 +329,13 @@ GEM google-cloud-translate-v3 (0.10.0) gapic-common (>= 0.20.0, < 2.a) google-cloud-errors (~> 1.0) + google-cloud-vision (2.0.2) + google-cloud-core (~> 1.6) + google-cloud-vision-v1 (~> 1.2) + google-cloud-vision-v1 (1.3.0) + gapic-common (~> 1.0) + google-cloud-errors (~> 1.0) + google-logging-utils (0.2.0) google-protobuf (3.25.7) googleapis-common-protos (1.6.0) google-protobuf (>= 3.18, < 5.a) @@ -334,9 +343,10 @@ GEM grpc (~> 1.41) googleapis-common-protos-types (1.20.0) google-protobuf (>= 3.18, < 5.a) - googleauth (1.11.2) + googleauth (1.14.0) faraday (>= 1.0, < 3.a) - google-cloud-env (~> 2.1) + google-cloud-env (~> 2.2) + google-logging-utils (~> 0.1) jwt (>= 1.4, < 3.0) multi_json (~> 1.11) os (>= 0.9, < 2.0) @@ -381,6 +391,7 @@ GEM httpclient (2.8.3) i18n (1.14.7) concurrent-ruby (~> 1.0) + iconv (1.1.0) image_processing (1.12.2) mini_magick (>= 4.9.5, < 5) ruby-vips (>= 2.0.17, < 3) @@ -550,6 +561,9 @@ GEM parser (3.3.8.0) ast (~> 2.4.1) racc + pdftoimage (0.2.1) + iconv (~> 1.0) + shellwords (~> 0.2.2) pg (1.5.3) pg_search (2.3.6) activerecord (>= 5.2) @@ -672,6 +686,7 @@ GEM rspec-support (3.13.1) rspec_junit_formatter (0.6.0) rspec-core (>= 2, < 4, != 2.12.0) + rtesseract (3.1.4) rubocop (1.75.6) json (~> 2.3) language_server-protocol (~> 3.17.0.2) @@ -747,6 +762,7 @@ GEM sentry-ruby (~> 5.19.0) sidekiq (>= 3.0) sexp_processor (4.17.0) + shellwords (0.2.2) shopify_api (14.8.0) activesupport concurrent-ruby @@ -927,6 +943,7 @@ DEPENDENCIES google-cloud-dialogflow-v2 (>= 0.24.0) google-cloud-storage (>= 1.48.0) google-cloud-translate-v3 (>= 0.7.0) + google-cloud-vision groupdate grpc haikunator @@ -950,6 +967,7 @@ DEPENDENCIES lograge (~> 0.14.0) maxminddb meta_request (>= 0.8.3) + mini_magick mock_redis neighbor net-smtp (~> 0.3.4) @@ -959,6 +977,7 @@ DEPENDENCIES omniauth-google-oauth2 (>= 1.1.3) omniauth-oauth2 omniauth-rails_csrf_protection (~> 1.0, >= 1.0.2) + pdftoimage pg pg_search pgvector @@ -978,6 +997,7 @@ DEPENDENCIES reverse_markdown rspec-rails (>= 6.1.5) rspec_junit_formatter + rtesseract rubocop rubocop-factory_bot rubocop-performance @@ -1020,4 +1040,4 @@ RUBY VERSION ruby 3.4.4p34 BUNDLED WITH - 2.5.16 + 2.6.7 diff --git a/db/migrate/20250623120000_add_document_type_and_file_to_captain_documents.rb b/db/migrate/20250623120000_add_document_type_and_file_to_captain_documents.rb new file mode 100644 index 000000000..12769e704 --- /dev/null +++ b/db/migrate/20250623120000_add_document_type_and_file_to_captain_documents.rb @@ -0,0 +1,6 @@ +class AddDocumentTypeAndFileToCaptainDocuments < ActiveRecord::Migration[7.0] + def change + add_column :captain_documents, :document_type, :integer, default: 0, null: false + add_index :captain_documents, :document_type + end +end \ No newline at end of file diff --git a/db/schema.rb b/db/schema.rb index d065a8ff4..3f5839eaa 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2025_05_23_031839) do +ActiveRecord::Schema[7.1].define(version: 2025_06_23_120000) do # These extensions should be enabled to support this database enable_extension "pg_stat_statements" enable_extension "pg_trgm" @@ -289,9 +289,11 @@ ActiveRecord::Schema[7.1].define(version: 2025_05_23_031839) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "status", default: 0, null: false + t.integer "document_type", default: 0, null: false t.index ["account_id"], name: "index_captain_documents_on_account_id" t.index ["assistant_id", "external_link"], name: "index_captain_documents_on_assistant_id_and_external_link", unique: true t.index ["assistant_id"], name: "index_captain_documents_on_assistant_id" + t.index ["document_type"], name: "index_captain_documents_on_document_type" t.index ["status"], name: "index_captain_documents_on_status" end @@ -907,7 +909,7 @@ ActiveRecord::Schema[7.1].define(version: 2025_05_23_031839) do t.text "header_text" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.jsonb "config", default: {"allowed_locales"=>["en"]} + t.jsonb "config", default: {"allowed_locales" => ["en"]} t.boolean "archived", default: false t.bigint "channel_web_widget_id" t.index ["channel_web_widget_id"], name: "index_portals_on_channel_web_widget_id" diff --git a/docker/Dockerfile b/docker/Dockerfile index 2d753337d..781440aa2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,6 +34,8 @@ RUN apk update && apk add --no-cache \ git \ curl \ xz \ + poppler-utils \ + tesseract-ocr \ && mkdir -p /var/app \ && gem install bundler @@ -128,6 +130,8 @@ RUN apk update && apk add --no-cache \ imagemagick \ git \ vips \ + poppler-utils \ + tesseract-ocr \ && gem install bundler COPY --from=node /usr/local/bin/node /usr/local/bin/ diff --git a/enterprise/app/models/captain/document.rb b/enterprise/app/models/captain/document.rb index 6ed8d6e8a..3e5b1e43e 100644 --- a/enterprise/app/models/captain/document.rb +++ b/enterprise/app/models/captain/document.rb @@ -4,6 +4,7 @@ # # id :bigint not null, primary key # content :text +# document_type :integer default("url"), not null # external_link :string not null # name :string # status :integer default("in_progress"), not null @@ -17,6 +18,7 @@ # index_captain_documents_on_account_id (account_id) # index_captain_documents_on_assistant_id (assistant_id) # index_captain_documents_on_assistant_id_and_external_link (assistant_id,external_link) UNIQUE +# index_captain_documents_on_document_type (document_type) # index_captain_documents_on_status (status) # class Captain::Document < ApplicationRecord @@ -53,6 +55,14 @@ class Captain::Document < ApplicationRecord scope :for_account, ->(account_id) { where(account_id: account_id) } scope :for_assistant, ->(assistant_id) { where(assistant_id: assistant_id) } + def url_document? + document_type == 'url' + end + + def pdf_document? + document_type == 'pdf' + end + private def enqueue_crawl_job @@ -65,14 +75,6 @@ class Captain::Document < ApplicationRecord end end - def url_document? - document_type == 'url' - end - - def pdf_document? - document_type == 'pdf' - end - def detect_document_type return unless external_link.present?