Merge pull request #1781 from next-l/tika-container

Tikaのコンテナを追加

Author: nabeta

.env.template

@@ -11,6 +11,8 @@ REDIS_URL=redis://redis/enju-leaf-${RAILS_ENV}
 
 SOLR_URL=http://solr:8983/solr/enju_leaf_${RAILS_ENV}
 
+TIKA_URL=http://tika:9998
+
 MINIO_ROOT_USER=enju
 MINIO_ROOT_PASSWORD=password
 
@@ -21,11 +23,12 @@ ENJU_LEAF_TIME_ZONE=Asia/Tokyo
 ENJU_LEAF_STORAGE_BUCKET=enju-leaf
 ENJU_LEAF_STORAGE_ENDPOINT=http://minio:9000
 ENJU_LEAF_ACTION_MAILER_DELIVERY_METHOD=test
+ENJU_LEAF_EXTRACT_TEXT=
 # ENJU_LEAF_RESOURCESYNC_BASE_URL=http://localhost:8080
 
 CANTALOUPE_BASE_URI=http://localhost:8182
 
-no_proxy=localhost,webpacker,minio,solr
+no_proxy=localhost,webpacker,minio,solr,tika
 NO_PROXY=${no_proxy}
 
 UID=1000

.env.test

@@ -5,8 +5,11 @@ DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/e
 
 SOLR_URL=http://localhost:8983/solr/enju_leaf_test
 
+TIKA_URL=http://localhost:9998
+
 MINIO_ROOT_USER=enju
 MINIO_ROOT_PASSWORD=password
 
 ENJU_LEAF_DEFAULT_LOCALE=en
 ENJU_LEAF_TIME_ZONE=UTC
+ENJU_LEAF_EXTRACT_TEXT=true

.github/workflows/rubyonrails.yml

@@ -34,7 +34,7 @@ jobs:
           bundler-cache: true
       # Add or replace test runners here
       - name: Start containers
-        run: cp .env.template .env && docker-compose up -d solr
+        run: cp .env.template .env && docker-compose up -d solr tika
       - name: Setup Code Climate test-reporter
         run: |
           curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter

app/models/manifestation.rb

@@ -235,7 +235,7 @@ class Manifestation < ApplicationRecord
   end
   after_create :clear_cached_numdocs
   after_destroy :index_series_statement
-  after_save :index_series_statement, :extract_text!
+  after_save :index_series_statement
   after_touch do |manifestation|
     manifestation.index
     manifestation.index_series_statement
@@ -367,24 +367,21 @@ def self.pickup(keyword = nil, current_user = nil)
   end
 
   def extract_text
-    return nil unless attachment.attached?
-    return nil unless ENV['ENJU_EXTRACT_TEXT'] == 'true'
+    return unless attachment.attached?
+    return unless ENV['ENJU_LEAF_EXTRACT_TEXT'] == 'true'
 
-    client = Faraday.new(url: ENV['SOLR_URL'] || Sunspot.config.solr.url) do |conn|
-      conn.request :multipart
+    client = Faraday.new(url: ENV['TIKA_URL'] || 'http://tika:9998') do |conn|
       conn.adapter :net_http
     end
-    response = client.post('update/extract?extractOnly=true&wt=json&extractFormat=text') do |req|
-      req.headers['Content-type'] = 'text/html'
-      req.body = attachment.download
+
+    response = client.put('/tika/text') do |req|
+      req.headers['Content-Type'] = attachment.content_type
+      req.headers['Content-Length'] = attachment.byte_size.to_s
+      req.body = Faraday::UploadIO.new(StringIO.new(attachment.download), attachment.content_type)
     end
-    update_column(:fulltext, JSON.parse(response.body)[""])
-  end
 
-  def extract_text!
-    extract_text
-    index
-    Sunspot.commit
+    payload = JSON.parse(response.body)['X-TIKA:content'].strip.tr("\t", " ").gsub(/\r?\n/, "")
+    payload
   end
 
   def created(agent)

docker-compose.yml

@@ -12,6 +12,8 @@ networks:
 x-app: &app
   env_file:
     - .env
+  environment:
+    - no_proxy: localhost,web,workers,tika
   build:
     context: .
     args:
@@ -168,3 +170,11 @@ services:
       interval: 30s
       timeout: 20s
       retries: 3
+
+  tika:
+    image: apache/tika:2.6.0.1
+    ports:
+      - 127.0.0.1:9998:9998
+    restart: always
+    networks:
+      internal:

spec/controllers/manifestations_controller_spec.rb

@@ -238,6 +238,7 @@ def valid_attributes
         manifestation = FactoryBot.create(:manifestation, description: "foo")
         periodical.derived_manifestations << manifestation
         periodical.save!
+        manifestation.save!
         get :index, params: { query: "foo" }
         manifestations = assigns(:manifestations)
         expect(manifestations).not_to be_blank

spec/models/loc_search_spec.rb

@@ -10,7 +10,7 @@
       expect(manifestation.original_title).to eq "Everything is miscellaneous : the power of the new digital disorder"
       expect(manifestation.manifestation_content_type.name).to eq "text"
       expect(manifestation.carrier_type.name).to eq "volume"
-      expect(manifestation.publishers.size).to eq 1
+      expect(manifestation.publishers.count).to eq 1
       expect(manifestation.publishers.first.full_name).to eq "Times Books"
       expect(manifestation.publication_place).to eq "New York"
       expect(manifestation.creators.size).to eq 1

spec/models/manifestation_spec.rb

@@ -273,6 +273,12 @@
       expect(manifestations(:manifestation_00101).reservable?).to be_falsy
     end
   end
+
+  it 'should extract fulltext' do
+    manifestation = FactoryBot.create(:manifestation)
+    manifestation.attachment.attach(io: File.open("spec/fixtures/files/resource_import_file_sample1.tsv"), filename: 'sample.txt')
+    expect(manifestation.extract_text).to match(/資料ID/)
+  end
 end
 
 # == Schema Information