Import training documents

Makefile

@@ -10,7 +10,6 @@ RESET := \033[0m
 
 # Commands configuration
 COMPOSE_CMD = COMPOSE_PROJECT_NAME=$(PROJECT_NAME) docker compose -f docker-compose.dev.yml
-DOCKER_TEST_CMD = $(COMPOSE_CMD) exec app bundle exec bash -c "export RAILS_ENV=test && rspec --format documentation"
 EXEC_CMD = $(COMPOSE_CMD) exec app
 
 .PHONY: help build rebuild stop start restart logs shell console format test test_fast db_reset migrate clean clean_volumes
@@ -69,10 +68,10 @@ format:
 	$(EXEC_CMD) bundle exec rubocop --autocorrect-all
 
 test:
-	$(DOCKER_TEST_CMD)
+	$(COMPOSE_CMD) exec app bundle exec bash -c "export RAILS_ENV=test && rspec --format documentation"
 
 test_fast:
-	$(DOCKER_TEST_CMD) --fail-fast
+	$(COMPOSE_CMD) exec app bundle exec bash -c "export RAILS_ENV=test && rspec --format documentation --fail-fast"
 
 migrate:
 	$(EXEC_CMD) bundle exec rails db:migrate

app/controllers/topics_controller.rb

@@ -1,4 +1,5 @@
 class TopicsController < ApplicationController
+  include ActiveStorage::SetCurrent
   include Pagy::Backend
 
   before_action :set_topic, only: [ :show, :edit, :tags, :update, :destroy, :archive ]

app/helpers/topics_helper.rb

@@ -0,0 +1,39 @@
+module TopicsHelper
+  def card_preview_media(file)
+    case file.content_type
+    in /image/ then render_image(file)
+    in /pdf/ then render_pdf(file)
+    in /video/ then render_video(file)
+    in /audio/ then render_audio(file)
+    else render_download_link(file)
+    end
+  end
+
+  private
+
+  def render_image(file)
+    image_tag(file.url, class: "img-fluid w-100")
+  end
+
+  def render_pdf(file)
+    content_tag(:div, class: "embed-responsive embed-responsive-item embed-responsive-16by9 w-100") do
+      content_tag(:object, data: file.url, type: "application/pdf", width: "100%", height: "400px") do
+        content_tag(:iframe, "", src: file.url, width: "100%", height: "100%", style: "border: none;") do
+          content_tag(:p, "Your browser does not support PDF viewing. #{link_to('Download the PDF', file.url)}")
+        end
+      end
+    end
+  end
+
+  def render_video(file)
+    video_tag(file.url, style: "width: 100%")
+  end
+
+  def render_audio(file)
+    audio_tag(file.url, controls: true, style: "width: 100%")
+  end
+
+  def render_download_link(file)
+    link_to file.filename, file.url
+  end
+end

app/models/topic.rb

@@ -26,14 +26,14 @@ class Topic < ApplicationRecord
   include Taggable
 
   STATES = %i[active archived].freeze
-  CONTENT_TYPES = %w[image/jpeg image/png image/svg+xml image/webp image/avif image/gif video/mp4 application/pdf].freeze
+  CONTENT_TYPES = %w[image/jpeg image/png image/svg+xml image/webp image/avif image/gif video/mp4 application/pdf audio/mpeg].freeze
 
   belongs_to :language
   belongs_to :provider
   has_many_attached :documents
 
   validates :title, :language_id, :provider_id, :published_at, presence: true
-  validates :documents, content_type: CONTENT_TYPES, size: { less_than: 10.megabytes }
+  validates :documents, content_type: CONTENT_TYPES, size: { less_than: 200.megabytes }
 
   enum :state, STATES.map.with_index.to_h
 

app/views/topics/_topic.html.erb

@@ -1,49 +1,79 @@
 <div id="<%= dom_id topic %>">
-  <p>
-    <strong>UID:</strong>
-    <%= topic.uid %>
-  </p>
-
-  <p>
-    <strong>Title:</strong>
-    <%= topic.title %>
-  </p>
-
-  <p>
-    <strong>Description:</strong>
-    <%= topic.description %>
-  </p>
-
-  <p>
-    <strong>Provider:</strong>
-    <%= link_to topic.provider.name, topic.provider %>
-  </p>
-
-  <p>
-    <strong>Language:</strong>
-    <%= link_to topic.language.name, topic.language %>
-  </p>
-
-  <p>
-    <strong>Publishing at:</strong>
-    <%= topic.published_at.strftime('%m/%d/%Y') %>
-  </p>
-
-  <div>
-    <p>
-      <strong>Tags:</strong>
-      <% topic.current_tags.each do |tag| %>
-        <%= link_to tag.name, tag_path(tag), class: "badge bg-success", target: "_blank" %>
-      <% end %>
-    </p>
+  <div class="section">
+    <h3 class="mb-4">Topic: <%= topic.id %></h3>
+    <div class="card mb-6">
+      <div class="card-header">
+        <div class="card-title">
+          <h3><%= topic.title %></h3>
+        </div>
+      </div>
+      <div class="card-body">
+        <div class="row mb-2">
+          <div class="col-md-3"><strong>UID:</strong></div>
+          <div class="col-md-9"><%= topic.uid %></div>
+        </div>
+        <div class="row mb-2">
+          <div class="col-md-3"><strong>Description:</strong></div>
+          <div class="col-md-9"><%= topic.description %></div>
+        </div>
+        <div class="row mb-2">
+          <div class="col-md-3"><strong>Provider:</strong></div>
+          <div class="col-md-9"><%= link_to topic.provider.name, topic.provider, class: "text-decoration-none" %></div>
+        </div>
+        <div class="row mb-2">
+          <div class="col-md-3"><strong>Language:</strong></div>
+          <div class="col-md-9"><%= link_to topic.language.name, topic.language, class: "text-decoration-none" %></div>
+        </div>
+        <div class="row mb-2">
+          <div class="col-md-3"><strong>Publishing at:</strong></div>
+          <div class="col-md-9"><%= topic.published_at.strftime('%m/%d/%Y') %></div>
+        </div>
+      </div>
+      <div class="card-footer">
+        <strong>Tags:</strong>
+        <% topic.current_tags.each do |tag| %>
+          <%= link_to tag.name, tag_path(tag), class: "badge bg-success text-decoration-none me-1", target: "_blank" %>
+        <% end %>
+      </div>
+    </div>
   </div>
 
-  <div>
-    <strong>Documents:</strong>
-    <ul>
+  <div class="section">
+    <div class="col-12">
+      <h3 class="mb-4">Documents</h3>
+    </div>
+    <div>
       <% topic.documents.each do |document| %>
-        <li><%= link_to document.filename, rails_blob_path(document), target: "_blank"%></li>
+        <div class="card">
+          <div class="card-content">
+            <div class="card-body">
+              <div class="card-title">
+                <h4><%= document.filename %></h4>
+              </div>
+            </div>
+
+            <%= card_preview_media(document) %>
+
+            <div class="card-body">
+              <div class="d-flex justify-content-between">
+                <div>
+                  <span class="btn btn-sm btn-outline-secondary">
+                    <i class="bi bi-calendar-date"></i>
+                    <%= document.created_at.strftime('%m/%d/%Y') %>
+                  </span>
+                  <span class="btn btn-sm btn-outline-secondary">
+                    <i class="bi bi-clipboard-data"></i>
+                    <%= number_to_human_size(document.byte_size) %>
+                  </span>
+                </div>
+                <%= link_to rails_blob_path(document), target: "_blank", class: "btn btn-primary" do %>
+                  <i class="bi bi-file-arrow-down"></i> Download
+                <% end %>
+              </div>
+            </div>
+          </div>
+        </div>
       <% end %>
-    </ul>
+    </div>
   </div>
 </div>

lib/autorequire/data_import.rb

@@ -26,6 +26,7 @@ def self.import_all
     import_topics
     import_tags
     import_topic_tags
+    import_training_documents
     restore_default_users
   end
 
@@ -184,4 +185,163 @@ def self.restore_default_users
 
     Provider.first.users << me unless Provider.first.users.include?(me)
   end
+
+  def self.import_training_documents
+    csv_data = load_training_documents_csv
+    import_stats = initialize_import_stats
+
+    valid_csv_rows = filter_rows_with_existing_topics(csv_data, import_stats)
+    azure_files = fetch_azure_files
+    importable_rows = match_csv_with_azure_files(valid_csv_rows, azure_files)
+
+    log_import_summary(valid_csv_rows, azure_files, importable_rows)
+
+    process_document_attachments(importable_rows, import_stats)
+    log_final_results(import_stats)
+  end
+
+  private
+
+  def self.load_training_documents_csv
+    CSV.read(file_path("CMEFiles.csv"), headers: true)
+  end
+
+  def self.initialize_import_stats
+    {
+      topics_without_csv: [],
+      successful_attachments: [],
+      failed_attachments: [],
+      error_files: [],
+    }
+  end
+
+  def self.filter_rows_with_existing_topics(csv_data, stats)
+    csv_data.filter_map do |row|
+      topic_id = row["Topic_ID"].to_i
+      if Topic.find_by(id: topic_id)
+        row
+      else
+        stats[:topics_without_csv] << topic_id
+        nil
+      end
+    end
+  end
+
+  def self.match_csv_with_azure_files(csv_rows, azure_files)
+    azure_files.filter_map do |file|
+      csv_rows.find { |row| row["File_Name"] == file[:name] }
+    end
+  end
+
+  def self.process_document_attachments(rows, stats)
+    rows.each do |row|
+      topic = Topic.find_by(id: row["Topic_ID"])
+      next unless topic
+
+      attach_document_to_topic(topic, row, stats)
+    end
+  end
+
+  def self.attach_document_to_topic(topic, row, stats)
+    file_path = get_file_path(topic.state, topic.language.name)
+    filename = row["File_Name"]
+
+    puts "Requesting: #{file_path}/#{filename}"
+
+    begin
+      file_content = download_azure_file(file_path, filename)
+
+      topic.documents.attach(
+        io: StringIO.new(file_content),
+        filename: filename,
+        content_type: detect_content_type(row["File_Type"])
+      )
+
+      if topic.save!
+        stats[:successful_attachments] << [ row, topic ]
+      else
+        stats[:failed_attachments] << [ row, topic ]
+      end
+
+    rescue AzureFileShares::Errors::ApiError, URI::InvalidURIError => e
+      handle_attachment_error(topic, filename, e, stats)
+    end
+  end
+
+  def self.download_azure_file(file_path, filename)
+    encoded_filename = URI.encode_www_form_component(filename)
+    AzureFileShares.client.files.download_file(
+      ENV["AZURE_STORAGE_SHARE_NAME"],
+      file_path,
+      encoded_filename
+    )
+  end
+
+  def self.handle_attachment_error(topic, filename, error, stats)
+    error_info = {
+      topic: topic,
+      file: filename,
+      error: error.message,
+    }
+    stats[:error_files] << error_info
+    puts "Error with file: #{filename} for topic #{topic.title} - #{error.message}"
+  end
+
+  def self.log_import_summary(csv_rows, azure_files, importable_rows)
+    puts "CSV rows with topics: #{csv_rows.size}"
+    puts "Azure files found: #{azure_files.size}"
+    puts "Importable files: #{importable_rows.size}"
+  end
+
+  def self.log_final_results(stats)
+    puts "Topics not found: #{stats[:topics_without_csv].size}"
+    puts "Successful attachments: #{stats[:successful_attachments].size}"
+    puts "Failed attachments: #{stats[:failed_attachments].size}"
+    puts "Files with errors: #{stats[:error_files].size}"
+  end
+
+  private
+
+  def self.get_file_path(state, language)
+    case [ state, language ]
+    in [ "active", "english" ]
+      "CMES-Pi/assets/Content"
+    in [ "archived", "english" ]
+      "CMES-Pi_Archive"
+    in [ "active", "spanish" ]
+      "SP_CMES-Pi/assets/Content"
+    in [ "archived", "spanish" ]
+      "SP_CMES-Pi_Archive"
+    end
+  end
+
+  def self.fetch_azure_files
+    client = AzureFileShares.client
+    azure_active_en = client.files.list(ENV["AZURE_STORAGE_SHARE_NAME"], self.get_file_path("active", "english"))
+    azure_active_es = client.files.list(ENV["AZURE_STORAGE_SHARE_NAME"], self.get_file_path("active", "spanish"))
+    azure_archived_en = client.files.list(ENV["AZURE_STORAGE_SHARE_NAME"], self.get_file_path("archived", "english"))
+    azure_archived_es = client.files.list(ENV["AZURE_STORAGE_SHARE_NAME"], self.get_file_path("archived", "spanish"))
+
+    [
+      azure_active_en[:files],
+      azure_active_es[:files],
+      azure_archived_en[:files],
+      azure_archived_es[:files],
+    ].flatten
+  end
+
+  def self.detect_content_type(filename)
+    case File.extname(filename).downcase
+    when ".mp3"
+      "audio/mpeg"
+    when ".pdf"
+      "application/pdf"
+    when ".jpg", ".jpeg"
+      "image/jpeg"
+    when ".png"
+      "image/png"
+    else
+      "application/octet-stream"
+    end
+  end
 end