[E] Enhance ingestion messaging when source paths are malformed

* Surface ingestion severity properly
* Add a gitignore to handle new zips in the spec/data directory
  without `git add -f`
* Update rake ingestion tasks to better log failures when they occur

Author: scryptmouse

api/app/jobs/ingestions/log_message_job.rb

@@ -2,8 +2,8 @@
 
 module Ingestions
   class LogMessageJob < AsyncApplicationJob
-    def perform(ingestion_id:, kind:, payload:)
-      IngestionMessage.create!(ingestion_id:, kind:, payload:)
+    def perform(ingestion_id:, kind:, payload:, severity: "unknown")
+      IngestionMessage.create!(ingestion_id:, kind:, payload:, severity:)
     end
   end
 end

api/app/models/ingestion.rb

@@ -20,7 +20,7 @@ class Ingestion < ApplicationRecord
   belongs_to :text, optional: true
   belongs_to :text_section, optional: true
   belongs_to :project
-  has_many :ingestion_messages, -> { reorder(created_at: :asc) }, inverse_of: :ingestion, dependent: :delete_all
+  has_many :ingestion_messages, -> { in_default_order }, inverse_of: :ingestion, dependent: :delete_all
 
   # Validations
   validates :source, presence: true, if: :file_based_ingestion?
@@ -131,7 +131,8 @@ def add(severity, message = nil, _progname = nil)
     ::Ingestions::LogMessageJob.perform_later(
       ingestion_id: id,
       kind: "log",
-      payload: line
+      payload: line,
+      severity: severity.downcase,
     )
   end
 
@@ -144,37 +145,59 @@ def commit_log
     self.log_buffer = []
   end
 
+  # @param [User] user
+  # @return [void]
   def begin_processing(user)
     update_column :processing_failed, false
 
     target_kind.begin_processing(user, self)
   end
 
+  # @param [StandardError, Ingestions::IngestionError, ActiveModel::Errors] errors
+  # @return [void]
   def handle_ingestion_exception(errors)
     update_column :processing_failed, true
 
     error("Processing failed.\n")
 
-    if errors.respond_to?(:full_messages)
-      output_errors(errors)
-    else
-      compose_and_output_backtrace(errors)
-    end
+    handle_ingestion_errors!(errors)
 
     processing_failure
   end
 
   private
 
+  # @param [StandardError, Ingestions::IngestionError, ActiveModel::Errors] errors
+  # @return [void]
+  def handle_ingestion_errors!(errors)
+    if errors.respond_to?(:full_messages)
+      output_errors(errors)
+    else
+      case errors
+      when StandardError
+        compose_and_output_backtrace(errors)
+      else
+        # :nocov:
+        fatal("Something went wrong with ingestion: #{errors.inspect}")
+        # :nocov:
+      end
+    end
+  end
+
+  # @param [StandardError, Ingestions::IngestionError] errors
+  # @return [void]
   def compose_and_output_backtrace(errors)
     output = errors.message
+
     Rails.backtrace_cleaner.clean(errors.backtrace).each do |line|
       output += "\n#{line}"
     end
 
     error(output)
   end
 
+  # @param [ActiveModel::Errors] errors
+  # @return [void]
   def output_errors(errors)
     errors.full_messages.each do |e|
       error(e)

api/app/models/ingestion_message.rb

@@ -2,6 +2,37 @@
 
 # A message stored during resource ingestion to be later returned to the client
 class IngestionMessage < ApplicationRecord
-  belongs_to :ingestion
+  LOG_LEVELS = %w[unknown debug info warn error fatal].freeze
+
+  LOG_LEVEL_ENUM = LOG_LEVELS.to_h { |level| [level, level] }.freeze
+
+  enum :severity, LOG_LEVEL_ENUM, suffix: :severity, allow_blank: false, default: "unknown"
+
+  belongs_to :ingestion, inverse_of: :ingestion_messages
+
+  scope :in_default_order, -> { reorder(created_at: :asc) }
+
+  scope :logs, -> { where(kind: "log") }
+  scope :log_message_matches, ->(needle) { logs.where(arel_log_message_matches(needle)) }
+
+  before_validation :normalize_severity!
+
   validates :kind, :payload, presence: true
+
+  private
+
+  # @return [void]
+  def normalize_severity!
+    self.severity = severity.to_s.downcase.presence_in(LOG_LEVELS) || "unknown"
+  end
+
+  class << self
+    # @param [String] needle
+    # @return [Arel::Nodes::Matches]
+    def arel_log_message_matches(needle)
+      ingestion_messages = IngestionMessage.arel_table
+
+      arel_named_fn("extract_ingestion_message_text", ingestion_messages[:payload]).matches(arel_quote(needle), nil, false)
+    end
+  end
 end

api/app/serializers/v1/ingestion_message_serializer.rb

@@ -6,6 +6,7 @@ class IngestionMessageSerializer < ManifoldSerializer
 
     typed_attribute :kind, Types::String.meta(read_only: true)
     typed_attribute :payload, Types::Any.meta(read_only: true)
+    typed_attribute :severity, Types::Any.meta(read_only: true)
     typed_attribute :id, Types::Serializer::ID
     typed_attribute :created_at, Types::String.meta(read_only: true)
   end

api/app/services/ingestions/strategies/manifest.rb

@@ -44,6 +44,17 @@ def preprocess
           rel_path = context.rel_path_without_ext source
           context.write_build_file "#{rel_path}.html", raw_html
         end
+
+        # This block is only called when there are missing source paths.
+        inspector.validate_local_sources! do |missing|
+          error "services.ingestions.pre_processor.manifest.missing_source_paths"
+
+          missing.each do |source_path|
+            error "services.ingestions.pre_processor.manifest.missing_source_path", source_path: source_path
+          end
+
+          raise Ingestions::IngestionError, "Manifest pre-processing failed: missing source paths"
+        end
       end
 
       def manifest

api/app/services/ingestions/strategy/manifest/inspector.rb

@@ -103,6 +103,30 @@ def external_sources
           source_map.select { |source| context.url? source["source_path"] }
         end
 
+        def local_sources
+          source_map.reject { |source| context.url? source["source_path"] }
+        end
+
+        def local_source_paths
+          local_sources.pluck("source_path")
+        end
+
+        # @yield [missing]
+        # @yieldparam [Array<String>] missing list of missing local source paths
+        # @yieldreturn [void]
+        # @return [Array<String>] list of missing local source paths
+        def validate_local_sources!
+          source_root_path = Pathname(context.source_root)
+
+          missing = local_source_paths.reject do |source_path|
+            source_root_path.join(source_path).exist?
+          end
+
+          yield missing if block_given? && missing.any?
+
+          return missing
+        end
+
         def update_source_map(original_path, new_path)
           item = source_map.detect do |source|
             source["source_path"] == original_path

api/config/locales/services/ingestions/en.yml

@@ -48,6 +48,10 @@ en:
           landmark_nav_title: 'Landmark nav title is "%{text}"'
           toc_nav_title: 'TOC nav title is "%{text}"'
           page_list_nav_title: 'Page list nav title is "%{text}"'
+      pre_processor:
+        manifest:
+          missing_source_paths: "One or more source paths were defined by the manifest but not found. Check spelling and capitalization."
+          missing_source_path: "Manifest could not find an expected source path: %{source_path}"
       post_processor:
         log:
           transform_toc_structure: 'Transforming TOC structure'

api/db/migrate/20251020225421_add_severity_to_ingestion_messages.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+class AddSeverityToIngestionMessages < ActiveRecord::Migration[7.0]
+  def change
+    create_enum :ingestion_message_severity, %w[
+      unknown debug info warn error fatal
+    ]
+
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL
+        CREATE FUNCTION public.normalize_ingestion_message_severity(text) RETURNS public.ingestion_message_severity AS $$
+        SELECT
+        CASE LOWER($1)
+        WHEN 'debug' THEN 'debug'::public.ingestion_message_severity
+        WHEN 'info' THEN 'info'::public.ingestion_message_severity
+        WHEN 'warn' THEN 'warn'::public.ingestion_message_severity
+        WHEN 'error' THEN 'error'::public.ingestion_message_severity
+        WHEN 'fatal' THEN 'fatal'::public.ingestion_message_severity
+        ELSE 'unknown'::public.ingestion_message_severity
+        END;
+        $$ LANGUAGE SQL IMMUTABLE CALLED ON NULL INPUT PARALLEL SAFE;
+
+        CREATE FUNCTION public.extract_ingestion_message_severity(jsonb) RETURNS public.ingestion_message_severity AS $$
+        SELECT
+        CASE
+        WHEN jsonb_typeof($1) = 'array' THEN public.normalize_ingestion_message_severity($1->>0)
+        ELSE 'unknown'::public.ingestion_message_severity
+        END;
+        $$ LANGUAGE SQL IMMUTABLE CALLED ON NULL INPUT PARALLEL SAFE;
+
+        CREATE FUNCTION public.extract_ingestion_message_text(jsonb) RETURNS text AS $$
+        SELECT
+        CASE
+        WHEN jsonb_typeof($1) = 'array' THEN $1->>1
+        ELSE NULL
+        END;
+        $$ LANGUAGE SQL IMMUTABLE CALLED ON NULL INPUT PARALLEL SAFE;
+        SQL
+      end
+
+      dir.down do
+        execute <<~SQL
+        DROP FUNCTION public.extract_ingestion_message_text(jsonb);
+        DROP FUNCTION public.extract_ingestion_message_severity(jsonb);
+        DROP FUNCTION public.normalize_ingestion_message_severity(text);
+        SQL
+      end
+    end
+
+    change_table :ingestion_messages do |t|
+      t.enum :severity, enum_type: :ingestion_message_severity, null: false, default: "unknown"
+
+      t.index %[(extract_ingestion_message_text(payload))],
+        name: "index_ingestion_messages_on_extracted_text",
+        where: %[kind = 'log']
+    end
+
+    reversible do |dir|
+      dir.up do
+        say_with_time "Migrating ingestion message severities" do
+          exec_update(<<~SQL, "Migrate severities")
+          UPDATE ingestion_messages
+          SET severity = public.extract_ingestion_message_severity(payload)
+          WHERE kind = 'log';
+          SQL
+        end
+      end
+    end
+  end
+end

api/db/structure.sql

@@ -135,6 +135,20 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA public;
 COMMENT ON EXTENSION "uuid-ossp" IS 'generate universally unique identifiers (UUIDs)';
 
 
+--
+-- Name: ingestion_message_severity; Type: TYPE; Schema: public; Owner: -
+--
+
+CREATE TYPE public.ingestion_message_severity AS ENUM (
+    'unknown',
+    'debug',
+    'info',
+    'warn',
+    'error',
+    'fatal'
+);
+
+
 --
 -- Name: manifold_lang; Type: TYPE; Schema: public; Owner: -
 --
@@ -145,6 +159,36 @@ CREATE TYPE public.manifold_lang AS ENUM (
 );
 
 
+--
+-- Name: extract_ingestion_message_severity(jsonb); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.extract_ingestion_message_severity(jsonb) RETURNS public.ingestion_message_severity
+    LANGUAGE sql IMMUTABLE PARALLEL SAFE
+    AS $_$
+SELECT
+CASE
+WHEN jsonb_typeof($1) = 'array' THEN public.normalize_ingestion_message_severity($1->>0)
+ELSE 'unknown'::public.ingestion_message_severity
+END;
+$_$;
+
+
+--
+-- Name: extract_ingestion_message_text(jsonb); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.extract_ingestion_message_text(jsonb) RETURNS text
+    LANGUAGE sql IMMUTABLE PARALLEL SAFE
+    AS $_$
+SELECT
+CASE
+WHEN jsonb_typeof($1) = 'array' THEN $1->>1
+ELSE NULL
+END;
+$_$;
+
+
 --
 -- Name: extract_text_section_content(jsonb); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -258,6 +302,25 @@ CREATE FUNCTION public.manifold_slugify(text) RETURNS text
 $_$;
 
 
+--
+-- Name: normalize_ingestion_message_severity(text); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.normalize_ingestion_message_severity(text) RETURNS public.ingestion_message_severity
+    LANGUAGE sql IMMUTABLE PARALLEL SAFE
+    AS $_$
+SELECT
+CASE LOWER($1)
+WHEN 'debug' THEN 'debug'::public.ingestion_message_severity
+WHEN 'info' THEN 'info'::public.ingestion_message_severity
+WHEN 'warn' THEN 'warn'::public.ingestion_message_severity
+WHEN 'error' THEN 'error'::public.ingestion_message_severity
+WHEN 'fatal' THEN 'fatal'::public.ingestion_message_severity
+ELSE 'unknown'::public.ingestion_message_severity
+END;
+$_$;
+
+
 --
 -- Name: to_unaccented_tsv(jsonb); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -1674,7 +1737,8 @@ CREATE TABLE public.ingestion_messages (
     kind text NOT NULL,
     payload jsonb NOT NULL,
     created_at timestamp(6) without time zone NOT NULL,
-    updated_at timestamp(6) without time zone NOT NULL
+    updated_at timestamp(6) without time zone NOT NULL,
+    severity public.ingestion_message_severity DEFAULT 'unknown'::public.ingestion_message_severity NOT NULL
 );
 
 
@@ -4929,6 +4993,13 @@ CREATE INDEX index_import_selections_on_source_text_id ON public.import_selectio
 CREATE INDEX index_import_selections_on_text_id ON public.import_selections USING btree (text_id);
 
 
+--
+-- Name: index_ingestion_messages_on_extracted_text; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_ingestion_messages_on_extracted_text ON public.ingestion_messages USING btree (public.extract_ingestion_message_text(payload)) WHERE (kind = 'log'::text);
+
+
 --
 -- Name: index_ingestion_messages_on_ingestion_id; Type: INDEX; Schema: public; Owner: -
 --
@@ -7770,6 +7841,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20250609192241'),
 ('20251016204352'),
 ('20251017174417'),
-('20251017211501');
+('20251017211501'),
+('20251020225421');