Merge pull request #1431 from indentlabs/claude/audit-document-analysis-gJPFq

Document analysis general bugfixes

Author: drusepth

app/controllers/document_analyses_controller.rb

@@ -40,7 +40,11 @@ def hub
   end
 
   def index
-    @document_analyses = DocumentAnalysis.all
+    if user_signed_in?
+      @document_analyses = DocumentAnalysis.joins(:document).where(documents: { user_id: current_user.id })
+    else
+      @document_analyses = DocumentAnalysis.joins(:document).where(documents: { privacy: 'public' })
+    end
   end
 
   def show
@@ -57,13 +61,18 @@ def style
   # end
 
   def sentiment
-    @document_sentiment_color = (@analysis.sentiment_score < 0) ? 'blue' : 'green'
+    unless @analysis&.has_sentiment_scores?
+      redirect_to analysis_document_path(@document), notice: "Sentiment data is not yet available for this document."
+      return
+    end
+
+    @document_sentiment_color = (@analysis.sentiment_score.to_f < 0) ? 'blue' : 'green'
     @document_emotion_data = Hash[{
-      "Anger"   => (100 * @analysis.anger_score).round(1),
-      "Fear"    => (100 * @analysis.fear_score).round(1),
-      "Sadness" => (100 * @analysis.sadness_score).round(1),
-      "Disgust" => (100 * @analysis.disgust_score).round(1),
-      "Joy"     => (100 * @analysis.joy_score).round(1)
+      "Anger"   => (100 * (@analysis.anger_score || 0)).round(1),
+      "Fear"    => (100 * (@analysis.fear_score || 0)).round(1),
+      "Sadness" => (100 * (@analysis.sadness_score || 0)).round(1),
+      "Disgust" => (100 * (@analysis.disgust_score || 0)).round(1),
+      "Joy"     => (100 * (@analysis.joy_score || 0)).round(1)
     }.sort_by(&:second).reverse]
     @document_dominant_emotion  = @document_emotion_data.keys.first
     @document_secondary_emotion = @document_emotion_data.keys.second

app/jobs/document_mention_job.rb

@@ -21,7 +21,8 @@ def perform(*args)
         # by the document owner, or else people could add arbitrary pages to quick reference
         # to view them.
         next unless Rails.application.config.content_types[:all].map(&:name).include?(entity_type)
-        related_page = entity_type.constantize.find(id)
+        related_page = entity_type.constantize.find_by(id: id)
+        next unless related_page.present?
         # todo we could also work off privacy here, so people could add public pages to quick ference
         # -- we'd have to delete those quick-references whenever a page went private though
         next unless related_page.user_id == document.user_id

app/models/concerns/has_parts_of_speech.rb

@@ -75,7 +75,7 @@ def interrogatives
     end
 
     def numbers
-      @numbers ||= text.strip
+      @numbers ||= plaintext.strip
         .split(' ')
         .select { |w| is_numeric?(w) }
         .uniq

app/models/documents/document_analysis.rb

@@ -14,6 +14,7 @@ class DocumentAnalysis < ApplicationRecord
 
   # usage: analysis.pos_percentage(:adjective) => 23.4
   def pos_percentage(pos_symbol)
+    return 0.0 if word_count.nil? || word_count == 0
     (send(pos_symbol.to_s + '_count').to_f / word_count * 100).round(2)
   end
 

app/models/documents/document_entity.rb

@@ -53,15 +53,15 @@ def linked_name_if_possible
   end
 
   def dominant_emotion
-    return { unknown: 1 } if emotions.values.uniq == [0]
+    return [[:unknown, 1]] if emotions.values.compact.empty? || emotions.values.uniq == [0]
 
-    emotions.sort_by { |emotion, score| score }.reverse
+    emotions.sort_by { |emotion, score| score.to_f }.reverse
   end
 
   def recessive_emotion
-    return { unknown: 1 } if emotions.values.uniq == [0]
+    return [[:unknown, 1]] if emotions.values.compact.empty? || emotions.values.uniq == [0]
 
-    emotions.sort_by { |emotion, score| score }
+    emotions.sort_by { |emotion, score| score.to_f }
   end
 
   def emotions

app/services/documents/analysis/content_service.rb

@@ -22,7 +22,7 @@ def self.analyze(analysis_id)
         analysis.save!
       end
 
-      def self.adult_content?(matchlist=:hate, content)
+      def self.adult_content?(content, matchlist: :hate)
         LanguageFilter::Filter.new(matchlist: matchlist.to_sym).matched(content)
       end
     end

app/services/documents/analysis/flesch_kincaid_service.rb

@@ -2,15 +2,16 @@ module Documents
   module Analysis
     class FleschKincaidService < Service
       def self.grade_level(document)
-        @flesch_kincaid_grade_level ||= [
+        return 0 if document.words.length == 0 || document.sentences.length == 0
+        [
           0.38 * (document.words.length.to_f / document.sentences.length),
           11.18 * (document.word_syllables.sum.to_f / document.words.length),
           -15.59
         ].sum.clamp(0, 16)
       end
 
       def self.age_minimum(document)
-        @flesch_kincaid_age_minimum ||= case reading_ease(document)
+        case reading_ease(document)
           when (90..100)  then 11
           when (71..89)   then 12
           when (67..69)   then 13
@@ -26,7 +27,8 @@ def self.age_minimum(document)
       end
 
       def self.reading_ease(document)
-        @flesch_kincaid_reading_ease ||= [
+        return 0 if document.words.length == 0 || document.sentences.length == 0
+        [
           206.835,
           -(1.015 * document.words.length.to_f / document.sentences.length),
           -(84.6 * document.word_syllables.sum.to_f / document.words.length)

app/services/documents/analysis/readability_service.rb

@@ -26,19 +26,22 @@ def self.analyze(analysis_id)
       end
 
       def self.forcast_grade_level(document)
-        @forcast_grade_level ||= 20 - (((document.words_with_syllables(1).length.to_f / document.words.length) * 150) / 10.0).clamp(1, 16)
+        return 1 if document.words.length == 0
+        (20 - (((document.words_with_syllables(1).length.to_f / document.words.length) * 150) / 10.0)).clamp(1, 16)
       end
 
       def self.coleman_liau_index(document)
-        @coleman_liau_index ||= [
+        return 1 if document.words.length == 0 || document.sentences.length == 0
+        [
           0.0588 * 100 * document.characters.reject { |l| [" ", "\t", "\r", "\n"].include?(l) }.length.to_f / document.words.length,
           -0.296 * 100/ (document.words.length.to_f / document.sentences.length),
           -15.8
         ].sum.clamp(1, 16)
       end
 
       def self.automated_readability_index(document)
-        @automated_readability_index ||= [
+        return 1 if document.words.length == 0 || document.sentences.length == 0
+        [
           4.71 * document.characters.reject(&:blank?).length.to_f / document.words.length,
           0.5 * document.words.length.to_f / document.sentences.length,
           -21.43
@@ -47,19 +50,21 @@ def self.automated_readability_index(document)
 
       def self.gunning_fog_index(document)
         #todo GFI word/suffix exclusions
-        @gunning_fog_index ||= 0.4 * (document.words.length.to_f/ document.sentences.length + 100 * (document.complex_words.length.to_f / document.words.length)).clamp(1, 16)
+        return 1 if document.words.length == 0 || document.sentences.length == 0
+        (0.4 * (document.words.length.to_f/ document.sentences.length + 100 * (document.complex_words.length.to_f / document.words.length))).clamp(1, 16)
       end
 
       def self.smog_grade(document)
-        @smog_grade ||= (1.043 * Math.sqrt(document.complex_words.length.to_f * (30.0 / document.sentences.length)) + 3.1291).clamp(1, 16)
+        return 1 if document.sentences.length == 0
+        (1.043 * Math.sqrt(document.complex_words.length.to_f * (30.0 / document.sentences.length)) + 3.1291).clamp(1, 16)
       end
 
       def self.linsear_write_grade(document)
-        @linsear_write_grade ||= TextStat.linsear_write_formula(document.plaintext).clamp(1, 16)
+        TextStat.linsear_write_formula(document.plaintext).clamp(1, 16)
       end
 
       def self.dale_chall_grade(document)
-        @dale_chall_grade ||= TextStat.dale_chall_readability_score(document.plaintext).clamp(1, 10)
+        TextStat.dale_chall_readability_score(document.plaintext).clamp(1, 10)
       end
 
       # deprecated in favor of TextStat.text_standard(text, float_output=False)

app/services/documents/analysis/syllables_service.rb

@@ -6,13 +6,14 @@ class SyllablesService < Service
       }
 
       def self.count(word)
-        word.downcase.gsub!(/[^a-z]/, '')
+        word = word.downcase.gsub(/[^a-z]/, '')
 
         return 1 if word.length <= 3
         return SYLLABLE_COUNT_OVERRIDES[word] if SYLLABLE_COUNT_OVERRIDES.key?(word)
 
-        word.sub(/(?:[^laeiouy]es|ed|[^laeiouy]e)$/, '').sub!(/^y/, '')
-        word.scan(/[aeiouy]{1,2}/).length
+        word = word.sub(/(?:[^laeiouy]es|ed|[^laeiouy]e)$/, '').sub(/^y/, '')
+        count = word.scan(/[aeiouy]{1,2}/).length
+        count > 0 ? count : 1
       end
     end
   end

test/fixtures/document_analyses.yml

@@ -2,60 +2,89 @@
 
 one:
   document: one
-  word_count: 1
+  word_count: 65
   page_count: 1
-  paragraph_count: 1
-  character_count: 1
-  readability_score: 1
-  combined_average_reading_level: 1.5
-  flesch_kincaid_grade_level: 1
-  flesch_kincaid_age_minimum: 1
-  flesch_kincaid_reading_ease: 1.5
-  forcast_grade_level: 1.5
-  coleman_liau_index: 1.5
-  automated_readability_index: 1.5
-  gunning_fog_index: 1.5
-  adjective_count: 1
-  noun_count: 1
-  verb_count: 1
-  pronoun_count: 1
-  preposition_count: 1
-  conjunction_count: 1
-  adverb_count: 1
-  determiner_count: 1
-  n_syllable_words: 
-  words_used_once_count: 1
-  words_used_repeatedly_count: 1
-  simple_words_count: 1
-  complex_words_count: 1
-  words_per_sentence: 
+  paragraph_count: 2
+  character_count: 350
+  sentence_count: 3
+  readability_score: 55
+  combined_average_reading_level: 8.5
+  flesch_kincaid_grade_level: 8
+  flesch_kincaid_age_minimum: 13
+  flesch_kincaid_reading_ease: 65.0
+  forcast_grade_level: 9.5
+  coleman_liau_index: 7.5
+  automated_readability_index: 8.0
+  gunning_fog_index: 10.5
+  smog_grade: 9.0
+  linsear_write_grade: 7.0
+  dale_chall_grade: 6.5
+  adjective_count: 5
+  noun_count: 15
+  verb_count: 12
+  pronoun_count: 4
+  preposition_count: 6
+  conjunction_count: 3
+  adverb_count: 2
+  determiner_count: 8
+  proper_noun_count: 1
+  interrogative_count: 0
+  n_syllable_words: {"1": 40, "2": 15, "3": 8, "4": 2}
+  words_used_once_count: 45
+  words_used_repeatedly_count: 20
+  simple_words_count: 50
+  complex_words_count: 15
+  unique_simple_words_count: 40
+  unique_complex_words_count: 12
+  words_per_sentence: [20, 25, 20]
+  sentiment_score: 0.25
+  sentiment_label: "positive"
+  language: "en"
+  joy_score: 0.65
+  sadness_score: 0.12
+  fear_score: 0.05
+  disgust_score: 0.03
+  anger_score: 0.08
+  completed_at: "2026-03-01 12:00:00"
+  queued_at: "2026-03-01 11:55:00"
+  progress: 100
 
 two:
   document: two
-  word_count: 1
+  word_count: 9
   page_count: 1
   paragraph_count: 1
-  character_count: 1
-  readability_score: 1
-  combined_average_reading_level: 1.5
-  flesch_kincaid_grade_level: 1
-  flesch_kincaid_age_minimum: 1
-  flesch_kincaid_reading_ease: 1.5
-  forcast_grade_level: 1.5
-  coleman_liau_index: 1.5
-  automated_readability_index: 1.5
-  gunning_fog_index: 1.5
-  adjective_count: 1
-  noun_count: 1
+  character_count: 50
+  sentence_count: 1
+  readability_score: 80
+  combined_average_reading_level: 3.0
+  flesch_kincaid_grade_level: 3
+  flesch_kincaid_age_minimum: 11
+  flesch_kincaid_reading_ease: 90.0
+  forcast_grade_level: 4.0
+  coleman_liau_index: 3.5
+  automated_readability_index: 3.0
+  gunning_fog_index: 4.0
+  smog_grade: 3.0
+  linsear_write_grade: 3.0
+  dale_chall_grade: 3.0
+  adjective_count: 2
+  noun_count: 3
   verb_count: 1
-  pronoun_count: 1
+  pronoun_count: 0
   preposition_count: 1
-  conjunction_count: 1
-  adverb_count: 1
+  conjunction_count: 0
+  adverb_count: 0
   determiner_count: 1
-  n_syllable_words: 
-  words_used_once_count: 1
-  words_used_repeatedly_count: 1
-  simple_words_count: 1
-  complex_words_count: 1
-  words_per_sentence: 
+  proper_noun_count: 0
+  interrogative_count: 0
+  n_syllable_words: {"1": 7, "2": 2}
+  words_used_once_count: 9
+  words_used_repeatedly_count: 0
+  simple_words_count: 9
+  complex_words_count: 0
+  unique_simple_words_count: 9
+  unique_complex_words_count: 0
+  words_per_sentence: [9]
+  progress: 100
+  queued_at: "2026-03-01 11:55:00"