Further improve performance of exercise statistics

This commit mainly improves the handling and determination of the deadline status (by preloading the records, using a dedicated SQL query).

Author: MrSerth

app/controllers/exercises_controller.rb

@@ -497,9 +497,10 @@ def statistics
     contributor_statistics = {InternalUser => {}, ExternalUser => {}, ProgrammingGroup => {}}
 
     query = SubmissionPolicy::DeadlineScope.new(current_user, Submission).resolve
-      .select('contributor_id, contributor_type, MAX(score) AS maximum_score, COUNT(id) AS runs')
+      .select("contributor_id, contributor_type, MAX(score) AS maximum_score, COUNT(id) AS runs, MAX(updated_at) FILTER (WHERE cause IN ('submit', 'assess', 'remoteSubmit', 'remoteAssess')) AS updated_at, exercise_id")
       .where(exercise_id: @exercise.id)
-      .group('contributor_id, contributor_type')
+      .group('contributor_id, contributor_type, exercise_id')
+      .includes(:contributor, :exercise)
 
     query.each do |tuple|
       contributor_statistics[tuple.contributor_type.constantize][tuple.contributor] = tuple

app/controllers/external_users_controller.rb

@@ -32,12 +32,14 @@ def working_time_query(tag = nil)
            bar.exercise_id,
            max(score) as maximum_score,
            count(bar.id) as runs,
-           sum(working_time_new) AS working_time
+           sum(working_time_new) AS working_time,
+           max_updated_at as updated_at
     FROM
       (SELECT contributor_id,
               exercise_id,
               score,
               id,
+              max_updated_at,
               CASE
                   WHEN #{StatisticsHelper.working_time_larger_delta} THEN '0'
                   ELSE working_time
@@ -46,7 +48,9 @@ def working_time_query(tag = nil)
          (SELECT contributor_id,
                  exercise_id,
                  max(score) AS score,
+                 max(filtered_submissions.updated_at) FILTER (WHERE cause IN ('submit', 'assess', 'remoteSubmit', 'remoteAssess')) AS max_updated_at,
                  filtered_submissions.id,
+                 filtered_submissions.updated_at,
                  (filtered_submissions.updated_at - lag(filtered_submissions.updated_at) over (PARTITION BY contributor_id, exercise_id
                                                      ORDER BY filtered_submissions.updated_at)) AS working_time
           FROM filtered_submissions
@@ -61,7 +65,8 @@ def working_time_query(tag = nil)
       ) AS bar
     #{tag.nil? ? '' : " JOIN exercise_tags et ON et.exercise_id = bar.exercise_id AND #{ExternalUser.sanitize_sql(['et.tag_id = ?', tag])}"}
     GROUP BY contributor_id,
-             bar.exercise_id;
+             bar.exercise_id,
+             max_updated_at;
     "
   end
 
@@ -75,12 +80,13 @@ def statistics
 
     statistics = {}
 
-    working_time_statistics = ApplicationRecord.connection.exec_query(working_time_query(tag&.id))
-    attempted_exercises = Exercise.where(id: working_time_statistics.pluck('exercise_id'))
+    # We fake the statistics hash to be "submissions"
+    # Available are: contributor_id, exercise_id, maximum_score, runs, working_time, updated_at
+    working_time_statistics = Submission.find_by_sql(working_time_query(tag&.id))
+    ActiveRecord::Associations::Preloader.new(records: working_time_statistics, associations: [:exercise]).call
     working_time_statistics.each do |tuple|
-      tuple = tuple.merge('working_time' => format_time_difference(tuple['working_time']))
-      exercise = attempted_exercises.find {|attempted_exercise| attempted_exercise.id == tuple['exercise_id'] }
-      statistics[exercise] = tuple
+      tuple['working_time'] = format_time_difference(tuple['working_time'])
+      statistics[tuple.exercise] = tuple
     end
 
     render locals: {

app/views/exercises/statistics.html.slim

@@ -80,13 +80,12 @@ h1 = @exercise
               td = link_to_if user_type == ExternalUser && policy(contributor).statistics?, contributor.displayname, statistics_external_user_exercise_path(contributor, @exercise)
               td = submission_stat['maximum_score'] || '0.0'
               td.align-middle
-                - latest_user_submission = @exercise.final_submission(contributor)
-                - if latest_user_submission.present?
-                  - if latest_user_submission.before_deadline?
+                - if submission_stat.updated_at.present?
+                  - if submission_stat.before_deadline?
                     .deadline-result.positive-result
-                  - elsif latest_user_submission.within_grace_period?
+                  - elsif submission_stat.within_grace_period?
                     .deadline-result.unknown-result
-                  - elsif latest_user_submission.after_late_deadline?
+                  - elsif submission_stat.after_late_deadline?
                     .deadline-result.negative-result
               td = submission_stat['runs'] if policy(@exercise).detailed_statistics?
               td = @exercise.average_working_time_for(contributor) || '00:00:00' if policy(@exercise).detailed_statistics?

app/views/external_users/statistics.html.slim

@@ -11,23 +11,19 @@ h1 = t('.title', user: @user.displayname)
           th.header = t('.runs') if policy(statistics.keys.first).detailed_statistics?
           th.header = t('.worktime') if policy(statistics.keys.first).detailed_statistics?
       tbody
-        - statistics.each do |exercise, stats|
-          // Grab any submission in context of study group (or all if admin). Then check for permission
-          - any_submission = @user.submissions.where(exercise:).first
-          - if any_submission && policy(any_submission.study_group).show?
-            tr
-              td = link_to exercise, statistics_external_user_exercise_path(@user, exercise)
-              td = stats['maximum_score'] || 0.0
-              td.align-middle
-                - latest_viewable_submission = exercise.final_submission(@user)
-                - if latest_viewable_submission.present?
-                  - if latest_viewable_submission.before_deadline?
-                    .deadline-result.positive-result.before_deadline
-                  - elsif latest_viewable_submission.within_grace_period?
-                    .deadline-result.unknown-result.within_grace_period
-                  - elsif latest_viewable_submission.after_late_deadline?
-                    .deadline-result.negative-result.after_late_deadline
-              td = stats['runs'] || 0 if policy(exercise).detailed_statistics?
-              td = stats['working_time'] || '00:00:00' if policy(exercise).detailed_statistics?
+        - statistics.each do |exercise, submission_stat|
+          tr
+            td = link_to exercise, statistics_external_user_exercise_path(@user, exercise)
+            td = submission_stat['maximum_score'] || 0.0
+            td.align-middle
+              - if submission_stat.updated_at.present?
+                - if submission_stat.before_deadline?
+                  .deadline-result.positive-result
+                - elsif submission_stat.within_grace_period?
+                  .deadline-result.unknown-result
+                - elsif submission_stat.after_late_deadline?
+                  .deadline-result.negative-result
+            td = submission_stat['runs'] || 0 if policy(exercise).detailed_statistics?
+            td = submission_stat['working_time'] || '00:00:00' if policy(exercise).detailed_statistics?
 - else
   = t('exercises.external_users.statistics.no_data_available')