Add retry all button to add all broken links back to queue in one click

classes/robot/crawler.php

@@ -73,6 +73,9 @@
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 class crawler {
+    /** @var array Retrieved courses from get_recentcourses */
+    private static $recentcourses = false;
+
     /**
      * Returns configuration object if it has been initialised.
      * If it is not initialises then it creates and returns it.
@@ -1306,6 +1309,13 @@ public function scrape($url) {
             $method = 'GET';
         }
 
+        curl_setopt($s, CURLOPT_BUFFERSIZE, 128);
+        curl_setopt($s, CURLOPT_NOPROGRESS, false);
+        curl_setopt($s, CURLOPT_PROGRESSFUNCTION, function ($downloadsize, $downloaded, $uploadsize, $uploaded) {
+            // If $Downloaded exceeds bigfilesize, returning non-0 breaks the connection!
+            return ($downloaded > (1024 * 1000 * self::get_config()->bigfilesize)) ? 1 : 0;
+        });
+
         $result = (object) [];
         $result->url              = $url;
 
@@ -1528,6 +1538,11 @@ public function should_be_authenticated($url) {
      * @return array
      */
     public function get_recentcourses() {
+
+        if (self::$recentcourses != false) {
+            return self::$recentcourses;
+        }
+
         global $DB;
         $config = self::get_config();
 
@@ -1558,6 +1573,7 @@ public function get_recentcourses() {
         }
         $rs->close();
 
+        self::$recentcourses = $recentcourses;
         return $recentcourses;
     }
 

lang/en/tool_crawler.php

@@ -173,6 +173,8 @@
 $string['retentionperiod'] = 'Retention period for bad URLs';
 $string['retentionperioddesc'] = 'How many days to keep bad URLs in database.';
 $string['retry'] = 'Retry';
+$string['retryall'] = 'Retry all';
+$string['retryallmessage'] = 'All links readded to queue to be retried';
 $string['robotcleanup'] = 'Robot cleanup';
 $string['robotstatus'] = 'Status';
 $string['seedurl'] = 'Seed URL';

report.php

@@ -34,6 +34,7 @@
 $perpage    = optional_param('perpage', 50, PARAM_INT);
 $courseid   = optional_param('course', 0, PARAM_INT);
 $retryid    = optional_param('retryid', 0, PARAM_INT);
+$retryall    = optional_param('retryall', 0, PARAM_BOOL);
 $start = $page * $perpage;
 
 $sqlfilter = '';
@@ -69,7 +70,11 @@
     require_capability('moodle/site:config', context_system::instance());
     admin_externalpage_setup('tool_crawler_' . $report);
 }
-echo $OUTPUT->header();
+
+// Do not start output if we are redirecting back to this page.
+if ($retryall == 0) {
+    echo $OUTPUT->header();
+}
 
 require('tabs.php');
 echo $tabs;
@@ -80,6 +85,7 @@
 }
 
 $datetimeformat = get_string('strftimerecentsecondshtml', 'tool_crawler');
+$retryallbutton = '';
 
 if ($report == 'broken' || $report == 'reference') {
     $reference = $report == 'reference';
@@ -138,6 +144,13 @@
     }
     $table->data = [];
     foreach ($data as $row) {
+        if ($retryall == 1) {
+            $retryid = $row->toid;
+            $persistent = new \tool_crawler\local\url();
+            $persistent->reset_for_recrawl($retryid);
+            continue;
+        }
+
         $text = trim($row->text);
         if ($text == "") {
             $text = get_string('missing', 'tool_crawler');
@@ -165,6 +178,19 @@
         }
         $table->data[] = $data;
     }
+    if ($retryall == 1) {
+        $url = $PAGE->url;
+        redirect($url->raw_out(true), get_string('retryallmessage', 'tool_crawler'));
+    }
+    if (!empty($table->data)) {
+        $retryallbutton = html_writer::link(
+            new moodle_url(
+                $navurl->out(),
+                ['retryall' => 1]
+            ),
+            get_string('retryall', 'tool_crawler')
+        );
+    }
 } else if ($report == 'queued') {
     $sql = " FROM {tool_crawler_url} a
        LEFT JOIN {course} c ON c.id = a.courseid
@@ -371,6 +397,7 @@
     ]
 ));
 echo get_string($report . '_header', 'tool_crawler');
+echo $retryallbutton;
 echo html_writer::table($table);
 echo $OUTPUT->paging_bar($count, $page, $perpage, $baseurl);
 echo $OUTPUT->footer();