course sync and cohort sync task optimisation:

  - Consolidate local_o365_teams_cache into local_o365_groups_cache for course sync and cohort sync tasks.
    - Extend local_o365_groups_cache with has_team, url, locked, team_details_last_attempted, lock_status_last_checked fields and supporting indexes.
    - Drop local_o365_teams_cache table.
  - Rewrite utils::update_groups_cache() to handle both groups and teams in one pass, with 5-minute rate limiting, lazy team-detail fetching (URL once, lock status every 7 days), and bulk DB operations.
  - Replace two separate cache-update calls in the course sync task with a single unified call; fix cohort sync task to proceed on rate-limit skip rather than abort.
  - Update all table references in tasks, admin UI, and deprecated update_teams_cache() to use the unified cache table.

Author: weilai-irl

local/o365/classes/feature/cohortsync/main.php

@@ -217,22 +217,20 @@ public function fetch_cohorts(): void {
     /**
      * Update Groups cache.
      *
-     * @return bool
+     * @return bool|null True if the cache was updated, null if skipped (rate limit), false on error.
      */
-    public function update_groups_cache(): bool {
+    public function update_groups_cache(): ?bool {
         global $DB;
 
         try {
-            if (utils::update_groups_cache($this->graphclient, 1)) {
+            $result = utils::update_groups_cache($this->graphclient, 1);
+            if ($result === true) {
                 $sql = 'SELECT *
                           FROM {local_o365_groups_cache}
                          WHERE not_found_since = 0';
                 $this->grouplist = $DB->get_records_sql($sql);
-
-                return true;
-            } else {
-                return false;
             }
+            return $result;
         } catch (moodle_exception $e) {
             utils::debug('Exception in update_groups_cache: ' . $e->getMessage(), __METHOD__, $e);
             return false;

local/o365/classes/feature/coursesync/main.php

@@ -1118,6 +1118,10 @@ private function restore_group(int $objectrecid, string $objectid, array $object
     /**
      * Update Teams cache.
      *
+     * @deprecated Use \local_o365\utils::update_groups_cache() instead, which handles both
+     *             groups and teams in the unified cache table. This function is no longer
+     *             called by any production code and will be removed in a future version.
+     *
      * @return bool
      */
     public function update_teams_cache(): bool {
@@ -1152,10 +1156,10 @@ public function update_teams_cache(): bool {
             return false;
         }
 
-        // Build existing teams records cache.
+        // Build existing teams records cache from unified groups cache table.
         $this->mtrace('Building existing teams cache records', 1);
         // Use recordset instead of get_records to reduce memory usage.
-        $existingcacherecordset = $DB->get_recordset('local_o365_teams_cache');
+        $existingcacherecordset = $DB->get_recordset('local_o365_groups_cache', ['has_team' => 1]);
         $existingcachebyoid = [];
         foreach ($existingcacherecordset as $existingcacherecord) {
             $existingcachebyoid[$existingcacherecord->objectid] = $existingcacherecord;
@@ -1168,10 +1172,12 @@ public function update_teams_cache(): bool {
         foreach ($teams as $team) {
             if (array_key_exists($team['id'], $existingcachebyoid)) {
                 // Update existing cache record.
+                $lockstatuschecked = false;
                 if (!$existingcachebyoid[$team['id']]->locked) {
                     // Need to update lock status.
                     try {
                         [$rawteam, $teamurl, $lockstatus] = $this->graphclient->get_team($team['id']);
+                        $lockstatuschecked = true;
                     } catch (moodle_exception $e) {
                         continue;
                     }
@@ -1183,7 +1189,10 @@ public function update_teams_cache(): bool {
                 $cacherecord->name = $team['displayName'];
                 $cacherecord->description = $team['description'];
                 $cacherecord->locked = $lockstatus;
-                $DB->update_record('local_o365_teams_cache', $cacherecord);
+                if ($lockstatuschecked) {
+                    $cacherecord->lock_status_last_checked = time();
+                }
+                $DB->update_record('local_o365_groups_cache', $cacherecord);
 
                 unset($existingcachebyoid[$team['id']]);
             } else {
@@ -1194,20 +1203,47 @@ public function update_teams_cache(): bool {
                     continue;
                 }
 
-                // Create new cache record.
-                $cacherecord = new stdClass();
-                $cacherecord->objectid = $team['id'];
-                $cacherecord->name = $team['displayName'];
-                $cacherecord->description = $team['description'];
-                $cacherecord->url = $teamurl;
-                $cacherecord->locked = $lockstatus;
-                $DB->insert_record('local_o365_teams_cache', $cacherecord);
+                // Upsert: a group-only row (has_team=0) may already exist for this objectid.
+                // Inserting a second row would create duplicates and break get_record() calls.
+                $cacherecord = $DB->get_record('local_o365_groups_cache', ['objectid' => $team['id']]);
+                if ($cacherecord) {
+                    // Promote the existing group row to a team row.
+                    $cacherecord->name = $team['displayName'];
+                    $cacherecord->description = $team['description'];
+                    $cacherecord->has_team = 1;
+                    $cacherecord->url = $teamurl;
+                    $cacherecord->locked = $lockstatus;
+                    $cacherecord->not_found_since = 0;
+                    $cacherecord->team_details_last_attempted = time();
+                    $cacherecord->lock_status_last_checked = time();
+                    $DB->update_record('local_o365_groups_cache', $cacherecord);
+                } else {
+                    $cacherecord = new stdClass();
+                    $cacherecord->objectid = $team['id'];
+                    $cacherecord->name = $team['displayName'];
+                    $cacherecord->description = $team['description'];
+                    $cacherecord->has_team = 1;
+                    $cacherecord->url = $teamurl;
+                    $cacherecord->locked = $lockstatus;
+                    $cacherecord->not_found_since = 0;
+                    $cacherecord->team_details_last_attempted = 0;
+                    $cacherecord->lock_status_last_checked = 0;
+                    $DB->insert_record('local_o365_groups_cache', $cacherecord);
+                }
             }
         }
 
-        $this->mtrace('Deleting old teams cache records', 1);
+        // Demote groups that are no longer teams: set has_team=0 and clear team-only fields.
+        // Deleting the row would cause data loss if the underlying group still exists;
+        // this matches the demotion behaviour in utils::update_groups_cache().
+        $this->mtrace('Demoting old teams cache records to group-only', 1);
         foreach ($existingcachebyoid as $oldcacherecord) {
-            $DB->delete_records('local_o365_teams_cache', ['id' => $oldcacherecord->id]);
+            $oldcacherecord->has_team = 0;
+            $oldcacherecord->url = null;
+            $oldcacherecord->locked = null;
+            $oldcacherecord->team_details_last_attempted = 0;
+            $oldcacherecord->lock_status_last_checked = 0;
+            $DB->update_record('local_o365_groups_cache', $oldcacherecord);
         }
 
         $this->mtrace('Finished updating teams cache.');
@@ -1222,15 +1258,17 @@ public function update_teams_cache(): bool {
 
     /**
      * Cleanup Teams connections records.
-     * This function will delete all Teams connection records with object IDs not found in the Teams cache.
-     * This function should only be called after Teams cache is updated - no cache update will be performed here.
+     * This function will delete all Teams connection records with object IDs not found in the cache.
+     * Teams are identified in the cache by has_team=1.
+     * This function should only be called after the cache is updated - no cache update will be performed here.
      *
      * @return void
      */
     public function cleanup_teams_connections() {
         global $DB;
 
-        $teamobjectids = $DB->get_fieldset_select('local_o365_teams_cache', 'objectid', '');
+        // Get all team object IDs from the unified cache (has_team=1).
+        $teamobjectids = $DB->get_fieldset_select('local_o365_groups_cache', 'objectid', 'has_team = 1');
 
         $this->mtrace('Clean up teams connection records...');
         if ($teamobjectids) {
@@ -1679,12 +1717,12 @@ public function get_all_group_ids(): array {
     public function is_team_locked(string $groupobjectid) {
         global $DB;
 
-        if ($teamcacherecord = $DB->get_record('local_o365_teams_cache', ['objectid' => $groupobjectid])) {
+        if ($teamcacherecord = $DB->get_record('local_o365_groups_cache', ['objectid' => $groupobjectid, 'has_team' => 1])) {
             switch ($teamcacherecord->locked) {
                 case TEAM_LOCKED_STATUS_UNKNOWN:
                     try {
                         [$team, $teamurl, $lockstatus] = $this->graphclient->get_team($groupobjectid);
-                        $DB->set_field('local_o365_teams_cache', 'locked', $lockstatus, ['objectid' => $groupobjectid]);
+                        $DB->set_field('local_o365_groups_cache', 'locked', $lockstatus, ['objectid' => $groupobjectid]);
                     } catch (moodle_exception $e) {
                         $lockstatus = TEAM_UNLOCKED;
                     }
@@ -1693,7 +1731,7 @@ public function is_team_locked(string $groupobjectid) {
                     try {
                         [$team, $teamurl, $lockstatus] = $this->graphclient->get_team($groupobjectid);
                         if ($lockstatus == TEAM_UNLOCKED) {
-                            $DB->set_field('local_o365_teams_cache', 'locked', $lockstatus, ['objectid' => $groupobjectid]);
+                            $DB->set_field('local_o365_groups_cache', 'locked', $lockstatus, ['objectid' => $groupobjectid]);
                         }
                     } catch (moodle_exception $e) {
                         $lockstatus = TEAM_UNLOCKED;

local/o365/classes/feature/coursesync/utils.php

@@ -293,7 +293,7 @@ public static function get_matching_team_options(string $currentoid = ''): array
 
         $matchedoids = $DB->get_fieldset_select('local_o365_objects', 'objectid', 'type = ? AND subtype = ?', ['group', 'course']);
 
-        $teamcacherecords = $DB->get_records('local_o365_teams_cache');
+        $teamcacherecords = $DB->get_records('local_o365_groups_cache', ['has_team' => 1]);
         foreach ($teamcacherecords as $key => $teamcacherecord) {
             if ($teamcacherecord->objectid == $currentoid || !in_array($teamcacherecord->objectid, $matchedoids)) {
                 if (!array_key_exists($teamcacherecord->name, $teamnamecache)) {

local/o365/classes/page/acp.php

@@ -1008,6 +1008,8 @@ public function mode_teamconnections() {
                     ['moodleid' => $course->id, 'type' => 'group', 'subtype' => 'course']
                 )
             ) {
+                $teamscachedata = ['objectid' => $grouprecord->objectid, 'has_team' => 1];
+
                 if (
                     $DB->record_exists(
                         'local_o365_objects',
@@ -1019,7 +1021,7 @@ public function mode_teamconnections() {
                     )
                 ) {
                     // Connected to both group and team.
-                    if ($teamscache = $DB->get_record('local_o365_teams_cache', ['objectid' => $grouprecord->objectid])) {
+                    if ($teamscache = $DB->get_record('local_o365_groups_cache', $teamscachedata)) {
                         // Team record can be found in cache.
                         $existingconnection = html_writer::link($teamscache->url, $teamscache->name);
                         if (
@@ -1057,7 +1059,7 @@ public function mode_teamconnections() {
                         $connectlabel = get_string('acp_teamconnections_table_connect', 'local_o365');
 
                         $actions = [html_writer::link($connecturl, $connectlabel)];
-                    } else if ($teamscache = $DB->get_record('local_o365_teams_cache', ['objectid' => $grouprecord->objectid])) {
+                    } else if ($teamscache = $DB->get_record('local_o365_groups_cache', $teamscachedata)) {
                         // Connect the course with the team.
                         $teamobjectrecord = ['type' => 'group', 'subtype' => 'courseteam', 'objectid' => $teamscache->objectid,
                             'moodleid' => $course->id, 'o365name' => $teamscache->name, 'timecreated' => time(),
@@ -1176,8 +1178,9 @@ public function mode_teamconnections_update_cache() {
         confirm_sesskey();
 
         $graphclient = \local_o365\feature\coursesync\utils::get_graphclient();
-        $coursesync = new main($graphclient);
-        $coursesync->update_teams_cache();
+        // Pass forceupdate=true so an explicit admin request is never silently skipped
+        // by the 5-minute rate limit that applies to automated task runs.
+        \local_o365\utils::update_groups_cache($graphclient, 0, true);
 
         $redirecturl = new moodle_url('/local/o365/acp.php', ['mode' => 'teamconnections']);
         redirect($redirecturl, get_string('acp_teamconnections_teams_cache_updated', 'local_o365'));
@@ -1230,7 +1233,7 @@ public function mode_teamconnections_connect() {
                 redirect($redirecturl);
             }
 
-            if (!$teamcacherecord = $DB->get_record('local_o365_teams_cache', ['id' => $teamid])) {
+            if (!$teamcacherecord = $DB->get_record('local_o365_groups_cache', ['id' => $teamid, 'has_team' => 1])) {
                 throw new moodle_exception('acp_teamconnections_exception_invalid_team_id', 'local_o365', $redirecturl);
             } else if (
                 $DB->record_exists(
@@ -1377,7 +1380,7 @@ public function mode_teamconnections_update() {
                 redirect($redirecturl);
             }
 
-            if (!$teamcacherecord = $DB->get_record('local_o365_teams_cache', ['id' => $teamid])) {
+            if (!$teamcacherecord = $DB->get_record('local_o365_groups_cache', ['id' => $teamid, 'has_team' => 1])) {
                 throw new moodle_exception('acp_teamconnections_exception_invalid_team_id', 'local_o365', $redirecturl);
             } else if (
                 $teamobjectrecord = $DB->get_record(

local/o365/classes/task/cohortsync.php

@@ -81,9 +81,14 @@ public function execute(): bool {
      */
     private function execute_sync(main $cohortsync): void {
         // First, update the group cache, and delete any groups that no longer exist.
+        // A null result means the update was rate-limited; proceed using the existing cache
+        // but skip cleanup (which depends on a fresh not-found list from the API).
         try {
-            if ($cohortsync->update_groups_cache()) {
+            $cacheupdateresult = $cohortsync->update_groups_cache();
+            if ($cacheupdateresult === true) {
                 utils::clean_up_not_found_groups();
+            } else if ($cacheupdateresult === null) {
+                utils::mtrace("Groups cache update skipped (rate limit). Proceeding with existing cache.", 1);
             } else {
                 utils::mtrace("Failed to update groups cache. Exiting.", 1);
                 return;
@@ -99,7 +104,7 @@ private function execute_sync(main $cohortsync): void {
         utils::mtrace("Found " . count($grouplist) . " groups.", 2);
         $grouplistbyoid = [];
         foreach ($grouplist as $group) {
-            $grouplistbyoid[$group->objectid] = $group;
+            $grouplistbyoid[$group['id']] = $group;
         }
 
         $mappings = $cohortsync->get_mappings();

local/o365/classes/task/coursesync.php

@@ -79,28 +79,21 @@ public function execute() {
         $coursesync = new main($graphclient, true);
         $coursesync->sync_courses();
 
-        // Wrap update_teams_cache in try-catch to handle API failures gracefully.
+        // Update the unified groups and teams cache.
+        // Returns true = cache updated, null = skipped (rate limit), false = error.
+        // Cleanup is only safe to run when the cache was actually refreshed (true).
         try {
-            if ($coursesync->update_teams_cache()) {
+            if (utils::update_groups_cache($graphclient, 1) === true) {
+                // Cleanup orphaned team connections and save not found groups.
                 $coursesync->cleanup_teams_connections();
-            }
-        } catch (Exception $e) {
-            mtrace('Error updating teams cache: ' . $e->getMessage());
-            utils::debug('Exception in update_teams_cache: ' . $e->getMessage(), __METHOD__, $e);
-            // Continue with other operations even if teams cache update fails.
-        }
-
-        $coursesync->cleanup_course_connection_records();
-
-        // Update the groups cache and save any not found groups.
-        try {
-            if (utils::update_groups_cache($graphclient, 1)) {
                 $coursesync->save_not_found_groups();
                 utils::clean_up_not_found_groups();
             }
         } catch (Exception $e) {
-            mtrace('Error updating groups cache: ' . $e->getMessage());
+            mtrace('Error updating groups/teams cache: ' . $e->getMessage());
             utils::debug('Exception in update_groups_cache: ' . $e->getMessage(), __METHOD__, $e);
         }
+
+        $coursesync->cleanup_course_connection_records();
     }
 }

local/o365/classes/task/processcourserequestapproval.php

@@ -96,10 +96,11 @@ public function execute(): bool {
         utils::set_course_sync_enabled($coursedata->courseid);
 
         // Update teams cache record status.
-        if ($teamcacherecord = $DB->get_record('local_o365_teams_cache', ['objectid' => $courserequestdata->teamoid])) {
+        $teamcachefilter = ['objectid' => $courserequestdata->teamoid, 'has_team' => 1];
+        if ($teamcacherecord = $DB->get_record('local_o365_groups_cache', $teamcachefilter)) {
             if ($teamcacherecord->locked != TEAM_LOCKED) {
                 $teamcacherecord->locked = TEAM_LOCKED;
-                $DB->update_record('local_o365_teams_cache', $teamcacherecord);
+                $DB->update_record('local_o365_groups_cache', $teamcacherecord);
             }
         }