Skip to content

Commit dbab4bb

Browse files
Alex-JordanAlex-Jordan
authored
Merge pull request openwebwork#3183 from drgrice1/lms-roster-sync-in-job-queue
Move LMS roster synchronization to the job queue.
2 parents 248966a + 70cadb9 commit dbab4bb

3 files changed

Lines changed: 309 additions & 257 deletions

File tree

‎lib/Mojolicious/WeBWorK.pm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ sub startup ($app) {
7676
$app->plugin(Minion => { $ce->{job_queue}{backend} => $ce->{job_queue}{database_dsn} });
7777
$app->minion->add_task(lti_mass_update => 'Mojolicious::WeBWorK::Tasks::LTIMassUpdate');
7878
$app->minion->add_task(lti_set_date_sync => 'Mojolicious::WeBWorK::Tasks::LTISetDateSync');
79+
$app->minion->add_task(lms_roster_sync => 'Mojolicious::WeBWorK::Tasks::LMSRosterSync');
7980
$app->minion->add_task(send_instructor_email => 'Mojolicious::WeBWorK::Tasks::SendInstructorEmail');
8081
$app->minion->add_task(send_achievement_email => 'Mojolicious::WeBWorK::Tasks::AchievementNotification');
8182

‎lib/Mojolicious/WeBWorK/Tasks/LMSRosterSync.pm‎

Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
package Mojolicious::WeBWorK::Tasks::LMSRosterSync;
2+
use Mojo::Base 'Minion::Job', -signatures, -async_await;
3+
4+
use Mojo::UserAgent;
5+
use Mojo::Date;
6+
7+
use WeBWorK::Authen::LTIAdvantage::SubmitGrade;
8+
use WeBWorK::CourseEnvironment;
9+
use WeBWorK::DB;
10+
use WeBWorK::Utils::DateTime qw(formatDateTime);
11+
use WeBWorK::Utils::Instructor qw(assignSetsToUsers);
12+
13+
# Synchronize requested set dates to the LMS.
14+
sub run ($job) {
15+
# Establish a lock guard that only allows 1 job at a time (technically more than one could run at a time if a job
16+
# takes more than an hour to complete). As soon as a job completes (or fails) the lock is released and a new job
17+
# can start. New jobs retry every minute until they can acquire their own lock.
18+
return $job->retry({ delay => 60 }) unless my $guard = $job->minion->guard('lms_roster_sync', 3600);
19+
20+
# Minion does not support asynchronous jobs with notification of job completion, and so the Mojolicious::Promise
21+
# wait method must be used. The synchronizeSetDates method is used so that the async/await syntax can be used
22+
# instead of using the wait method on each method that needs to be awaited which would be tedious. So the wait
23+
# method only needs to be used once here.
24+
$job->synchronizeRoster->wait();
25+
26+
return;
27+
}
28+
29+
async sub synchronizeRoster ($job) {
30+
my $courseID = $job->info->{notes}{courseID};
31+
return $job->fail('The course id was not passed when this job was enqueued.') unless $courseID;
32+
33+
my $ce = eval { WeBWorK::CourseEnvironment->new({ courseName => $courseID }) };
34+
return $job->fail('Could not construct course environment.') unless $ce;
35+
36+
$job->{language_handle} = WeBWorK::Localize::getLoc($ce->{language} || 'en');
37+
38+
return $job->fail($job->maketext('This course is not configured to import users from the LMS via LTI.'))
39+
if !$ce->{LTIVersion}
40+
|| $ce->{LTIVersion} ne 'v1p3'
41+
|| !$ce->{LTI}{v1p3}{preferred_source_of_username};
42+
43+
my $db = WeBWorK::DB->new($ce);
44+
return $job->fail($job->maketext('Could not obtain database connection.')) unless $db;
45+
46+
my $namesRolesServiceURL = $db->getSettingValue('LTINamesRolesServiceURL');
47+
return $job->fail($job->maketext('The LTI names/roles service URL is not available.')) unless $namesRolesServiceURL;
48+
49+
my $accessToken =
50+
await WeBWorK::Authen::LTIAdvantage::SubmitGrade->new(({ ce => $ce, db => $db, app => $job->app }, 1))
51+
->get_access_token;
52+
return $job->fail($job->maketext('Unable to obtain access token.')) unless $accessToken;
53+
54+
my @namesRoles;
55+
56+
while (1) {
57+
my $namesRolesServiceRequest = await Mojo::UserAgent->new->get_p($namesRolesServiceURL,
58+
{ Authorization => "$accessToken->{token_type} $accessToken->{access_token}" })->catch(sub ($err) {
59+
return $err;
60+
});
61+
return $job->fail(
62+
$job->maketext("Error communicating with the names and roles service URL: $namesRolesServiceRequest\n"))
63+
unless ref $namesRolesServiceRequest;
64+
65+
my $namesRolesServiceResult = $namesRolesServiceRequest->result;
66+
if ($namesRolesServiceResult->is_success) {
67+
my $namesRoles = $namesRolesServiceResult->json->{members};
68+
return $job->fail($job->maketext('Invalid data received from the LMS.')) unless ref $namesRoles eq 'ARRAY';
69+
push(@namesRoles, @$namesRoles);
70+
71+
if ($namesRolesServiceResult->headers->link
72+
&& $namesRolesServiceResult->headers->link =~ /<([^>]*)>;\s*rel="next"/)
73+
{
74+
$namesRolesServiceURL = $1;
75+
} else {
76+
last;
77+
}
78+
} else {
79+
return $job->fail($job->maketext(
80+
'There was an error obtaining the list of users from the LMS: [_1]',
81+
$namesRolesServiceResult->message
82+
));
83+
}
84+
}
85+
86+
my (@messages, @addedUsers, @userAchievementRecordsToAdd, @globalAchievementRecordsToAdd, %usersInLMSCourse);
87+
my $updatedUsers = 0;
88+
89+
my %users = map { $_->user_id => $_ } $db->getUsersWhere({ user_id => { not_like => 'set_id:%' } });
90+
91+
my @achievements = $db->getAchievementsWhere({ enabled => 1 }, ['achievement_id']);
92+
93+
my $preferredSourceOfUsername = $ce->{LTI}{v1p3}{namesroles_service_preferred_source_of_username}
94+
|| $ce->{LTI}{v1p3}{preferred_source_of_username};
95+
my $fallbackPasswordSource = $ce->{LTI}{v1p3}{namesroles_service_fallback_source_of_username}
96+
|| $ce->{LTI}{v1p3}{fallback_source_of_username};
97+
my $preferredSourceOfStudentId = $ce->{LTI}{v1p3}{namesroles_service_preferred_source_of_student_id}
98+
|| $ce->{LTI}{v1p3}{preferred_source_of_student_id};
99+
100+
for my $user (@namesRoles) {
101+
my ($userIdSource, $typeOfSource) = ('', '');
102+
my $userId = $user->{$preferredSourceOfUsername};
103+
if (defined $userId) {
104+
$userIdSource = $preferredSourceOfUsername;
105+
$typeOfSource =
106+
"$userIdSource which was "
107+
. ($ce->{LTI}{v1p3}{namesroles_service_preferred_source_of_username}
108+
? 'namesroles_service_preferred_source_of_username'
109+
: 'preferred_source_of_username');
110+
} elsif ($fallbackPasswordSource && !defined $userId && defined $user->{$fallbackPasswordSource}) {
111+
$userIdSource = $fallbackPasswordSource;
112+
$typeOfSource =
113+
"$userIdSource which was"
114+
. ($ce->{LTI}{v1p3}{namesroles_service_fallback_source_of_username}
115+
? 'namesroles_service_fallback_source_of_username'
116+
: 'fallback_source_of_username');
117+
$userId = $user->{$fallbackPasswordSource};
118+
}
119+
120+
unless (defined $userId) {
121+
$job->app->log->info("\n=====================================\n"
122+
. "Unable to determine a webwork user id for LMS user:\n"
123+
. $job->app->dumper($user)
124+
. "\n=====================================")
125+
if $ce->{debug_lti_parameters};
126+
next;
127+
}
128+
129+
$userId =~ s/@.*$// if $userIdSource eq 'email' && $ce->{LTI}{v1p3}{strip_domain_from_email};
130+
$userId = lc($userId) if $ce->{LTI}{v1p3}{lowercase_username};
131+
132+
my $studentId = $preferredSourceOfStudentId ? ($user->{$preferredSourceOfStudentId} // '') : '';
133+
134+
if ($ce->{debug_lti_parameters}) {
135+
$job->app->log->info("\n=========== USER SUMMARY ============\n"
136+
. "----------- LMS USER DATA -----------\n"
137+
. $job->app->dumper($user)
138+
. "-------------------------------------\n"
139+
. "User id is |$userId| (obtained from $typeOfSource)\n"
140+
. "User email address is |$user->{email}|\n"
141+
. "Student id is |$studentId|\n"
142+
. "=====================================");
143+
}
144+
145+
$usersInLMSCourse{$userId} = 1;
146+
147+
# Note that the only reliably obtained roles here are the membership roles. The issue is that these roles
148+
# are allowed to be abbreviated (i.e., the http://purl.imsglobal.org/... part may be entirely omitted
149+
# according to the specification). Moodle does this, but Canvas does not. However, both seem to add a prefix
150+
# for non-membership roles. Also, "institution" roles are not sent, so it is not even possible to honor the
151+
# $ce->{LTI}{v1p3}{AllowInstitutionRoles} setting.
152+
my @LTIroles = map {s|^http://purl.imsglobal.org/vocab/lis/v2/membership#||r} @{ $user->{roles} };
153+
154+
$job->app->log->info("The LTI roles defined for $userId are: \n-- " . join("\n-- ", @LTIroles))
155+
if $ce->{debug_lti_parameters};
156+
157+
if (!defined($ce->{userRoles}{ $ce->{LTI}{v1p3}{LMSrolesToWeBWorKroles}{ $LTIroles[0] } })) {
158+
$job->app->log->info("Skipping $userId. Cannot find a WeBWorK role that corresponds to the "
159+
. "LMS role of $LTIroles[0] for this user.")
160+
if $ce->{debug_lti_parameters};
161+
next;
162+
}
163+
164+
my $permissionLevel = $ce->{userRoles}{ $ce->{LTI}{v1p3}{LMSrolesToWeBWorKroles}{ $LTIroles[0] } };
165+
if (@LTIroles > 1) {
166+
for (@LTIroles[ 1 .. $#LTIroles ]) {
167+
my $wwRole = $ce->{LTI}{v1p3}{LMSrolesToWeBWorKroles}{$_};
168+
next unless defined $wwRole;
169+
$permissionLevel = $ce->{userRoles}{$wwRole} if $permissionLevel < $ce->{userRoles}{$wwRole};
170+
}
171+
}
172+
if ($permissionLevel > $ce->{userRoles}{ $ce->{LTIAccountCreationCutoff} }) {
173+
$job->app->log->info("Skipping $userId. User has a role above the LTI "
174+
. "account creation cutoff of $ce->{LTIAccountCreationCutoff}.")
175+
if $ce->{debug_lti_parameters};
176+
next;
177+
}
178+
179+
if ($users{$userId}) {
180+
next unless $ce->{LMSManageUserData};
181+
182+
# Create a temporary user with the LMS credentials and compare the user to the existing user.
183+
my $tempUser = $db->newUser(
184+
user_id => $userId,
185+
lis_source_did => $user->{user_id},
186+
last_name => $user->{family_name} =~ s/\+/ /gr,
187+
first_name => $user->{given_name} =~ s/\+/ /gr,
188+
email_address => $user->{email},
189+
status => $user->{status} eq 'Active' ? 'C' : 'D',
190+
comment =>
191+
formatDateTime(time, 'datetime_format_short', $ce->{siteDefaults}{timezone}, $ce->{language}),
192+
student_id => $studentId,
193+
section => '',
194+
recitation => ''
195+
);
196+
197+
my $change_made = 0;
198+
for my $element (qw(last_name first_name email_address status student_id)) {
199+
if ($users{$userId}->$element ne $tempUser->$element) {
200+
$change_made = 1;
201+
$job->app->log->info("WeBWorK user has $element: "
202+
. $users{$userId}->$element
203+
. ", but LMS user has $element: "
204+
. $tempUser->$element)
205+
if $ce->{debug_lti_parameters};
206+
$users{$userId}->$element($tempUser->$element);
207+
}
208+
}
209+
210+
if ($change_made) {
211+
++$updatedUsers;
212+
$tempUser->comment(
213+
formatDateTime(time, 'datetime_format_short', $ce->{siteDefaults}{timezone}, $ce->{language}));
214+
eval { $db->putUser($tempUser) };
215+
if ($@) {
216+
$job->app->log->error("Failed to update user $userId when importing LMS user: $@");
217+
push(@messages, $job->maketext('Failed to update user [_1].', $userId));
218+
} else {
219+
push(@messages, $job->maketext('Updated user [_1].', $userId));
220+
}
221+
} else {
222+
push(@messages, $job->maketext("[_1] not changed.", $userId));
223+
}
224+
} else {
225+
push(@messages, $job->maketext('Added user [_1] with permission level [_2].', $userId, $permissionLevel));
226+
push(@addedUsers, $userId);
227+
228+
my $newUser = $db->newUser(
229+
user_id => $userId,
230+
lis_source_did => $user->{user_id},
231+
last_name => $user->{family_name} =~ s/\+/ /gr,
232+
first_name => $user->{given_name} =~ s/\+/ /gr,
233+
email_address => $user->{email},
234+
status => $user->{status} eq 'Active' ? 'C' : 'D',
235+
comment =>
236+
formatDateTime(time, 'datetime_format_short', $ce->{siteDefaults}{timezone}, $ce->{language}),
237+
student_id => $studentId,
238+
section => '',
239+
recitation => ''
240+
);
241+
$db->addUser($newUser);
242+
243+
$db->addPermissionLevel($db->newPermissionLevel(user_id => $userId, permission => $permissionLevel));
244+
245+
for (@achievements) {
246+
push(@userAchievementRecordsToAdd,
247+
$db->newUserAchievement(user_id => $userId, achievement_id => $_->achievement_id));
248+
}
249+
push(@globalAchievementRecordsToAdd,
250+
$db->newGlobalUserAchievement(user_id => $userId, achievement_points => 0));
251+
252+
$users{$userId} = $newUser;
253+
}
254+
}
255+
256+
# Assign visible sets to the added users.
257+
assignSetsToUsers($db, $ce, [ map { $_->[0] } $db->listGlobalSetsWhere({ visible => 1 }) ], \@addedUsers)
258+
if @addedUsers;
259+
260+
# Assign achievements to the added users.
261+
$db->UserAchievement->insert_records(\@userAchievementRecordsToAdd) if @userAchievementRecordsToAdd;
262+
$db->GlobalUserAchievement->insert_records(\@globalAchievementRecordsToAdd)
263+
if @globalAchievementRecordsToAdd;
264+
265+
my %permissionLevels =
266+
map { $_->user_id => $_->permission } $db->getPermissionLevelsWhere({ user_id => { not_like => 'set_id:%' } });
267+
268+
# Mark all users not in the LMS roster and at or below the LTIAccountCreationCutoff as dropped.
269+
my @droppedUsers;
270+
for my $user (values %users) {
271+
next
272+
if $usersInLMSCourse{ $user->user_id }
273+
|| ($permissionLevels{ $user->user_id } // 0) > $ce->{userRoles}{ $ce->{LTIAccountCreationCutoff} }
274+
|| $user->status eq 'D';
275+
$user->status('D');
276+
push(@messages, $job->maketext('Dropped user [_1]', $user->user_id));
277+
push(@droppedUsers, $user);
278+
}
279+
$db->User->update_records(\@droppedUsers) if @droppedUsers;
280+
281+
push(
282+
@messages,
283+
$ce->{LMSManageUserData}
284+
? $job->maketext(
285+
'[_1] [plural,_1,user] added, [_2] [plural,_2,user] updated, '
286+
. '[_3] [plural,_3,user] not in LMS [plural,_3,was,were] dropped',
287+
scalar(@addedUsers),
288+
$updatedUsers,
289+
scalar(@droppedUsers)
290+
)
291+
: $job->maketext(
292+
'[_1] [plural,_1,user] added, [_2] [plural,_2,user] not in LMS [plural,_2,was,were] dropped',
293+
scalar(@addedUsers), scalar(@droppedUsers)
294+
)
295+
);
296+
return $job->finish(@messages > 1 ? \@messages : $messages[0]);
297+
}
298+
299+
sub maketext ($job, @args) {
300+
return &{ $job->{language_handle} }(@args);
301+
}
302+
303+
1;

0 commit comments

Comments
 (0)