Skip to content

Commit 61163aa

Browse files
authored
Merge pull request #10 from mlibrary/add-cli
Add CLI, Metrics, bug fixes Adds thor cli; Adds yabeda and prometheus for metrics; Adds sematic logger and milemarker for logging. Refactors some of the code to enable better logging of errors.
2 parents 7c9a82f + 979d5ac commit 61163aa

19 files changed

+588
-194
lines changed

.gitignore

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
coverage
55
.byebug_history
66
.irb_history
7-
scratch/*.xml
8-
scratch/*.zip
7+
output/*.xml
8+
output/*.tsv
9+
output/*.zip
910
vendor/

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ RUN groupadd -g ${GID} -o app
1919
RUN useradd -m -d /app -u ${UID} -g ${GID} -o -s /bin/bash app
2020

2121
ENV GEM_HOME=/gems
22-
ENV PATH="$PATH:/gems/bin"
22+
ENV PATH="$PATH:/gems/bin:/app/exe"
2323
RUN mkdir -p /gems && chown ${UID}:${GID} /gems
2424

2525

Gemfile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ gem "csv"
55
gem "net-ldap"
66
gem "shale"
77
gem "rexml"
8+
gem "thor"
9+
gem "yabeda-prometheus"
10+
gem "semantic_logger"
11+
12+
gem "milemarker",
13+
git: "https://github.com/niquerio/milemarker",
14+
branch: "add-logger-to-runtime-dependencies"
815

916
group :development, :test do
1017
gem "pry"

Gemfile.lock

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,30 @@
1+
GIT
2+
remote: https://github.com/niquerio/milemarker
3+
revision: ce3cf65886015c08e9467254a0a9670f00abfc4b
4+
branch: add-logger-to-runtime-dependencies
5+
specs:
6+
milemarker (1.0.1)
7+
logger
8+
19
GEM
210
remote: https://rubygems.org/
311
specs:
12+
anyway_config (2.7.2)
13+
ruby-next-core (~> 1.0)
414
ast (2.4.3)
515
base64 (0.3.0)
616
bigdecimal (3.3.1)
717
byebug (12.0.0)
818
coderay (1.1.3)
19+
concurrent-ruby (1.3.5)
920
csv (3.3.5)
1021
diff-lcs (1.6.2)
1122
docile (1.4.1)
23+
dry-initializer (3.2.0)
1224
json (2.15.1)
1325
language_server-protocol (3.17.0.5)
1426
lint_roller (1.1.0)
27+
logger (1.7.0)
1528
method_source (1.1.0)
1629
net-ldap (0.20.0)
1730
base64
@@ -22,6 +35,8 @@ GEM
2235
ast (~> 2.4.1)
2336
racc
2437
prism (1.6.0)
38+
prometheus-client (4.2.5)
39+
base64
2540
pry (0.15.2)
2641
coderay (~> 1.1)
2742
method_source (~> 1.0)
@@ -63,7 +78,10 @@ GEM
6378
lint_roller (~> 1.1)
6479
rubocop (>= 1.75.0, < 2.0)
6580
rubocop-ast (>= 1.38.0, < 2.0)
81+
ruby-next-core (1.1.2)
6682
ruby-progressbar (1.13.0)
83+
semantic_logger (4.17.0)
84+
concurrent-ruby (~> 1.0)
6785
shale (1.2.2)
6886
bigdecimal
6987
simplecov (0.22.0)
@@ -84,9 +102,18 @@ GEM
84102
standard-performance (1.8.0)
85103
lint_roller (~> 1.1)
86104
rubocop-performance (~> 1.25.0)
105+
thor (1.4.0)
87106
unicode-display_width (3.2.0)
88107
unicode-emoji (~> 4.1)
89108
unicode-emoji (4.1.0)
109+
yabeda (0.14.0)
110+
anyway_config (>= 1.0, < 3)
111+
concurrent-ruby
112+
dry-initializer
113+
yabeda-prometheus (0.9.1)
114+
prometheus-client (>= 3.0, < 5.0)
115+
rack
116+
yabeda (~> 0.10)
90117

91118
PLATFORMS
92119
x86_64-linux
@@ -95,14 +122,18 @@ DEPENDENCIES
95122
base64
96123
byebug
97124
csv
125+
milemarker!
98126
net-ldap
99127
pry
100128
rack-test
101129
rexml
102130
rspec
131+
semantic_logger
103132
shale
104133
simplecov
105134
standard
135+
thor
136+
yabeda-prometheus
106137

107138
BUNDLED WITH
108139
2.7.2

exe/patron

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/usr/bin/env ruby
2+
$LOAD_PATH.unshift(File.absolute_path(File.join(__dir__, "..", "lib")))
3+
4+
require "bundler/setup"
5+
require "cli"
6+
CLI.start

lib/cli.rb

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
require "thor"
2+
require_relative "process_ldap"
3+
4+
class CLI < Thor
5+
def self.exit_on_failure?
6+
true
7+
end
8+
9+
desc "full", "processes all of the users in mcommunity"
10+
method_option :output_directory, type: :string, required: false, default: S.output_directory, desc: "Path to directory for output files"
11+
method_option :base_name, type: :string, required: false, default: "patron_full_#{Date.today.strftime("%Y%m%d")}", desc: "Basename for files put in the output directory"
12+
method_option :size, type: :numeric, required: false, desc: "The maximum number of results to request"
13+
def full
14+
new_options = {
15+
size: options[:size],
16+
base_name: options[:base_name],
17+
output_directory: options[:output_directory]
18+
}
19+
ProcessLdap.new(**new_options).process
20+
end
21+
22+
desc "daily FROM_DATE", "Processes all users in mcommunity modified or created after the FROM_DATE 000000 UTC. For full coverage, the FROM_DATE for a daily cronjob should be yesterday."
23+
method_option :output_directory, type: :string, required: false, default: S.output_directory, desc: "Path to directory for output files"
24+
method_option :base_name, type: :string, required: false, default: "patron_daily_#{Date.today.strftime("%Y%m%d")}", desc: "Basename for files put in the output directory"
25+
method_option :size, type: :numeric, required: false, desc: "The maximum number of results to request"
26+
def daily(from_date)
27+
new_options = {
28+
date: format_date(from_date),
29+
base_name: options[:base_name],
30+
output_directory: options[:output_directory],
31+
size: options[:size]
32+
}
33+
ProcessLdapDaily.new(**new_options).process
34+
end
35+
36+
desc "range", "processes users for a given date range"
37+
method_option :start_date, type: :string, required: true
38+
method_option :end_date, type: :string, required: false, desc: "The end of the date range. Inclusive. If not given, it defaults to whatever start_date is."
39+
method_option :size, type: :numeric, required: false, desc: "The maximum number of results to request"
40+
method_option :output_directory, type: :string, required: false, default: S.output_directory, desc: "Path to directory for output files"
41+
method_option :base_name, type: :string, required: false, desc: "Basename for files put in the output directory; default is patron_range_YYYYMMDD_to_YYYYMMDD"
42+
def range
43+
new_options = {
44+
start_date: format_date(options[:start_date]),
45+
end_date: format_date(options[:end_date] || options[:start_date]),
46+
size: options[:size],
47+
output_directory: options[:output_directory]
48+
}
49+
50+
new_options[:base_name] = options[:base_name] || "patron_range_#{new_options[:start_date]}_to_#{new_options[:end_date]}"
51+
52+
ProcessLdapModifyDateRange.new(**new_options).process
53+
end
54+
55+
desc "one UNIQNAME", "returns the xml for the uniqname"
56+
def one(uniqname)
57+
ProcessLdapOneUser.new(uniqname: uniqname).process
58+
end
59+
60+
desc "ldap UNIQNAME", "returns the ldap info for a user"
61+
def ldap(uniqname)
62+
ProcessLdapOneUser.new(uniqname: uniqname).ldap_output
63+
end
64+
65+
private
66+
67+
no_commands do
68+
def format_date(date)
69+
date = DateTime.parse(date) if date.is_a? String
70+
date.strftime("%Y%m%d")
71+
rescue Date::Error
72+
abort("ERROR: parameter/option #{date} must be a valid date string\n")
73+
end
74+
end
75+
end

lib/patron.rb

Lines changed: 43 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
require_relative "patron/student"
99
require_relative "patron/ann_arbor_student"
1010
require_relative "patron/regional_student"
11+
require_relative "patron/skipped"
1112

1213
require_relative "patron/name"
1314

@@ -20,23 +21,23 @@ def self.inst_roles_for(data)
2021
end
2122
end
2223

23-
def self.valid_for(data)
24-
return if test_user?(data)
25-
inst_roles = inst_roles_for(data)
26-
result = inst_roles.filter_map do |inst_role|
27-
user = for_inst_role(inst_role: inst_role, data: data)
28-
user if user.includable?
24+
def self.for(data)
25+
if test_user?(data)
26+
return Skipped.new(data: data, exclude_reasons: ["test_user"])
2927
end
30-
result&.first
31-
end
3228

33-
def self.exclude_reasons_for(data)
34-
return ["Uniqname: #{data["uid"].first}\tExclude Reason: test_user"] if test_user?(data)
3529
inst_roles = inst_roles_for(data)
36-
inst_roles.map do |inst_role|
30+
exclude_reasons = []
31+
inst_roles.each do |inst_role|
3732
user = for_inst_role(inst_role: inst_role, data: data)
38-
"Uniqname: #{user.primary_id}\tInst Role: #{inst_role["key"]}\tExclude Reason: #{user.exclude_reason}"
33+
if user.includable?
34+
return user
35+
else
36+
S.logger.debug("exclude_reason", {uniqname: user.uniqname, reason: user.exclude_reason, inst_role: inst_role})
37+
exclude_reasons.push(user.exclude_reason)
38+
end
3939
end
40+
Skipped.new(data: data, exclude_reasons: exclude_reasons)
4041
end
4142

4243
def self.test_user?(data)
@@ -71,15 +72,28 @@ def self.for_inst_role(inst_role:, data:)
7172
extend Forwardable
7273

7374
def_delegators :@name, :first_name, :last_name, :middle_name, :middle_name?
75+
attr_reader :exclude_reasons
7476

7577
def initialize(data:, name: Name.new(data), current_schedule: CurrentSchedule.new)
7678
@data = data
7779
@name = name
7880
@current_schedule = current_schedule
79-
end
80-
81-
def includable?
82-
raise NotImplementedError
81+
@exclude_reasons = []
82+
end
83+
84+
[
85+
"campus_code",
86+
"email_type",
87+
"exclude_reason",
88+
"includable?",
89+
"job_description",
90+
"statistic_category",
91+
"umich_address_type",
92+
"user_group"
93+
].each do |method|
94+
define_method method do
95+
raise NotImplementedError, "#{self.class}.#{method}"
96+
end
8397
end
8498

8599
def uniqname
@@ -106,28 +120,11 @@ def purge_date
106120
expiry_date.next_year(2)
107121
end
108122

109-
def campus_code
110-
raise NotImplementedError
111-
end
112-
113-
def user_group
114-
raise NotImplementedError
115-
end
116-
117123
def status
118124
"ACTIVE"
119125
end
120126

121-
def job_description
122-
raise NotImplementedError
123-
end
124-
125-
def statistic_category
126-
raise NotImplementedError
127-
end
128-
129-
def umich_address_type
130-
raise NotImplementedError
127+
def sponsor_reason
131128
end
132129

133130
def umich_address
@@ -166,10 +163,6 @@ def email_address
166163
@data["mail"]&.first
167164
end
168165

169-
def email_type
170-
raise NotImplementedError
171-
end
172-
173166
def phone_number
174167
@data["telephonenumber"]&.first || @data["umichpermanentphone"]&.first
175168
end
@@ -260,18 +253,26 @@ def to_json
260253
to_h.to_json
261254
end
262255

256+
def write(file_handle)
257+
file_handle.write PatronMapper::User.from_hash(to_h).to_xml(pretty: true)
258+
end
259+
263260
def ldap_fields(array)
264261
array.map do |row|
265262
ldap_field(row)
266263
end
267264
end
268265

269266
def ldap_field(row)
270-
OpenStruct.new(row.split("}:").map do |element|
267+
parsed_field = row.split("}:").map do |element|
271268
array = element.gsub(/["{}]/, "").split("=")
272-
array[1] = nil if array.length == 1
273-
array
274-
end.to_h)
269+
if array.length == 1
270+
[array[0], nil]
271+
else
272+
[array[0], array&.slice(1..-1)&.join("=")]
273+
end
274+
end
275+
OpenStruct.new(parsed_field.to_h)
275276
end
276277

277278
class Address

0 commit comments

Comments
 (0)