From 898f83b64510e7a2aaa404f983f921bcfc3f0700 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Wed, 22 Nov 2023 09:36:34 -0600 Subject: [PATCH 01/18] Initial Commit for the new project-usermap script Uses LDAP search to find active and provisioned groups then compares them to the list project groups in COmanage to determine which LDAP groups and users to build the map out of. --- osg-comanage-project-usermap.py | 152 +++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 12 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 879a255..668a9fc 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 +import re import os import sys import json import getopt +import subprocess import collections import urllib.error import urllib.request @@ -16,6 +18,36 @@ MAXTIMEOUT = 625 TIMEOUTMULTIPLE = 5 +LDAP_AUTH_COMMAND = [ + "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", +] + +LDAP_GROUP_MEMBERS_COMMAND = [ + "ldapsearch", + "-H", + "ldaps://ldap.cilogon.org", + "-D", + "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", + "-w", "{}", + "-b", + "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", + "-s", + "one", + "(cn=*)", +] + +LDAP_ACTIVE_USERS_COMMAND = [ + "ldapsearch", + "-LLL", + "-H", "ldaps://ldap.cilogon.org", + "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", + "-x", + "-w", "{}", + "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", + "(isMemberOf=CO:members:active)", "voPersonApplicationUID", + "|", "grep", "voPersonApplicationUID", + "|", "sort", +] _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] @@ -142,13 +174,19 @@ def get_osg_co_groups__map(): return { g["Id"]: g["Name"] for g in data } -def co_group_is_ospool(gid): +def co_group_is_project(gid): #print(f"co_group_is_ospool({gid})") resp_data = get_co_group_identifiers(gid) data = get_datalist(resp_data, "Identifiers") return any( i["Type"] == "ospoolproject" for i in data ) +def get_co_group_osggid(gid): + resp_data = get_co_group_identifiers(gid) + data = get_datalist(resp_data, "Identifiers") + return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"] + + def get_co_group_members__pids(gid): #print(f"get_co_group_members__pids({gid})") resp_data = get_co_group_members(gid) @@ -192,6 +230,84 @@ def parse_options(args): options.authstr = mkauthstr(user, passwd) +def get_ldap_group_members_data(): + gidNumber_str = "gidNumber: " + gidNumber_regex = re.compile(gidNumber_str) + member_str = f"hasMember: " + member_regex = re.compile(member_str) + + auth_str = subprocess.run( + LDAP_AUTH_COMMAND, + stdout=subprocess.PIPE + ).stdout.decode('utf-8').strip() + + ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND + ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{}")] = auth_str + + data_file = subprocess.run( + ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') + + search_results = list(filter( + lambda x: not re.compile("#|dn|cn|objectClass").match(x), + (line for line in data_file))) + + search_results.reverse() + + group_data_dict = dict() + index = 0 + while index < len(search_results) - 1: + while not gidNumber_regex.match(search_results[index]): + index += 1 + gid = search_results[index].replace(gidNumber_str, "") + members_list = [] + while search_results[index] != "": + if member_regex.match(search_results[index]): + members_list.append(search_results[index].replace(member_str, "")) + index += 1 + group_data_dict[gid] = members_list + index += 1 + + return group_data_dict + + +def get_ldap_active_users(): + auth_str = subprocess.run( + LDAP_AUTH_COMMAND, + stdout=subprocess.PIPE + ).stdout.decode('utf-8').strip() + + ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{}")] = auth_str + + active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') + users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) + return users + + +def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names): + users_to_projects_map = dict() + for osggid in project_to_user_map: + for user in project_to_user_map[osggid]: + if user in active_users: + if user not in users_to_projects_map: + users_to_projects_map[user] = [osggids_to_names[osggid]] + else: + users_to_projects_map[user].append(osggids_to_names[osggid]) + + return users_to_projects_map + + +def get_co_api_data(): + #TODO add cacheing for COManage API data + + groups = get_osg_co_groups__map() + project_osggids_to_name = dict() + for id,name in groups.items(): + if co_group_is_project(id): + project_osggids_to_name[get_co_group_osggid(id)] = name + return project_osggids_to_name + + def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): pid_gids = collections.defaultdict(set) @@ -211,17 +327,29 @@ def filter_by_group(pid_gids, groups, filter_group_name): def get_osguser_groups(filter_group_name=None): - groups = get_osg_co_groups__map() - ospool_gids = filter(co_group_is_ospool, groups) - gid_pids = { gid: get_co_group_members__pids(gid) for gid in ospool_gids } - all_pids = set( pid for gid in gid_pids for pid in gid_pids[gid] ) - pid_osguser = { pid: get_co_person_osguser(pid) for pid in all_pids } - pid_gids = gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser) - if filter_group_name is not None: - pid_gids = filter_by_group(pid_gids, groups, filter_group_name) - - return { pid_osguser[pid]: sorted(map(groups.get, gids)) - for pid, gids in pid_gids.items() } + project_osggids_to_name = get_co_api_data() + ldap_groups_members = get_ldap_group_members_data() + ldap_users = get_ldap_active_users() + + active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) + project_to_user_map = { + osggid : ldap_groups_members[osggid] + for osggid in active_project_osggids + } + all_project_users = set( + username for osggid in project_to_user_map for username in project_to_user_map[osggid] + ) + all_active_project_users = all_project_users.intersection(ldap_users) + usernames_to_project_map = create_user_to_projects_map( + project_to_user_map, + all_active_project_users, + project_osggids_to_name, + ) + + #if filter_group_name is not None: + #pid_gids = filter_by_group(pid_gids, groups, filter_group_name) + + return usernames_to_project_map def print_usermap_to_file(osguser_groups, file): From 2ffdc5d260958686590a155ab7d61c51a53ce939 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Wed, 22 Nov 2023 12:01:52 -0600 Subject: [PATCH 02/18] Remove f string with no placeholders --- osg-comanage-project-usermap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 668a9fc..8b60738 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -233,7 +233,7 @@ def parse_options(args): def get_ldap_group_members_data(): gidNumber_str = "gidNumber: " gidNumber_regex = re.compile(gidNumber_str) - member_str = f"hasMember: " + member_str = "hasMember: " member_regex = re.compile(member_str) auth_str = subprocess.run( From 3efe8a66bdc749b5010ebb443aceb0d46bf79538 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Mon, 27 Nov 2023 13:55:12 -0600 Subject: [PATCH 03/18] Reimplement filtering by members of a group And clean up of no-longer-used methods remove collections import --- osg-comanage-project-usermap.py | 41 +++++++++------------------------ 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 8b60738..fd02617 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -6,7 +6,6 @@ import json import getopt import subprocess -import collections import urllib.error import urllib.request @@ -28,7 +27,7 @@ "ldaps://ldap.cilogon.org", "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", - "-w", "{}", + "-w", "{auth}", "-b", "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "-s", @@ -42,9 +41,9 @@ "-H", "ldaps://ldap.cilogon.org", "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", "-x", - "-w", "{}", + "-w", "{auth}", "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", - "(isMemberOf=CO:members:active)", "voPersonApplicationUID", + "{filter}", "voPersonApplicationUID", "|", "grep", "voPersonApplicationUID", "|", "sort", ] @@ -242,13 +241,13 @@ def get_ldap_group_members_data(): ).stdout.decode('utf-8').strip() ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND - ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{}")] = auth_str + ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{auth}")] = auth_str data_file = subprocess.run( ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') search_results = list(filter( - lambda x: not re.compile("#|dn|cn|objectClass").match(x), + lambda x: not re.compile("#|dn:|cn:|objectClass:").match(x), (line for line in data_file))) search_results.reverse() @@ -270,14 +269,17 @@ def get_ldap_group_members_data(): return group_data_dict -def get_ldap_active_users(): +def get_ldap_active_users(filter_group_name): auth_str = subprocess.run( LDAP_AUTH_COMMAND, stdout=subprocess.PIPE ).stdout.decode('utf-8').strip() + + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND - ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{}")] = auth_str + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) @@ -308,28 +310,10 @@ def get_co_api_data(): return project_osggids_to_name -def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): - pid_gids = collections.defaultdict(set) - - for gid in gid_pids: - for pid in gid_pids[gid]: - if pid_osguser[pid] is not None: - pid_gids[pid].add(gid) - - return pid_gids - - -def filter_by_group(pid_gids, groups, filter_group_name): - groups_idx = { v: k for k,v in groups.items() } - filter_gid = groups_idx[filter_group_name] # raises KeyError if missing - filter_group_pids = set(get_co_group_members__pids(filter_gid)) - return { p: g for p,g in pid_gids.items() if p in filter_group_pids } - - def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() ldap_groups_members = get_ldap_group_members_data() - ldap_users = get_ldap_active_users() + ldap_users = get_ldap_active_users(filter_group_name) active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) project_to_user_map = { @@ -345,9 +329,6 @@ def get_osguser_groups(filter_group_name=None): all_active_project_users, project_osggids_to_name, ) - - #if filter_group_name is not None: - #pid_gids = filter_by_group(pid_gids, groups, filter_group_name) return usernames_to_project_map From 895775321bcd0e3d1d663face70ba07b38e64692 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Mon, 27 Nov 2023 15:59:28 -0600 Subject: [PATCH 04/18] Add caching for COmanage API data Writes the osggids to project name dict to a file containing the epoch time the cache was made. If the file is found to exist and was made in the past 0.5 hours (by default), the program will read from the cache instead of making COmanage API calls. Otherwise, the program will get the project data from the API and overwrite any existing cache. --- osg-comanage-project-usermap.py | 39 ++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index fd02617..4570dd7 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -4,6 +4,7 @@ import os import sys import json +import time import getopt import subprocess import urllib.error @@ -16,6 +17,8 @@ MINTIMEOUT = 5 MAXTIMEOUT = 625 TIMEOUTMULTIPLE = 5 +CACHE_FILENAME = "COmanage_Projects_cache.txt" +CACHE_LIFETIME_HOURS = 0.5 LDAP_AUTH_COMMAND = [ "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", @@ -275,14 +278,16 @@ def get_ldap_active_users(filter_group_name): stdout=subprocess.PIPE ).stdout.decode('utf-8').strip() - filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None + else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') - users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) + users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") + else "" for line in active_users) return users @@ -299,9 +304,7 @@ def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_na return users_to_projects_map -def get_co_api_data(): - #TODO add cacheing for COManage API data - +def get_groups_data_from_api(): groups = get_osg_co_groups__map() project_osggids_to_name = dict() for id,name in groups.items(): @@ -310,6 +313,32 @@ def get_co_api_data(): return project_osggids_to_name +def get_co_api_data(): + try: + r = open(CACHE_FILENAME, "r") + lines = r.readlines() + if float(lines[0]) >= (time.time() - (60 * 60 * CACHE_LIFETIME_HOURS)): + entries = lines[1:len(lines)] + project_osggids_to_name = dict() + for entry in entries: + osggid_name_pair = entry.split(":") + if len(osggid_name_pair) == 2: + project_osggids_to_name[osggid_name_pair[0]] = osggid_name_pair[1] + else: + raise OSError + except OSError: + with open(CACHE_FILENAME, "w") as w: + project_osggids_to_name = get_groups_data_from_api() + print(time.time(), file=w) + for osggid, name in project_osggids_to_name.items(): + print(f"{osggid}:{name}", file=w) + finally: + if r: + r.close() + + return project_osggids_to_name + + def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() ldap_groups_members = get_ldap_group_members_data() From 82fcc65c7df0f5a73c0c6310c52198a8a00d5ccf Mon Sep 17 00:00:00 2001 From: William Swanson Date: Tue, 5 Dec 2023 15:07:33 -0600 Subject: [PATCH 05/18] Name change and add UNIX cluster group creation --- ...fier_assigner.py => project_group_setup.py | 46 ++++++++++++++++--- 1 file changed, 39 insertions(+), 7 deletions(-) rename group_identifier_assigner.py => project_group_setup.py (82%) mode change 100755 => 100644 diff --git a/group_identifier_assigner.py b/project_group_setup.py old mode 100755 new mode 100644 similarity index 82% rename from group_identifier_assigner.py rename to project_group_setup.py index 7eae705..5f042ba --- a/group_identifier_assigner.py +++ b/project_group_setup.py @@ -11,6 +11,7 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" OSG_CO_ID = 8 +CLUSTER_ID = 10 MINTIMEOUT = 5 MAXTIMEOUT = 625 TIMEOUTMULTIPLE = 5 @@ -23,12 +24,14 @@ OSPOOL_PROJECT_PREFIX_STR = "Yes-" PROJECT_GIDS_START = 200000 + _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] OPTIONS: -u USER[:PASS] specify USER and optionally PASS on command line -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -p CLUSTER_ID specify UNIX Cluster ID (default = {CLUSTER_ID}) -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS -e ENDPOINT specify REST endpoint @@ -58,6 +61,7 @@ class Options: endpoint = ENDPOINT user = "co_8.william_test" osg_co_id = OSG_CO_ID + unix_id = CLUSTER_ID outfile = None authstr = None min_timeout = MINTIMEOUT @@ -202,6 +206,8 @@ def parse_options(args): options.user = arg if op == "-c": options.osg_co_id = int(arg) + if op == "-p": + options.unix_id = int(arg) if op == "-d": passfd = int(arg) if op == "-f": @@ -226,8 +232,14 @@ def main(args): co_groups = get_osg_co_groups()["CoGroups"] highest_osggid = 0 + project_groups = set() projects_to_assign_identifiers = [] + unix_cluster_groups = call_api("unix_cluster/unix_cluster_groups.json", unix_cluster_id=options.unix_id) + clustered_group_ids = set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) + projects_needing_cluster_groups = set() + projects_needing_provisioning = set() + for group in co_groups: gid = group["Id"] identifier_data = get_co_group_identifiers(gid) @@ -238,16 +250,24 @@ def main(args): project_id_index = identifier_index(identifier_list, "ospoolproject") if project_id_index != -1: project_id = str(identifier_list[project_id_index]["Identifier"]) - is_project = re.compile(OSPOOL_PROJECT_PREFIX_STR + "*").match(project_id) is not None - else: - is_project = False + if re.compile(OSPOOL_PROJECT_PREFIX_STR + "*").match(project_id) is not None: + project_groups.add(gid) osggid_index = identifier_index(identifier_list, "osggid") if osggid_index != -1: highest_osggid = max(highest_osggid, int(identifier_list[osggid_index]["Identifier"])) - elif is_project is True: - project_name = project_id.replace(OSPOOL_PROJECT_PREFIX_STR, "", 1).lower() - projects_to_assign_identifiers.append((gid, project_name,)) + + if gid in project_groups: + if osggid_index == -1: + project_name = project_id.replace(OSPOOL_PROJECT_PREFIX_STR, "", 1).lower() + project_data = ( + gid, + project_name, + ) + projects_to_assign_identifiers.append(project_data) + + if not gid in clustered_group_ids: + projects_needing_cluster_groups.add(gid) for gid, project_name in projects_to_assign_identifiers: # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name @@ -257,6 +277,19 @@ def main(args): add_identifier_to_group(gid, type="osggroup", identifier_name=project_name) print(f"project {project_name}: added osggid {osggid_to_assign} and osg project name {project_name}") + for gid in projects_needing_cluster_groups: + request = { + "RequestType": "UnixClusterGroups", + "Version": "1.0", + "UnixClusterGroups": [{"Version": "1.0", "UnixClusterId": options.unix_id, "CoGroupId": gid}], + } + call_api3( + POST, + "unix_cluster/unix_cluster_groups.json", + request, + ) + print(f"project group {gid}: added UNIX Cluster Group") + if __name__ == "__main__": try: @@ -264,4 +297,3 @@ def main(args): except urllib.error.HTTPError as e: print(e, file=sys.stderr) sys.exit(1) - From e94a1593807725f1b132db92329a1fb8dcd0dbfe Mon Sep 17 00:00:00 2001 From: William Swanson Date: Tue, 5 Dec 2023 15:24:27 -0600 Subject: [PATCH 06/18] Add project group LDAP provisioning --- project_group_setup.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/project_group_setup.py b/project_group_setup.py index 5f042ba..f794d9d 100644 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -12,6 +12,7 @@ ENDPOINT = "https://registry-test.cilogon.org/registry/" OSG_CO_ID = 8 CLUSTER_ID = 10 +LDAP_TARGET_ID = 9 MINTIMEOUT = 5 MAXTIMEOUT = 625 TIMEOUTMULTIPLE = 5 @@ -31,7 +32,8 @@ OPTIONS: -u USER[:PASS] specify USER and optionally PASS on command line -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) - -p CLUSTER_ID specify UNIX Cluster ID (default = {CLUSTER_ID}) + -g CLUSTER_ID specify UNIX Cluster ID (default = {CLUSTER_ID}) + -l LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS -e ENDPOINT specify REST endpoint @@ -62,6 +64,7 @@ class Options: user = "co_8.william_test" osg_co_id = OSG_CO_ID unix_id = CLUSTER_ID + provision_target = LDAP_TARGET_ID outfile = None authstr = None min_timeout = MINTIMEOUT @@ -206,8 +209,10 @@ def parse_options(args): options.user = arg if op == "-c": options.osg_co_id = int(arg) - if op == "-p": + if op == "-g": options.unix_id = int(arg) + if op == "-l": + options.provision_target = int(arg) if op == "-d": passfd = int(arg) if op == "-f": @@ -238,7 +243,6 @@ def main(args): unix_cluster_groups = call_api("unix_cluster/unix_cluster_groups.json", unix_cluster_id=options.unix_id) clustered_group_ids = set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) projects_needing_cluster_groups = set() - projects_needing_provisioning = set() for group in co_groups: gid = group["Id"] @@ -290,6 +294,10 @@ def main(args): ) print(f"project group {gid}: added UNIX Cluster Group") + for project_gid in project_groups: + #Provision all project groups + call_api2(POST, f"co_provisioning_targets/provision/{options.provision_target}/cogroupid:{project_gid}.json") + if __name__ == "__main__": try: From 458af3735fd51d297c9f1a05df4e4734d1d80b19 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 7 Dec 2023 11:36:54 -0600 Subject: [PATCH 07/18] project_groups_setup refactoring --- project_group_setup.py | 219 ++++++++++++++++++++++++++--------------- 1 file changed, 142 insertions(+), 77 deletions(-) diff --git a/project_group_setup.py b/project_group_setup.py index f794d9d..7eb5072 100644 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -4,6 +4,7 @@ import re import sys import json +#import ldap3 import getopt import urllib.error import urllib.request @@ -11,7 +12,7 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" OSG_CO_ID = 8 -CLUSTER_ID = 10 +UNIX_CLUSTER_ID = 10 LDAP_TARGET_ID = 9 MINTIMEOUT = 5 MAXTIMEOUT = 625 @@ -32,7 +33,7 @@ OPTIONS: -u USER[:PASS] specify USER and optionally PASS on command line -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) - -g CLUSTER_ID specify UNIX Cluster ID (default = {CLUSTER_ID}) + -g CLUSTER_ID specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID}) -l LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS @@ -63,7 +64,7 @@ class Options: endpoint = ENDPOINT user = "co_8.william_test" osg_co_id = OSG_CO_ID - unix_id = CLUSTER_ID + ucid = UNIX_CLUSTER_ID provision_target = LDAP_TARGET_ID outfile = None authstr = None @@ -129,7 +130,7 @@ def call_api3(method, target, data, **kw): currentTimeout *= TIMEOUTMULTIPLE else: sys.exit( - f"Exception raised after maximum timeout {options.max_timeout} seconds reached. " + f"Exception raised after maximum retrys and timeout {options.max_timeout} seconds reached. " + f"Exception reason: {exception.reason}.\n Request: {req.full_url}" ) @@ -155,29 +156,39 @@ def get_co_person_identifiers(pid): return call_api("identifiers.json", copersonid=pid) +def get_unix_cluster_groups(ucid): + return call_api("unix_cluster/unix_cluster_groups.json", unix_cluster_id=ucid) + + +def get_unix_cluster_groups_ids(ucid): + unix_cluster_groups = get_unix_cluster_groups(ucid) + return set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) + + def get_datalist(data, listname): return data[listname] if data else [] -def identifier_index(id_list, id_type): +def identifier_from_list(id_list, id_type): id_type_list = [id["Type"] for id in id_list] try: - return id_type_list.index(id_type) + id_index = id_type_list.index(id_type) + return id_list[id_index]["Identifier"] except ValueError: - return -1 + return None def identifier_matches(id_list, id_type, regex_string): pattern = re.compile(regex_string) - index = identifier_index(id_list, id_type) - return (index != -1) & (pattern.match(id_list[index]["Identifier"]) is not None) + value = identifier_from_list(id_list, id_type) + return (value is not None) & (pattern.match(value) is not None) -def add_identifier_to_group(gid, type, identifier_name): +def add_identifier_to_group(gid, type, identifier_value): new_identifier_info = { "Version": "1.0", "Type": type, - "Identifier": identifier_name, + "Identifier": identifier_value, "Login": False, "Person": {"Type": "Group", "Id": str(gid)}, "Status": "Active", @@ -187,7 +198,20 @@ def add_identifier_to_group(gid, type, identifier_name): "Version": "1.0", "Identifiers": [new_identifier_info], } - return call_api3(POST, "identifiers.json", data) + call_api3(POST, "identifiers.json", data) + + +def add_unix_cluster_group(gid): + request = { + "RequestType": "UnixClusterGroups", + "Version": "1.0", + "UnixClusterGroups": [{"Version": "1.0", "UnixClusterId": options.ucid, "CoGroupId": gid}], + } + call_api3(POST, "unix_cluster/unix_cluster_groups.json", request) + + +def ldap_provision_group(gid): + call_api2(POST, f"co_provisioning_targets/provision/{options.provision_target}/cogroupid:{gid}.json") def parse_options(args): @@ -210,7 +234,7 @@ def parse_options(args): if op == "-c": options.osg_co_id = int(arg) if op == "-g": - options.unix_id = int(arg) + options.ucid = int(arg) if op == "-l": options.provision_target = int(arg) if op == "-d": @@ -230,78 +254,119 @@ def parse_options(args): options.authstr = mkauthstr(user, passwd) +def append_if_project(project_groups, group): + # If this group has a ospoolproject id, and it starts with "Yes-", it's a project + if identifier_matches(group["ID_List"], "ospoolproject", (OSPOOL_PROJECT_PREFIX_STR + "*")): + # Add a dict of the relavent data for this project to the project_groups list + project_groups.append(group) + + +def update_highest_osggid(highest_osggid, group): + # Get the value of the osggid identifier, if this group has one + osggid = identifier_from_list(group["ID_List"], "osggid") + # If this group has a osggid, keep a hold of the highest one we've seen so far + if osggid is not None: + return max(highest_osggid, int(osggid)) + + +def get_comanage_data(): + comanage_data = {"Projects": [], "highest_osggid": 0} + + co_groups = get_osg_co_groups()["CoGroups"] + for group_data in co_groups: + try: + identifier_list = get_co_group_identifiers(group_data["Id"])["Identifiers"] + # Store this groups data in a dictionary to avoid repeated API calls + group = {"Gid": group_data["Id"], "Name": group_data["Name"], "ID_List": identifier_list} + + append_if_project(comanage_data["Projects"], group) + + comanage_data["highest_osggid"] = update_highest_osggid(comanage_data["highest_osggid"], group) + except TypeError: + pass + return comanage_data + + +def get_projects_to_setup(project_groups): + projects_to_setup = { + "Need Identifiers": [], + "Need Cluster Groups": [], + "Need Provisioning": [], + } + + # CO Groups associated with a UNIX Cluster Group + clustered_group_ids = get_unix_cluster_groups_ids(options.ucid) + + for project in project_groups: + # If this project doesn't have an osggid already assigned to it... + if identifier_from_list(project["ID_List"], "osggid") is None: + # Prep the project to have the proper identifiers added to it + projects_to_setup["Need Identifiers"].append(project) + + # If this project doesn't have a UNIX Cluster Group associated with it... + if not project["Gid"] in clustered_group_ids: + # Prep it to have one made for it and to be provisioned in LDAP + projects_to_setup["Need Cluster Groups"].append(project) + projects_to_setup["Need Provisioning"].append(project) + + return projects_to_setup + + +def add_missing_group_identifier(project, id_type, value): + # If the group doesn't already have an id of this type... + if identifier_from_list(project["ID_List"], id_type) is None: + add_identifier_to_group(project["Gid"], id_type, value) + print(f'project {project["Gid"]}: aded id {value} of type {id_type}') + + +def assign_identifiers_to_project(project, id_dict): + for k, v in id_dict.items(): + # Add an identifier of type k and value v to this group, if it dones't have them already + add_missing_group_identifier(project, k, v) + + +def assign_identifiers(project_list, highest_osggid): + highest = highest_osggid + for project in project_list: + # Project name identifier is the CO Group name in lower case + project_name = project["Name"].lower() + + # Determine what osggid to assign this project, + # based on the starting range and the highest osggid seen in existing groups + osggid_to_assign = max(highest + 1, options.project_gid_startval) + highest = osggid_to_assign + + identifiers_to_add = {"osggid": osggid_to_assign, "osggroup": project_name} + + assign_identifiers_to_project(project, identifiers_to_add) + + +def create_unix_cluster_groups(project_list): + for project in project_list: + add_unix_cluster_group(project["Gid"]) + print(f'project group {project["Gid"]}: added UNIX Cluster Group') + + +def provision_groups(project_list): + for project in project_list: + ldap_provision_group(project["Gid"]) + print(f'project group {project["Gid"]}: Provisioned Group') + + def main(args): parse_options(args) - # get groups with 'OSPool project name' matching "Yes-*" that don't have a 'OSG GID' + comanage_data = get_comanage_data() + projects_to_setup = get_projects_to_setup(comanage_data["Projects"]) - co_groups = get_osg_co_groups()["CoGroups"] - highest_osggid = 0 - project_groups = set() - projects_to_assign_identifiers = [] - - unix_cluster_groups = call_api("unix_cluster/unix_cluster_groups.json", unix_cluster_id=options.unix_id) - clustered_group_ids = set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) - projects_needing_cluster_groups = set() - - for group in co_groups: - gid = group["Id"] - identifier_data = get_co_group_identifiers(gid) - - if identifier_data: - identifier_list = identifier_data["Identifiers"] - - project_id_index = identifier_index(identifier_list, "ospoolproject") - if project_id_index != -1: - project_id = str(identifier_list[project_id_index]["Identifier"]) - if re.compile(OSPOOL_PROJECT_PREFIX_STR + "*").match(project_id) is not None: - project_groups.add(gid) - - osggid_index = identifier_index(identifier_list, "osggid") - if osggid_index != -1: - highest_osggid = max(highest_osggid, int(identifier_list[osggid_index]["Identifier"])) - - if gid in project_groups: - if osggid_index == -1: - project_name = project_id.replace(OSPOOL_PROJECT_PREFIX_STR, "", 1).lower() - project_data = ( - gid, - project_name, - ) - projects_to_assign_identifiers.append(project_data) - - if not gid in clustered_group_ids: - projects_needing_cluster_groups.add(gid) - - for gid, project_name in projects_to_assign_identifiers: - # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name - osggid_to_assign = max(highest_osggid + 1, options.project_gid_startval) - highest_osggid = osggid_to_assign - add_identifier_to_group(gid, type="osggid", identifier_name=osggid_to_assign) - add_identifier_to_group(gid, type="osggroup", identifier_name=project_name) - print(f"project {project_name}: added osggid {osggid_to_assign} and osg project name {project_name}") - - for gid in projects_needing_cluster_groups: - request = { - "RequestType": "UnixClusterGroups", - "Version": "1.0", - "UnixClusterGroups": [{"Version": "1.0", "UnixClusterId": options.unix_id, "CoGroupId": gid}], - } - call_api3( - POST, - "unix_cluster/unix_cluster_groups.json", - request, - ) - print(f"project group {gid}: added UNIX Cluster Group") - - for project_gid in project_groups: - #Provision all project groups - call_api2(POST, f"co_provisioning_targets/provision/{options.provision_target}/cogroupid:{project_gid}.json") + assign_identifiers(projects_to_setup["Need Identifiers"], comanage_data["highest_osggid"]) + create_unix_cluster_groups(projects_to_setup["Need Cluster Groups"]) + provision_groups(projects_to_setup["Need Provisioning"]) if __name__ == "__main__": try: main(sys.argv[1:]) - except urllib.error.HTTPError as e: + except OSError as e: print(e, file=sys.stderr) sys.exit(1) From 085bb8d232c1883a220a257eadd6b9ca87f2ea4c Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 14 Dec 2023 11:30:17 -0600 Subject: [PATCH 08/18] Add ldap search for unprovisioned projects Also, move get_projects_to_setup() functionality into several different methods for clarity. --- project_group_setup.py | 86 +++++++++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/project_group_setup.py b/project_group_setup.py index 7eb5072..2daa576 100644 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -4,13 +4,15 @@ import re import sys import json -#import ldap3 import getopt import urllib.error import urllib.request +from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" +LDAP_SERVER = "ldaps://ldap.cilogon.org" +LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" OSG_CO_ID = 8 UNIX_CLUSTER_ID = 10 LDAP_TARGET_ID = 9 @@ -35,6 +37,7 @@ -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) -g CLUSTER_ID specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID}) -l LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) + -p LDAP authtok specify LDAP server authtok -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS -e ENDPOINT specify REST endpoint @@ -68,6 +71,7 @@ class Options: provision_target = LDAP_TARGET_ID outfile = None authstr = None + ldap_authtok = None min_timeout = MINTIMEOUT max_timeout = MAXTIMEOUT project_gid_startval = PROJECT_GIDS_START @@ -169,6 +173,16 @@ def get_datalist(data, listname): return data[listname] if data else [] +def get_ldap_groups(): + ldap_group_osggids = set() + server = Server(LDAP_SERVER, get_info=ALL) + connection = Connection(server, LDAP_USER, options.ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES) + for group in response: + ldap_group_osggids.add(group["attributes"]["gidNumber"]) + return ldap_group_osggids + + def identifier_from_list(id_list, id_type): id_type_list = [id["Type"] for id in id_list] try: @@ -216,7 +230,7 @@ def ldap_provision_group(gid): def parse_options(args): try: - ops, args = getopt.getopt(args, "u:c:d:f:e:o:t:T:h") + ops, args = getopt.getopt(args, "u:c:g:l:p:d:f:e:o:t:T:h") except getopt.GetoptError: usage() @@ -237,6 +251,8 @@ def parse_options(args): options.ucid = int(arg) if op == "-l": options.provision_target = int(arg) + if op == "-p": + options.ldap_authtok = arg if op == "-d": passfd = int(arg) if op == "-f": @@ -267,6 +283,8 @@ def update_highest_osggid(highest_osggid, group): # If this group has a osggid, keep a hold of the highest one we've seen so far if osggid is not None: return max(highest_osggid, int(osggid)) + else: + return highest_osggid def get_comanage_data(): @@ -287,28 +305,60 @@ def get_comanage_data(): return comanage_data -def get_projects_to_setup(project_groups): - projects_to_setup = { - "Need Identifiers": [], - "Need Cluster Groups": [], - "Need Provisioning": [], - } - - # CO Groups associated with a UNIX Cluster Group - clustered_group_ids = get_unix_cluster_groups_ids(options.ucid) - +def get_projects_needing_identifiers(project_groups): + projects_needing_identifiers = [] for project in project_groups: # If this project doesn't have an osggid already assigned to it... if identifier_from_list(project["ID_List"], "osggid") is None: # Prep the project to have the proper identifiers added to it - projects_to_setup["Need Identifiers"].append(project) + projects_needing_identifiers.append(project) + return projects_needing_identifiers - # If this project doesn't have a UNIX Cluster Group associated with it... - if not project["Gid"] in clustered_group_ids: - # Prep it to have one made for it and to be provisioned in LDAP - projects_to_setup["Need Cluster Groups"].append(project) - projects_to_setup["Need Provisioning"].append(project) +def get_projects_needing_cluster_groups(project_groups): + # CO Groups associated with a UNIX Cluster Group + clustered_group_ids = get_unix_cluster_groups_ids(options.ucid) + try: + # All project Gids + project_gids = set(project["Gid"] for project in project_groups) + # Project Gids for projects without UNIX cluster groups + project_gids_lacking_cluster_groups = project_gids.difference(clustered_group_ids) + # All projects needing UNIX cluster groups + projects_needing_unix_groups = ( + project + for project in project_groups + if project["Gid"] in project_gids_lacking_cluster_groups + ) + return projects_needing_unix_groups + except TypeError: + return set() + + +def get_projects_needing_provisioning(project_groups): + # project groups provisioned in LDAP + ldap_group_osggids = get_ldap_groups() + try: + # All project osggids + project_osggids = set(identifier_from_list(project["ID_List"], "osggid") for project in project_groups) + # project osggids not provisioned in ldap + project_osggids_to_provision = project_osggids.difference(ldap_group_osggids) + # All projects that aren't provisioned in ldap + projects_to_provision = ( + project + for project in project_groups + if identifier_from_list(project["ID_List"], "osggid") in project_osggids_to_provision + ) + return projects_to_provision + except TypeError: + return set() + + +def get_projects_to_setup(project_groups): + projects_to_setup = { + "Need Identifiers": get_projects_needing_identifiers(project_groups), + "Need Cluster Groups": get_projects_needing_cluster_groups(project_groups), + "Need Provisioning": get_projects_needing_provisioning(project_groups), + } return projects_to_setup From 60139cb1d572af6c70dccb8b735a6ea08dfbb391 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 14 Dec 2023 13:16:03 -0600 Subject: [PATCH 09/18] pip install ldap3 inside github CI action --- .github/workflows/python-linters.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-linters.yml b/.github/workflows/python-linters.yml index 5ca1d44..fb459a0 100644 --- a/.github/workflows/python-linters.yml +++ b/.github/workflows/python-linters.yml @@ -36,6 +36,7 @@ jobs: run: | python -m pip install --upgrade pip pip --cache-dir ~/pip-cache install pylint + pip --cache-dir ~/pip-cache install ldap3 - name: Run Pylint env: PYTHON_FILES: ${{ needs.python-files.outputs.filelist }} From 50925c94e22803c6f72a4761559ba5d55f81ae1e Mon Sep 17 00:00:00 2001 From: William Swanson Date: Wed, 20 Dec 2023 11:34:07 -0600 Subject: [PATCH 10/18] Created COManage scripts method library Created a file to hold methods that are, or will likely be, used by multiple scripts in the osg-comanage-rpoject-usermap repo. Refactored existing scripts to use the new library. --- comanage_scripts_utils.py | 220 ++++++++++++++++++++++++++++++++ create_project.py | 133 ++----------------- group_fixup.py | 170 +++--------------------- osg-comanage-project-usermap.py | 107 +++------------- project_group_setup.py | 213 ++++--------------------------- 5 files changed, 289 insertions(+), 554 deletions(-) create mode 100644 comanage_scripts_utils.py diff --git a/comanage_scripts_utils.py b/comanage_scripts_utils.py new file mode 100644 index 0000000..80d1d29 --- /dev/null +++ b/comanage_scripts_utils.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 + +import os +import re +import sys +import json +import urllib.error +import urllib.request +from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC + +MIN_TIMEOUT = 5 +MAX_TIMEOUT = 625 +TIMEOUTMULTIPLE = 5 + +GET = "GET" +PUT = "PUT" +POST = "POST" +DELETE = "DELETE" + + +def getpw(user, passfd, passfile): + if ":" in user: + user, pw = user.split(":", 1) + elif passfd is not None: + pw = os.fdopen(passfd).readline().rstrip("\n") + elif passfile is not None: + pw = open(passfile).readline().rstrip("\n") + elif "PASS" in os.environ: + pw = os.environ["PASS"] + else: + raise PermissionError + #when script needs to say PASS required, raise a permission error + #usage("PASS required") + return user, pw + + +def mkauthstr(user, passwd): + from base64 import encodebytes + raw_authstr = "%s:%s" % (user, passwd) + return encodebytes(raw_authstr.encode()).decode().replace("\n", "") + + +def mkrequest(method, target, data, endpoint, authstr, **kw): + url = os.path.join(endpoint, target) + if kw: + url += "?" + "&".join("{}={}".format(k,v) for k,v in kw.items()) + req = urllib.request.Request(url, json.dumps(data).encode("utf-8")) + req.add_header("Authorization", "Basic %s" % authstr) + req.add_header("Content-Type", "application/json") + req.get_method = lambda: method + return req + + +def call_api(target, endpoint, authstr, **kw): + return call_api2(GET, target, endpoint, authstr, **kw) + + +def call_api2(method, target, endpoint, authstr, **kw): + return call_api3(method, target, data=None, endpoint=endpoint, authstr=authstr, **kw) + + +def call_api3(method, target, data, endpoint, authstr, **kw): + req = mkrequest(method, target, data, endpoint, authstr, **kw) + trying = True + currentTimeout = MIN_TIMEOUT + while trying: + try: + resp = urllib.request.urlopen(req, timeout=currentTimeout) + payload = resp.read() + trying = False + except urllib.error.URLError as exception: + if currentTimeout < MAX_TIMEOUT: + currentTimeout *= TIMEOUTMULTIPLE + else: + sys.exit( + f"Exception raised after maximum retrys and/or timeout {MAX_TIMEOUT} seconds reached. " + + f"Exception reason: {exception.reason}.\n Request: {req.full_url}" + ) + + return json.loads(payload) if payload else None + + +def get_osg_co_groups(osg_co_id, endpoint, authstr): + return call_api("co_groups.json", endpoint, authstr, coid=osg_co_id) + + +def get_co_group_identifiers(gid, endpoint, authstr): + return call_api("identifiers.json", endpoint, authstr, cogroupid=gid) + + +def get_co_group_members(gid, endpoint, authstr): + return call_api("co_group_members.json", endpoint, authstr, cogroupid=gid) + + +def get_co_person_identifiers(pid, endpoint, authstr): + return call_api("identifiers.json", endpoint, authstr, copersonid=pid) + + +def get_co_group(gid, endpoint, authstr): + resp_data = call_api("co_groups/%s.json" % gid, endpoint, authstr) + grouplist = get_datalist(resp_data, "CoGroups") + if not grouplist: + raise RuntimeError("No such CO Group Id: %s" % gid) + return grouplist[0] + + +def get_identifier(id_, endpoint, authstr): + resp_data = call_api("identifiers/%s.json" % id_, endpoint, authstr) + idfs = get_datalist(resp_data, "Identifiers") + if not idfs: + raise RuntimeError("No such Identifier Id: %s" % id_) + return idfs[0] + + +def get_unix_cluster_groups(ucid, endpoint, authstr): + return call_api("unix_cluster/unix_cluster_groups.json", endpoint, authstr, unix_cluster_id=ucid) + + +def get_unix_cluster_groups_ids(ucid, endpoint, authstr): + unix_cluster_groups = get_unix_cluster_groups(ucid, endpoint, authstr) + return set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) + + +def delete_identifier(id_, endpoint, authstr): + return call_api2(DELETE, "identifiers/%s.json" % id_, endpoint, authstr) + + +def get_datalist(data, listname): + return data[listname] if data else [] + + +def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): + ldap_group_osggids = set() + server = Server(ldap_server, get_info=ALL) + connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES) + for group in response: + ldap_group_osggids.add(group["attributes"]["gidNumber"]) + return ldap_group_osggids + + +def identifier_from_list(id_list, id_type): + id_type_list = [id["Type"] for id in id_list] + try: + id_index = id_type_list.index(id_type) + return id_list[id_index]["Identifier"] + except ValueError: + return None + + +def identifier_matches(id_list, id_type, regex_string): + pattern = re.compile(regex_string) + value = identifier_from_list(id_list, id_type) + return (value is not None) & (pattern.match(value) is not None) + + +def rename_co_group(gid, group, newname, endpoint, authstr): + # minimal edit CoGroup Request includes Name+CoId+Status+Version + new_group_info = { + "Name" : newname, + "CoId" : group["CoId"], + "Status" : group["Status"], + "Version" : group["Version"] + } + data = { + "CoGroups" : [new_group_info], + "RequestType" : "CoGroups", + "Version" : "1.0" + } + return call_api3(PUT, "co_groups/%s.json" % gid, data, endpoint, authstr) + + +def add_identifier_to_group(gid, type, identifier_value, endpoint, authstr): + new_identifier_info = { + "Version": "1.0", + "Type": type, + "Identifier": identifier_value, + "Login": False, + "Person": {"Type": "Group", "Id": str(gid)}, + "Status": "Active", + } + data = { + "RequestType": "Identifiers", + "Version": "1.0", + "Identifiers": [new_identifier_info], + } + return call_api3(POST, "identifiers.json", data, endpoint, authstr) + + +def add_unix_cluster_group(gid, ucid, endpoint, authstr): + data = { + "RequestType": "UnixClusterGroups", + "Version": "1.0", + "UnixClusterGroups": [{"Version": "1.0", "UnixClusterId": ucid, "CoGroupId": gid}], + } + return call_api3(POST, "unix_cluster/unix_cluster_groups.json", data, endpoint, authstr) + + +def provision_group(gid, provision_target, endpoint, authstr): + path = f"co_provisioning_targets/provision/{provision_target}/cogroupid:{gid}.json" + data = { + "RequestType" : "CoGroupProvisioning", + "Version" : "1.0", + "Synchronous" : True + } + return call_api3(POST, path, data, endpoint, authstr) + +def provision_group_members(gid, prov_id, endpoint, authstr): + data = { + "RequestType" : "CoPersonProvisioning", + "Version" : "1.0", + "Synchronous" : True + } + responses = {} + for member in get_co_group_members(gid, endpoint, authstr)["CoGroupMembers"]: + if member["Person"]["Type"] == "CO": + pid = member["Person"]["Id"] + path = f"co_provisioning_targets/provision/{prov_id}/copersonid:{pid}.json" + responses[pid] = call_api3(POST, path, data, endpoint, authstr) + return responses diff --git a/create_project.py b/create_project.py index fbc28c5..c024916 100755 --- a/create_project.py +++ b/create_project.py @@ -7,6 +7,7 @@ import getopt import urllib.error import urllib.request +import comanage_scripts_utils as utils SCRIPT = os.path.basename(__file__) @@ -14,11 +15,6 @@ USER = "co_7.group_fixup" OSG_CO_ID = 7 -GET = "GET" -PUT = "PUT" -POST = "POST" -DELETE = "DELETE" - _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] COGroupNameOrId ProjectName @@ -63,127 +59,17 @@ class Options: options = Options() -def getpw(user, passfd, passfile): - if ':' in user: - user, pw = user.split(':', 1) - elif passfd is not None: - pw = os.fdopen(passfd).readline().rstrip('\n') - elif passfile is not None: - pw = open(passfile).readline().rstrip('\n') - elif 'PASS' in os.environ: - pw = os.environ['PASS'] - else: - usage("PASS required") - return user, pw - - -def mkauthstr(user, passwd): - from base64 import encodebytes - raw_authstr = '%s:%s' % (user, passwd) - return encodebytes(raw_authstr.encode()).decode().replace('\n', '') - - -def mkrequest(target, **kw): - return mkrequest2(GET, target, **kw) - - -def mkrequest2(method, target, **kw): - return mkrequest3(method, target, data=None, **kw) - - -def mkrequest3(method, target, data, **kw): - url = os.path.join(options.endpoint, target) - if kw: - url += "?" + "&".join( "{}={}".format(k,v) for k,v in kw.items() ) - req = urllib.request.Request(url, json.dumps(data).encode("utf-8")) - req.add_header("Authorization", "Basic %s" % options.authstr) - req.add_header("Content-Type", "application/json") - req.get_method = lambda: method - return req - - -def call_api(target, **kw): - return call_api2(GET, target, **kw) - - -def call_api2(method, target, **kw): - return call_api3(method, target, data=None, **kw) - - -def call_api3(method, target, data, **kw): - req = mkrequest3(method, target, data, **kw) - resp = urllib.request.urlopen(req) - payload = resp.read() - return json.loads(payload) if payload else None - - -# primary api calls - - -def get_osg_co_groups(): - return call_api("co_groups.json", coid=options.osg_co_id) - - -def get_co_group_identifiers(gid): - return call_api("identifiers.json", cogroupid=gid) - - -def get_co_group_members(gid): - return call_api("co_group_members.json", cogroupid=gid) - - -def get_co_person_identifiers(pid): - return call_api("identifiers.json", copersonid=pid) - - -def get_co_group(gid): - resp_data = call_api("co_groups/%s.json" % gid) - grouplist = get_datalist(resp_data, "CoGroups") - if not grouplist: - raise RuntimeError("No such CO Group Id: %s" % gid) - return grouplist[0] - - -def get_identifier(id_): - resp_data = call_api("identifiers/%s.json" % id_) - idfs = get_datalist(resp_data, "Identifiers") - if not idfs: - raise RuntimeError("No such Identifier Id: %s" % id_) - return idfs[0] - - -def get_datalist(data, listname): - return data[listname] if data else [] - - # script-specific functions def add_project_identifier_to_group(gid, project_name): identifier_name = "Yes-%s" % project_name type_ = "ospoolproject" - return add_identifier_to_group(gid, type_, identifier_name) - - -def add_identifier_to_group(gid, type_, identifier_name): - new_identifier_info = { - "Version" : "1.0", - "Type" : type_, - "Identifier" : identifier_name, - "Login" : False, - "Person" : {"Type": "Group", "Id": str(gid)}, - "Status" : "Active" - } - data = { - "RequestType" : "Identifiers", - "Version" : "1.0", - "Identifiers" : [new_identifier_info] - } - return call_api3(POST, "identifiers.json", data) + return utils.add_identifier_to_group(gid, type_, identifier_name, options.endpoint, options.authstr) def gname_to_gid(gname): - resp_data = get_osg_co_groups() - groups = get_datalist(resp_data, "CoGroups") + resp_data = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) + groups = utils.get_datalist(resp_data, "CoGroups") matching = [ g for g in groups if g["Name"] == gname ] if len(matching) > 1: @@ -225,8 +111,11 @@ def parse_options(args): if op == '-f': passfile = arg if op == '-e': options.endpoint = arg - user, passwd = getpw(options.user, passfd, passfile) - options.authstr = mkauthstr(user, passwd) + try: + user, passwd = utils.getpw(options.user, passfd, passfile) + options.authstr = utils.mkauthstr(user, passwd) + except PermissionError: + usage("PASS required") def main(args): @@ -235,7 +124,7 @@ def main(args): if options.gname: options.gid = gname_to_gid(options.gname) else: - options.gname = get_co_group(options.gid)["Name"] + options.gname = utils.get_co_group(options.gid, options.endpoint, options.authstr)["Name"] print('Creating new Identifier for project "%s"\n' 'for CO Group "%s" (%s)' @@ -247,7 +136,7 @@ def main(args): print("Server Response:") print(json.dumps(resp, indent=2, sort_keys=True)) - new_identifier = get_identifier(resp["Id"]) + new_identifier = utils.get_identifier(resp["Id"], options.endpoint, options.authstr) print("") print("New Identifier Object:") print(json.dumps(new_identifier, indent=2, sort_keys=True)) diff --git a/group_fixup.py b/group_fixup.py index e017568..c76d1ed 100755 --- a/group_fixup.py +++ b/group_fixup.py @@ -3,11 +3,11 @@ import os import re import sys -import json import getopt import collections import urllib.error import urllib.request +import comanage_scripts_utils as utils SCRIPT = os.path.basename(__file__) @@ -16,11 +16,6 @@ OSG_CO_ID = 7 LDAP_PROV_ID = 6 -GET = "GET" -PUT = "PUT" -POST = "POST" -DELETE = "DELETE" - _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] @@ -74,102 +69,17 @@ class Options: options = Options() -def getpw(user, passfd, passfile): - if ':' in user: - user, pw = user.split(':', 1) - elif passfd is not None: - pw = os.fdopen(passfd).readline().rstrip('\n') - elif passfile is not None: - pw = open(passfile).readline().rstrip('\n') - elif 'PASS' in os.environ: - pw = os.environ['PASS'] - else: - usage("PASS required") - return user, pw - - -def mkauthstr(user, passwd): - from base64 import encodebytes - raw_authstr = '%s:%s' % (user, passwd) - return encodebytes(raw_authstr.encode()).decode().replace('\n', '') - - -def mkrequest(target, **kw): - return mkrequest2(GET, target, **kw) - - -def mkrequest2(method, target, **kw): - return mkrequest3(method, target, data=None, **kw) - - -def mkrequest3(method, target, data, **kw): - url = os.path.join(options.endpoint, target) - if kw: - url += "?" + "&".join( "{}={}".format(k,v) for k,v in kw.items() ) - req = urllib.request.Request(url, json.dumps(data).encode("utf-8")) - req.add_header("Authorization", "Basic %s" % options.authstr) - req.add_header("Content-Type", "application/json") - req.get_method = lambda: method - return req - - -def call_api(target, **kw): - return call_api2(GET, target, **kw) - - -def call_api2(method, target, **kw): - return call_api3(method, target, data=None, **kw) - - -def call_api3(method, target, data, **kw): - req = mkrequest3(method, target, data, **kw) - resp = urllib.request.urlopen(req) - payload = resp.read() - return json.loads(payload) if payload else None - - -# primary api calls - - -def get_osg_co_groups(): - return call_api("co_groups.json", coid=options.osg_co_id) - - -def get_co_group_identifiers(gid): - return call_api("identifiers.json", cogroupid=gid) - - -def get_co_group_members(gid): - return call_api("co_group_members.json", cogroupid=gid) - - -def get_co_person_identifiers(pid): - return call_api("identifiers.json", copersonid=pid) - - -def get_co_group(gid): - resp_data = call_api("co_groups/%s.json" % gid) - grouplist = get_datalist(resp_data, "CoGroups") - if not grouplist: - raise RuntimeError("No such CO Group Id: %s" % gid) - return grouplist[0] - - -def get_datalist(data, listname): - return data[listname] if data else [] - - # api call results massagers def get_unixcluster_autogroups(): - groups = get_osg_co_groups() + groups = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) return [ g for g in groups["CoGroups"] if "automatically by UnixCluster" in g["Description"] ] def get_misnamed_unixcluster_groups(): - groups = get_osg_co_groups() + groups = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) return [ g for g in groups["CoGroups"] if "UnixCluster Group" in g["Name"] ] @@ -220,7 +130,7 @@ def show_all_unixcluster_groups(): def show_one_unixcluster_group(gid): - group = get_co_group(gid) + group = utils.get_co_group(gid, options.endpoint, options.authstr) show_misnamed_unixcluster_group(group) @@ -231,8 +141,8 @@ def show_misnamed_unixcluster_groups(): def show_group_identifiers(gid): - resp_data = get_co_group_identifiers(gid) - identifiers = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) + identifiers = utils.get_datalist(resp_data, "Identifiers") for i in identifiers: print(' - Identifier {Id}: ({Type}) "{Identifier}"'.format(**i)) @@ -245,69 +155,22 @@ def show_group_identifiers(gid): # fixup functions -def delete_identifier(id_): - return call_api2(DELETE, "identifiers/%s.json" % id_) - - -def rename_co_group(gid, group, newname): - # minimal edit CoGroup Request includes Name+CoId+Status+Version - new_group_info = { - "Name" : newname, - "CoId" : group["CoId"], - "Status" : group["Status"], - "Version" : group["Version"] - } - data = { - "CoGroups" : [new_group_info], - "RequestType" : "CoGroups", - "Version" : "1.0" - } - return call_api3(PUT, "co_groups/%s.json" % gid, data) - - -def provision_group(gid): - prov_id = options.prov_id - path = f"co_provisioning_targets/provision/{prov_id}/cogroupid:{gid}.json" - data = { - "RequestType" : "CoGroupProvisioning", - "Version" : "1.0", - "Synchronous" : True - } - return call_api3(POST, path, data) - - -def provision_group_members(gid): - prov_id = options.prov_id - data = { - "RequestType" : "CoPersonProvisioning", - "Version" : "1.0", - "Synchronous" : True - } - responses = {} - for member in get_co_group_members(gid)["CoGroupMembers"]: - if member["Person"]["Type"] == "CO": - pid = member["Person"]["Id"] - path = f"co_provisioning_targets/provision/{prov_id}/copersonid:{pid}.json" - responses[pid] = call_api3(POST, path, data) - return responses - - def fixup_unixcluster_group(gid): - group = get_co_group(gid) + group = utils.get_co_group(gid, options.endpoint, options.authstr) oldname = group["Name"] newname = get_fixed_unixcluster_group_name(oldname) - resp_data = get_co_group_identifiers(gid) - identifiers = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) + identifiers = utils.get_datalist(resp_data, "Identifiers") ids_to_delete = get_identifiers_to_delete(identifiers) show_misnamed_unixcluster_group(group) if oldname != newname: - rename_co_group(gid, group, newname) + utils.rename_co_group(gid, group, newname, options.endpoint, options.authstr) for id_ in ids_to_delete: - delete_identifier(id_) + utils.delete_identifier(id_, options.endpoint, options.authstr) - provision_group(gid) - provision_group_members(gid) + utils.provision_group(gid, options.prov_id, options.endpoint, options.authstr) + utils.provision_group_members(gid, options.prov_id, options.endpoint, options.authstr) # http errors raise exceptions, so at this point we apparently succeeded print(":thumbsup:") @@ -349,8 +212,11 @@ def parse_options(args): if op == '--fix-all': options.fix_all = True - user, passwd = getpw(options.user, passfd, passfile) - options.authstr = mkauthstr(user, passwd) + try: + user, passwd = utils.getpw(options.user, passfd, passfile) + options.authstr = utils.mkauthstr(user, passwd) + except PermissionError: + usage("PASS required") def main(args): diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 879a255..2da3da5 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -2,19 +2,16 @@ import os import sys -import json import getopt import collections import urllib.error import urllib.request +import comanage_scripts_utils as utils SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" OSG_CO_ID = 8 -MINTIMEOUT = 5 -MAXTIMEOUT = 625 -TIMEOUTMULTIPLE = 5 _usage = f"""\ @@ -29,8 +26,6 @@ (default = {ENDPOINT}) -o outfile specify output file (default: write to stdout) -g filter_group filter users by group name (eg, 'ap1-login') - -t minTimeout set minimum timeout, in seconds, for API call (default to {MINTIMEOUT}) - -T maxTimeout set maximum timeout, in seconds, for API call (default to {MAXTIMEOUT}) -h display this help text PASS for USER is taken from the first of: @@ -55,118 +50,45 @@ class Options: outfile = None authstr = None filtergrp = None - min_timeout = MINTIMEOUT - max_timeout = MAXTIMEOUT options = Options() -def getpw(user, passfd, passfile): - if ':' in user: - user, pw = user.split(':', 1) - elif passfd is not None: - pw = os.fdopen(passfd).readline().rstrip('\n') - elif passfile is not None: - pw = open(passfile).readline().rstrip('\n') - elif 'PASS' in os.environ: - pw = os.environ['PASS'] - else: - usage("PASS required") - return user, pw - - -def mkauthstr(user, passwd): - from base64 import encodebytes - raw_authstr = '%s:%s' % (user, passwd) - return encodebytes(raw_authstr.encode()).decode().replace('\n', '') - - -def mkrequest(target, **kw): - url = os.path.join(options.endpoint, target) - if kw: - url += "?" + "&".join( "{}={}".format(k,v) for k,v in kw.items() ) - req = urllib.request.Request(url) - req.add_header("Authorization", "Basic %s" % options.authstr) - req.get_method = lambda: 'GET' - return req - - -def call_api(target, **kw): - req = mkrequest(target, **kw) - trying = True - currentTimeout = options.min_timeout - while trying: - try: - resp = urllib.request.urlopen(req, timeout=currentTimeout) - payload = resp.read() - trying = False - except urllib.error.URLError as exception: - if (currentTimeout < options.max_timeout): - currentTimeout *= TIMEOUTMULTIPLE - else: - sys.exit(f"Exception raised after maximum timeout {options.max_timeout} seconds reached. " - + f"Exception reason: {exception.reason}.\n Request: {req.full_url}") - - return json.loads(payload) if payload else None - - -def get_osg_co_groups(): - return call_api("co_groups.json", coid=options.osg_co_id) - - -# primary api calls - -def get_co_group_identifiers(gid): - return call_api("identifiers.json", cogroupid=gid) - - -def get_co_group_members(gid): - return call_api("co_group_members.json", cogroupid=gid) - - -def get_co_person_identifiers(pid): - return call_api("identifiers.json", copersonid=pid) - - -def get_datalist(data, listname): - return data[listname] if data else [] - - # api call results massagers def get_osg_co_groups__map(): #print("get_osg_co_groups__map()") - resp_data = get_osg_co_groups() - data = get_datalist(resp_data, "CoGroups") + resp_data = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "CoGroups") return { g["Id"]: g["Name"] for g in data } def co_group_is_ospool(gid): #print(f"co_group_is_ospool({gid})") - resp_data = get_co_group_identifiers(gid) - data = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "Identifiers") return any( i["Type"] == "ospoolproject" for i in data ) def get_co_group_members__pids(gid): #print(f"get_co_group_members__pids({gid})") - resp_data = get_co_group_members(gid) - data = get_datalist(resp_data, "CoGroupMembers") + resp_data = utils.get_co_group_members(gid, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "CoGroupMembers") return [ m["Person"]["Id"] for m in data ] def get_co_person_osguser(pid): #print(f"get_co_person_osguser({pid})") - resp_data = get_co_person_identifiers(pid) - data = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_person_identifiers(pid, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "Identifiers") typemap = { i["Type"]: i["Identifier"] for i in data } return typemap.get("osguser") def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:t:T:h') + ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:h') except getopt.GetoptError: usage() @@ -185,11 +107,12 @@ def parse_options(args): if op == '-e': options.endpoint = arg if op == '-o': options.outfile = arg if op == '-g': options.filtergrp = arg - if op == '-t': options.min_timeout = float(arg) - if op == '-T': options.max_timeout = float(arg) - user, passwd = getpw(options.user, passfd, passfile) - options.authstr = mkauthstr(user, passwd) + try: + user, passwd = utils.getpw(options.user, passfd, passfile) + options.authstr = utils.mkauthstr(user, passwd) + except PermissionError: + usage("PASS required") def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): diff --git a/project_group_setup.py b/project_group_setup.py index 2daa576..a6da3c8 100644 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -1,29 +1,17 @@ #!/usr/bin/env python3 import os -import re import sys -import json import getopt -import urllib.error -import urllib.request -from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC +import comanage_scripts_utils as utils SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" -LDAP_SERVER = "ldaps://ldap.cilogon.org" -LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" +LDAP_SERVER = "ldaps://ldap-test.cilogon.org" +LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" OSG_CO_ID = 8 UNIX_CLUSTER_ID = 10 LDAP_TARGET_ID = 9 -MINTIMEOUT = 5 -MAXTIMEOUT = 625 -TIMEOUTMULTIPLE = 5 - -GET = "GET" -PUT = "PUT" -POST = "POST" -DELETE = "DELETE" OSPOOL_PROJECT_PREFIX_STR = "Yes-" PROJECT_GIDS_START = 200000 @@ -43,8 +31,6 @@ -e ENDPOINT specify REST endpoint (default = {ENDPOINT}) -o outfile specify output file (default: write to stdout) - -t minTimeout set minimum timeout, in seconds, for API call (default to {MINTIMEOUT}) - -T maxTimeout set maximum timeout, in seconds, for API call (default to {MAXTIMEOUT}) -h display this help text PASS for USER is taken from the first of: @@ -72,165 +58,15 @@ class Options: outfile = None authstr = None ldap_authtok = None - min_timeout = MINTIMEOUT - max_timeout = MAXTIMEOUT project_gid_startval = PROJECT_GIDS_START options = Options() -def getpw(user, passfd, passfile): - if ":" in user: - user, pw = user.split(":", 1) - elif passfd is not None: - pw = os.fdopen(passfd).readline().rstrip("\n") - elif passfile is not None: - pw = open(passfile).readline().rstrip("\n") - elif "PASS" in os.environ: - pw = os.environ["PASS"] - else: - usage("PASS required") - return user, pw - - -def mkauthstr(user, passwd): - from base64 import encodebytes - - raw_authstr = "%s:%s" % (user, passwd) - return encodebytes(raw_authstr.encode()).decode().replace("\n", "") - - -def mkrequest(method, target, data, **kw): - url = os.path.join(options.endpoint, target) - if kw: - url += "?" + "&".join("{}={}".format(k, v) for k, v in kw.items()) - req = urllib.request.Request(url, json.dumps(data).encode("utf-8")) - req.add_header("Authorization", "Basic %s" % options.authstr) - req.add_header("Content-Type", "application/json") - req.get_method = lambda: method - return req - - -def call_api(target, **kw): - return call_api2(GET, target, **kw) - - -def call_api2(method, target, **kw): - return call_api3(method, target, data=None, **kw) - - -def call_api3(method, target, data, **kw): - req = mkrequest(method, target, data, **kw) - trying = True - currentTimeout = options.min_timeout - while trying: - try: - resp = urllib.request.urlopen(req, timeout=currentTimeout) - payload = resp.read() - trying = False - except urllib.error.URLError as exception: - if currentTimeout < options.max_timeout: - currentTimeout *= TIMEOUTMULTIPLE - else: - sys.exit( - f"Exception raised after maximum retrys and timeout {options.max_timeout} seconds reached. " - + f"Exception reason: {exception.reason}.\n Request: {req.full_url}" - ) - - return json.loads(payload) if payload else None - - -def get_osg_co_groups(): - return call_api("co_groups.json", coid=options.osg_co_id) - - -# primary api calls - - -def get_co_group_identifiers(gid): - return call_api("identifiers.json", cogroupid=gid) - - -def get_co_group_members(gid): - return call_api("co_group_members.json", cogroupid=gid) - - -def get_co_person_identifiers(pid): - return call_api("identifiers.json", copersonid=pid) - - -def get_unix_cluster_groups(ucid): - return call_api("unix_cluster/unix_cluster_groups.json", unix_cluster_id=ucid) - - -def get_unix_cluster_groups_ids(ucid): - unix_cluster_groups = get_unix_cluster_groups(ucid) - return set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) - - -def get_datalist(data, listname): - return data[listname] if data else [] - - -def get_ldap_groups(): - ldap_group_osggids = set() - server = Server(LDAP_SERVER, get_info=ALL) - connection = Connection(server, LDAP_USER, options.ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES) - for group in response: - ldap_group_osggids.add(group["attributes"]["gidNumber"]) - return ldap_group_osggids - - -def identifier_from_list(id_list, id_type): - id_type_list = [id["Type"] for id in id_list] - try: - id_index = id_type_list.index(id_type) - return id_list[id_index]["Identifier"] - except ValueError: - return None - - -def identifier_matches(id_list, id_type, regex_string): - pattern = re.compile(regex_string) - value = identifier_from_list(id_list, id_type) - return (value is not None) & (pattern.match(value) is not None) - - -def add_identifier_to_group(gid, type, identifier_value): - new_identifier_info = { - "Version": "1.0", - "Type": type, - "Identifier": identifier_value, - "Login": False, - "Person": {"Type": "Group", "Id": str(gid)}, - "Status": "Active", - } - data = { - "RequestType": "Identifiers", - "Version": "1.0", - "Identifiers": [new_identifier_info], - } - call_api3(POST, "identifiers.json", data) - - -def add_unix_cluster_group(gid): - request = { - "RequestType": "UnixClusterGroups", - "Version": "1.0", - "UnixClusterGroups": [{"Version": "1.0", "UnixClusterId": options.ucid, "CoGroupId": gid}], - } - call_api3(POST, "unix_cluster/unix_cluster_groups.json", request) - - -def ldap_provision_group(gid): - call_api2(POST, f"co_provisioning_targets/provision/{options.provision_target}/cogroupid:{gid}.json") - - def parse_options(args): try: - ops, args = getopt.getopt(args, "u:c:g:l:p:d:f:e:o:t:T:h") + ops, args = getopt.getopt(args, "u:c:g:l:p:d:f:e:o:h") except getopt.GetoptError: usage() @@ -261,25 +97,24 @@ def parse_options(args): options.endpoint = arg if op == "-o": options.outfile = arg - if op == "-t": - options.min_timeout = float(arg) - if op == "-T": - options.max_timeout = float(arg) - user, passwd = getpw(options.user, passfd, passfile) - options.authstr = mkauthstr(user, passwd) + try: + user, passwd = utils.getpw(options.user, passfd, passfile) + options.authstr = utils.mkauthstr(user, passwd) + except PermissionError: + usage("PASS required") def append_if_project(project_groups, group): # If this group has a ospoolproject id, and it starts with "Yes-", it's a project - if identifier_matches(group["ID_List"], "ospoolproject", (OSPOOL_PROJECT_PREFIX_STR + "*")): + if utils.identifier_matches(group["ID_List"], "ospoolproject", (OSPOOL_PROJECT_PREFIX_STR + "*")): # Add a dict of the relavent data for this project to the project_groups list project_groups.append(group) def update_highest_osggid(highest_osggid, group): # Get the value of the osggid identifier, if this group has one - osggid = identifier_from_list(group["ID_List"], "osggid") + osggid = utils.identifier_from_list(group["ID_List"], "osggid") # If this group has a osggid, keep a hold of the highest one we've seen so far if osggid is not None: return max(highest_osggid, int(osggid)) @@ -290,12 +125,12 @@ def update_highest_osggid(highest_osggid, group): def get_comanage_data(): comanage_data = {"Projects": [], "highest_osggid": 0} - co_groups = get_osg_co_groups()["CoGroups"] + co_groups = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr)["CoGroups"] for group_data in co_groups: try: - identifier_list = get_co_group_identifiers(group_data["Id"])["Identifiers"] + identifier_list = utils.get_co_group_identifiers(group_data["Id"], options.endpoint, options.authstr) # Store this groups data in a dictionary to avoid repeated API calls - group = {"Gid": group_data["Id"], "Name": group_data["Name"], "ID_List": identifier_list} + group = {"Gid": group_data["Id"], "Name": group_data["Name"], "ID_List": identifier_list["Identifiers"]} append_if_project(comanage_data["Projects"], group) @@ -309,7 +144,7 @@ def get_projects_needing_identifiers(project_groups): projects_needing_identifiers = [] for project in project_groups: # If this project doesn't have an osggid already assigned to it... - if identifier_from_list(project["ID_List"], "osggid") is None: + if utils.identifier_from_list(project["ID_List"], "osggid") is None: # Prep the project to have the proper identifiers added to it projects_needing_identifiers.append(project) return projects_needing_identifiers @@ -317,7 +152,7 @@ def get_projects_needing_identifiers(project_groups): def get_projects_needing_cluster_groups(project_groups): # CO Groups associated with a UNIX Cluster Group - clustered_group_ids = get_unix_cluster_groups_ids(options.ucid) + clustered_group_ids = utils.get_unix_cluster_groups_ids(options.ucid, options.endpoint, options.authstr) try: # All project Gids project_gids = set(project["Gid"] for project in project_groups) @@ -336,17 +171,19 @@ def get_projects_needing_cluster_groups(project_groups): def get_projects_needing_provisioning(project_groups): # project groups provisioned in LDAP - ldap_group_osggids = get_ldap_groups() + ldap_group_osggids = utils.get_ldap_groups(LDAP_SERVER, LDAP_USER, options.ldap_authtok) try: # All project osggids - project_osggids = set(identifier_from_list(project["ID_List"], "osggid") for project in project_groups) + project_osggids = set( + int(utils.identifier_from_list(project["ID_List"], "osggid")) for project in project_groups + ) # project osggids not provisioned in ldap project_osggids_to_provision = project_osggids.difference(ldap_group_osggids) # All projects that aren't provisioned in ldap projects_to_provision = ( project for project in project_groups - if identifier_from_list(project["ID_List"], "osggid") in project_osggids_to_provision + if int(utils.identifier_from_list(project["ID_List"], "osggid")) in project_osggids_to_provision ) return projects_to_provision except TypeError: @@ -364,8 +201,8 @@ def get_projects_to_setup(project_groups): def add_missing_group_identifier(project, id_type, value): # If the group doesn't already have an id of this type... - if identifier_from_list(project["ID_List"], id_type) is None: - add_identifier_to_group(project["Gid"], id_type, value) + if utils.identifier_from_list(project["ID_List"], id_type) is None: + utils.add_identifier_to_group(project["Gid"], id_type, value, options.endpoint, options.authstr) print(f'project {project["Gid"]}: aded id {value} of type {id_type}') @@ -393,13 +230,13 @@ def assign_identifiers(project_list, highest_osggid): def create_unix_cluster_groups(project_list): for project in project_list: - add_unix_cluster_group(project["Gid"]) + utils.add_unix_cluster_group(project["Gid"], options.ucid, options.endpoint, options.authstr) print(f'project group {project["Gid"]}: added UNIX Cluster Group') def provision_groups(project_list): for project in project_list: - ldap_provision_group(project["Gid"]) + utils.provision_group(project["Gid"], options.provision_target, options.endpoint, options.authstr) print(f'project group {project["Gid"]}: Provisioned Group') From 6978619bc748ebc6fb1a8e4fca30680eb9be85b5 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Wed, 20 Dec 2023 11:37:22 -0600 Subject: [PATCH 11/18] Fix provisioning a project that needed an osggid Projects that needed an OSGGID identifier weren't being provisioned if a UNIX cluster group already existed. Added identifier updating after adding missing identifiers and moved adding identifiers to before checking what projects need UNIX cluster groups and provisioning. --- project_group_setup.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/project_group_setup.py b/project_group_setup.py index a6da3c8..41c76cf 100644 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -190,15 +190,6 @@ def get_projects_needing_provisioning(project_groups): return set() -def get_projects_to_setup(project_groups): - projects_to_setup = { - "Need Identifiers": get_projects_needing_identifiers(project_groups), - "Need Cluster Groups": get_projects_needing_cluster_groups(project_groups), - "Need Provisioning": get_projects_needing_provisioning(project_groups), - } - return projects_to_setup - - def add_missing_group_identifier(project, id_type, value): # If the group doesn't already have an id of this type... if utils.identifier_from_list(project["ID_List"], id_type) is None: @@ -208,8 +199,11 @@ def add_missing_group_identifier(project, id_type, value): def assign_identifiers_to_project(project, id_dict): for k, v in id_dict.items(): - # Add an identifier of type k and value v to this group, if it dones't have them already + # Add an identifier of type k and value v to this group, if it doesn't have them already add_missing_group_identifier(project, k, v) + # Update the project object to incldue the new identifiers + new_identifiers = utils.get_co_group_identifiers(project["Gid"], options.endpoint, options.authstr)["Identifiers"] + project["ID_List"] = new_identifiers def assign_identifiers(project_list, highest_osggid): @@ -244,11 +238,17 @@ def main(args): parse_options(args) comanage_data = get_comanage_data() - projects_to_setup = get_projects_to_setup(comanage_data["Projects"]) + projects = comanage_data["Projects"] + highest_current_osggid = comanage_data["highest_osggid"] + + projects_needing_identifiers = get_projects_needing_identifiers(projects) + assign_identifiers(projects_needing_identifiers, highest_current_osggid) + + projects_needing_cluster_groups = get_projects_needing_cluster_groups(projects) + create_unix_cluster_groups(projects_needing_cluster_groups) - assign_identifiers(projects_to_setup["Need Identifiers"], comanage_data["highest_osggid"]) - create_unix_cluster_groups(projects_to_setup["Need Cluster Groups"]) - provision_groups(projects_to_setup["Need Provisioning"]) + projects_needing_provisioning = get_projects_needing_provisioning(projects) + provision_groups(projects_needing_provisioning) if __name__ == "__main__": From f8898d30b9f07b187fab514f0dc7a1bf5d527942 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Wed, 20 Dec 2023 11:58:28 -0600 Subject: [PATCH 12/18] Spelling / wording / whitespace changes No functional changes --- comanage_scripts_utils.py | 4 +++- group_fixup.py | 2 -- osg-comanage-project-usermap.py | 1 - project_group_setup.py | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/comanage_scripts_utils.py b/comanage_scripts_utils.py index 80d1d29..96c1c41 100644 --- a/comanage_scripts_utils.py +++ b/comanage_scripts_utils.py @@ -8,10 +8,12 @@ import urllib.request from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC + MIN_TIMEOUT = 5 MAX_TIMEOUT = 625 TIMEOUTMULTIPLE = 5 + GET = "GET" PUT = "PUT" POST = "POST" @@ -73,7 +75,7 @@ def call_api3(method, target, data, endpoint, authstr, **kw): currentTimeout *= TIMEOUTMULTIPLE else: sys.exit( - f"Exception raised after maximum retrys and/or timeout {MAX_TIMEOUT} seconds reached. " + f"Exception raised after maximum number of retries and/or timeout {MAX_TIMEOUT} seconds reached. " + f"Exception reason: {exception.reason}.\n Request: {req.full_url}" ) diff --git a/group_fixup.py b/group_fixup.py index c76d1ed..ca064c7 100755 --- a/group_fixup.py +++ b/group_fixup.py @@ -151,7 +151,6 @@ def show_group_identifiers(gid): print(' ** Identifier Ids to delete: %s' % ', '.join(ids_to_delete)) - # fixup functions @@ -242,4 +241,3 @@ def main(args): except (RuntimeError, urllib.error.HTTPError) as e: print(e, file=sys.stderr) sys.exit(1) - diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 2da3da5..04b5983 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -173,4 +173,3 @@ def main(args): except urllib.error.HTTPError as e: print(e, file=sys.stderr) sys.exit(1) - diff --git a/project_group_setup.py b/project_group_setup.py index 41c76cf..a574aca 100644 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -194,14 +194,14 @@ def add_missing_group_identifier(project, id_type, value): # If the group doesn't already have an id of this type... if utils.identifier_from_list(project["ID_List"], id_type) is None: utils.add_identifier_to_group(project["Gid"], id_type, value, options.endpoint, options.authstr) - print(f'project {project["Gid"]}: aded id {value} of type {id_type}') + print(f'project {project["Gid"]}: added id {value} of type {id_type}') def assign_identifiers_to_project(project, id_dict): for k, v in id_dict.items(): # Add an identifier of type k and value v to this group, if it doesn't have them already add_missing_group_identifier(project, k, v) - # Update the project object to incldue the new identifiers + # Update the project object to include the new identifiers new_identifiers = utils.get_co_group_identifiers(project["Gid"], options.endpoint, options.authstr)["Identifiers"] project["ID_List"] = new_identifiers From 30227b16aad9029b475ec2296e7a6315aa165e28 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Wed, 20 Dec 2023 13:42:33 -0600 Subject: [PATCH 13/18] Add more comments and remove COManage data dict --- project_group_setup.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/project_group_setup.py b/project_group_setup.py index a574aca..52b0052 100644 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -123,7 +123,8 @@ def update_highest_osggid(highest_osggid, group): def get_comanage_data(): - comanage_data = {"Projects": [], "highest_osggid": 0} + projects_list = [] + highest_osggid = 0 co_groups = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr)["CoGroups"] for group_data in co_groups: @@ -132,12 +133,14 @@ def get_comanage_data(): # Store this groups data in a dictionary to avoid repeated API calls group = {"Gid": group_data["Id"], "Name": group_data["Name"], "ID_List": identifier_list["Identifiers"]} - append_if_project(comanage_data["Projects"], group) + # Add this group to the project list if it's a project, otherwise skip. + append_if_project(projects_list, group) - comanage_data["highest_osggid"] = update_highest_osggid(comanage_data["highest_osggid"], group) + # Update highest_osggid, if this group has an osggid and it's higher than the current highest osggid. + highest_osggid = update_highest_osggid(highest_osggid, group) except TypeError: pass - return comanage_data + return (projects_list, highest_osggid) def get_projects_needing_identifiers(project_groups): @@ -191,8 +194,9 @@ def get_projects_needing_provisioning(project_groups): def add_missing_group_identifier(project, id_type, value): - # If the group doesn't already have an id of this type... + # If the group doesn't already have an id of this type ... if utils.identifier_from_list(project["ID_List"], id_type) is None: + # ... add the identifier to the group utils.add_identifier_to_group(project["Gid"], id_type, value, options.endpoint, options.authstr) print(f'project {project["Gid"]}: added id {value} of type {id_type}') @@ -237,16 +241,23 @@ def provision_groups(project_list): def main(args): parse_options(args) - comanage_data = get_comanage_data() - projects = comanage_data["Projects"] - highest_current_osggid = comanage_data["highest_osggid"] + # Make all of the nessisary calls to COManage's API for the data we'll need to set up projects. + # Projects is a List of dicts with keys Gid, Name, and Identifiers, the project's list of identifiers. + # Highest_current_osggid is the highest OSGGID that's currently assigned to any CO Group. + projects, highest_current_osggid = get_comanage_data() + # From all the project groups in COManage, find the ones that need OSGGIDs or OSG GroupNames, + # then assign them the identifiers that they're missing. projects_needing_identifiers = get_projects_needing_identifiers(projects) assign_identifiers(projects_needing_identifiers, highest_current_osggid) + # From all the project groups in COManage, find the ones that don't have UNIX Cluster Groups, + # then create UNIX Cluster Groups for them. projects_needing_cluster_groups = get_projects_needing_cluster_groups(projects) create_unix_cluster_groups(projects_needing_cluster_groups) + # From all the project groups in COManage, find the ones that aren't already provisioned in LDAP, + # then have COManage provision the project/UNIX Cluster Group in LDAP. projects_needing_provisioning = get_projects_needing_provisioning(projects) provision_groups(projects_needing_provisioning) From a4d64779f58b064ce58a8502b0278bff493b8fe0 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 28 Dec 2023 11:18:55 -0600 Subject: [PATCH 14/18] Initial Commit for the new project-usermap script Uses LDAP search to find active and provisioned groups then compares them to the list project groups in COmanage to determine which LDAP groups and users to build the map out of. --- osg-comanage-project-usermap.py | 152 +++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 12 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 04b5983..7bbfb11 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +import re import os import sys import getopt +import subprocess import collections import urllib.error import urllib.request @@ -13,6 +15,36 @@ ENDPOINT = "https://registry-test.cilogon.org/registry/" OSG_CO_ID = 8 +LDAP_AUTH_COMMAND = [ + "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", +] + +LDAP_GROUP_MEMBERS_COMMAND = [ + "ldapsearch", + "-H", + "ldaps://ldap.cilogon.org", + "-D", + "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", + "-w", "{}", + "-b", + "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", + "-s", + "one", + "(cn=*)", +] + +LDAP_ACTIVE_USERS_COMMAND = [ + "ldapsearch", + "-LLL", + "-H", "ldaps://ldap.cilogon.org", + "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", + "-x", + "-w", "{}", + "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", + "(isMemberOf=CO:members:active)", "voPersonApplicationUID", + "|", "grep", "voPersonApplicationUID", + "|", "sort", +] _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] @@ -64,13 +96,19 @@ def get_osg_co_groups__map(): return { g["Id"]: g["Name"] for g in data } -def co_group_is_ospool(gid): +def co_group_is_project(gid): #print(f"co_group_is_ospool({gid})") resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) data = utils.get_datalist(resp_data, "Identifiers") return any( i["Type"] == "ospoolproject" for i in data ) +def get_co_group_osggid(gid): + resp_data = get_co_group_identifiers(gid) + data = get_datalist(resp_data, "Identifiers") + return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"] + + def get_co_group_members__pids(gid): #print(f"get_co_group_members__pids({gid})") resp_data = utils.get_co_group_members(gid, options.endpoint, options.authstr) @@ -115,6 +153,84 @@ def parse_options(args): usage("PASS required") +def get_ldap_group_members_data(): + gidNumber_str = "gidNumber: " + gidNumber_regex = re.compile(gidNumber_str) + member_str = f"hasMember: " + member_regex = re.compile(member_str) + + auth_str = subprocess.run( + LDAP_AUTH_COMMAND, + stdout=subprocess.PIPE + ).stdout.decode('utf-8').strip() + + ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND + ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{}")] = auth_str + + data_file = subprocess.run( + ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') + + search_results = list(filter( + lambda x: not re.compile("#|dn|cn|objectClass").match(x), + (line for line in data_file))) + + search_results.reverse() + + group_data_dict = dict() + index = 0 + while index < len(search_results) - 1: + while not gidNumber_regex.match(search_results[index]): + index += 1 + gid = search_results[index].replace(gidNumber_str, "") + members_list = [] + while search_results[index] != "": + if member_regex.match(search_results[index]): + members_list.append(search_results[index].replace(member_str, "")) + index += 1 + group_data_dict[gid] = members_list + index += 1 + + return group_data_dict + + +def get_ldap_active_users(): + auth_str = subprocess.run( + LDAP_AUTH_COMMAND, + stdout=subprocess.PIPE + ).stdout.decode('utf-8').strip() + + ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{}")] = auth_str + + active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') + users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) + return users + + +def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names): + users_to_projects_map = dict() + for osggid in project_to_user_map: + for user in project_to_user_map[osggid]: + if user in active_users: + if user not in users_to_projects_map: + users_to_projects_map[user] = [osggids_to_names[osggid]] + else: + users_to_projects_map[user].append(osggids_to_names[osggid]) + + return users_to_projects_map + + +def get_co_api_data(): + #TODO add cacheing for COManage API data + + groups = get_osg_co_groups__map() + project_osggids_to_name = dict() + for id,name in groups.items(): + if co_group_is_project(id): + project_osggids_to_name[get_co_group_osggid(id)] = name + return project_osggids_to_name + + def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): pid_gids = collections.defaultdict(set) @@ -134,17 +250,29 @@ def filter_by_group(pid_gids, groups, filter_group_name): def get_osguser_groups(filter_group_name=None): - groups = get_osg_co_groups__map() - ospool_gids = filter(co_group_is_ospool, groups) - gid_pids = { gid: get_co_group_members__pids(gid) for gid in ospool_gids } - all_pids = set( pid for gid in gid_pids for pid in gid_pids[gid] ) - pid_osguser = { pid: get_co_person_osguser(pid) for pid in all_pids } - pid_gids = gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser) - if filter_group_name is not None: - pid_gids = filter_by_group(pid_gids, groups, filter_group_name) - - return { pid_osguser[pid]: sorted(map(groups.get, gids)) - for pid, gids in pid_gids.items() } + project_osggids_to_name = get_co_api_data() + ldap_groups_members = get_ldap_group_members_data() + ldap_users = get_ldap_active_users() + + active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) + project_to_user_map = { + osggid : ldap_groups_members[osggid] + for osggid in active_project_osggids + } + all_project_users = set( + username for osggid in project_to_user_map for username in project_to_user_map[osggid] + ) + all_active_project_users = all_project_users.intersection(ldap_users) + usernames_to_project_map = create_user_to_projects_map( + project_to_user_map, + all_active_project_users, + project_osggids_to_name, + ) + + #if filter_group_name is not None: + #pid_gids = filter_by_group(pid_gids, groups, filter_group_name) + + return usernames_to_project_map def print_usermap_to_file(osguser_groups, file): From da46112e395731a7dd88c7b220a41436b63efef9 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 28 Dec 2023 11:18:55 -0600 Subject: [PATCH 15/18] Remove f string with no placeholders --- osg-comanage-project-usermap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 7bbfb11..4fb150b 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -156,7 +156,7 @@ def parse_options(args): def get_ldap_group_members_data(): gidNumber_str = "gidNumber: " gidNumber_regex = re.compile(gidNumber_str) - member_str = f"hasMember: " + member_str = "hasMember: " member_regex = re.compile(member_str) auth_str = subprocess.run( From da9f34faf1ec5d93dd52b05bfff80c57da48906a Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 28 Dec 2023 11:18:55 -0600 Subject: [PATCH 16/18] Reimplement filtering by members of a group And clean up of no-longer-used methods remove collections import --- osg-comanage-project-usermap.py | 41 +++++++++------------------------ 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 4fb150b..963e952 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -5,7 +5,6 @@ import sys import getopt import subprocess -import collections import urllib.error import urllib.request import comanage_scripts_utils as utils @@ -25,7 +24,7 @@ "ldaps://ldap.cilogon.org", "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", - "-w", "{}", + "-w", "{auth}", "-b", "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "-s", @@ -39,9 +38,9 @@ "-H", "ldaps://ldap.cilogon.org", "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", "-x", - "-w", "{}", + "-w", "{auth}", "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", - "(isMemberOf=CO:members:active)", "voPersonApplicationUID", + "{filter}", "voPersonApplicationUID", "|", "grep", "voPersonApplicationUID", "|", "sort", ] @@ -165,13 +164,13 @@ def get_ldap_group_members_data(): ).stdout.decode('utf-8').strip() ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND - ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{}")] = auth_str + ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{auth}")] = auth_str data_file = subprocess.run( ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') search_results = list(filter( - lambda x: not re.compile("#|dn|cn|objectClass").match(x), + lambda x: not re.compile("#|dn:|cn:|objectClass:").match(x), (line for line in data_file))) search_results.reverse() @@ -193,14 +192,17 @@ def get_ldap_group_members_data(): return group_data_dict -def get_ldap_active_users(): +def get_ldap_active_users(filter_group_name): auth_str = subprocess.run( LDAP_AUTH_COMMAND, stdout=subprocess.PIPE ).stdout.decode('utf-8').strip() + + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND - ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{}")] = auth_str + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str + ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) @@ -231,28 +233,10 @@ def get_co_api_data(): return project_osggids_to_name -def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser): - pid_gids = collections.defaultdict(set) - - for gid in gid_pids: - for pid in gid_pids[gid]: - if pid_osguser[pid] is not None: - pid_gids[pid].add(gid) - - return pid_gids - - -def filter_by_group(pid_gids, groups, filter_group_name): - groups_idx = { v: k for k,v in groups.items() } - filter_gid = groups_idx[filter_group_name] # raises KeyError if missing - filter_group_pids = set(get_co_group_members__pids(filter_gid)) - return { p: g for p,g in pid_gids.items() if p in filter_group_pids } - - def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() ldap_groups_members = get_ldap_group_members_data() - ldap_users = get_ldap_active_users() + ldap_users = get_ldap_active_users(filter_group_name) active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) project_to_user_map = { @@ -268,9 +252,6 @@ def get_osguser_groups(filter_group_name=None): all_active_project_users, project_osggids_to_name, ) - - #if filter_group_name is not None: - #pid_gids = filter_by_group(pid_gids, groups, filter_group_name) return usernames_to_project_map From 20555800b2b50a25504da44bb84dff80f6f14050 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 28 Dec 2023 11:20:31 -0600 Subject: [PATCH 17/18] Add caching for COmanage API data Writes the osggids to project name dict to a file containing the epoch time the cache was made. If the file is found to exist and was made in the past 0.5 hours (by default), the program will read from the cache instead of making COmanage API calls. Otherwise, the program will get the project data from the API and overwrite any existing cache. --- osg-comanage-project-usermap.py | 43 +++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 963e952..2f641dc 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -3,6 +3,8 @@ import re import os import sys +import json +import time import getopt import subprocess import urllib.error @@ -13,6 +15,11 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" OSG_CO_ID = 8 +MINTIMEOUT = 5 +MAXTIMEOUT = 625 +TIMEOUTMULTIPLE = 5 +CACHE_FILENAME = "COmanage_Projects_cache.txt" +CACHE_LIFETIME_HOURS = 0.5 LDAP_AUTH_COMMAND = [ "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", @@ -198,14 +205,16 @@ def get_ldap_active_users(filter_group_name): stdout=subprocess.PIPE ).stdout.decode('utf-8').strip() - filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None + else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') - users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") else "" for line in active_users) + users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") + else "" for line in active_users) return users @@ -222,9 +231,7 @@ def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_na return users_to_projects_map -def get_co_api_data(): - #TODO add cacheing for COManage API data - +def get_groups_data_from_api(): groups = get_osg_co_groups__map() project_osggids_to_name = dict() for id,name in groups.items(): @@ -233,6 +240,32 @@ def get_co_api_data(): return project_osggids_to_name +def get_co_api_data(): + try: + r = open(CACHE_FILENAME, "r") + lines = r.readlines() + if float(lines[0]) >= (time.time() - (60 * 60 * CACHE_LIFETIME_HOURS)): + entries = lines[1:len(lines)] + project_osggids_to_name = dict() + for entry in entries: + osggid_name_pair = entry.split(":") + if len(osggid_name_pair) == 2: + project_osggids_to_name[osggid_name_pair[0]] = osggid_name_pair[1] + else: + raise OSError + except OSError: + with open(CACHE_FILENAME, "w") as w: + project_osggids_to_name = get_groups_data_from_api() + print(time.time(), file=w) + for osggid, name in project_osggids_to_name.items(): + print(f"{osggid}:{name}", file=w) + finally: + if r: + r.close() + + return project_osggids_to_name + + def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() ldap_groups_members = get_ldap_group_members_data() From 3d78c61039ebb0d46c251856487665e675ef7853 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Fri, 29 Dec 2023 10:26:46 -0600 Subject: [PATCH 18/18] project-usermap requested changes --- comanage_scripts_utils.py | 35 ++++++++- osg-comanage-project-usermap.py | 130 ++++++-------------------------- 2 files changed, 58 insertions(+), 107 deletions(-) diff --git a/comanage_scripts_utils.py b/comanage_scripts_utils.py index 96c1c41..093aaa3 100644 --- a/comanage_scripts_utils.py +++ b/comanage_scripts_utils.py @@ -6,7 +6,7 @@ import json import urllib.error import urllib.request -from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC +from ldap3 import Server, Connection, ALL, SAFE_SYNC MIN_TIMEOUT = 5 @@ -42,6 +42,14 @@ def mkauthstr(user, passwd): return encodebytes(raw_authstr.encode()).decode().replace("\n", "") +def get_ldap_authtok(ldap_authfile): + if ldap_authfile is not None: + ldap_authtok = open(ldap_authfile).readline().rstrip("\n") + else: + raise PermissionError + return ldap_authtok + + def mkrequest(method, target, data, endpoint, authstr, **kw): url = os.path.join(endpoint, target) if kw: @@ -135,12 +143,35 @@ def get_ldap_groups(ldap_server, ldap_user, ldap_authtok): ldap_group_osggids = set() server = Server(ldap_server, get_info=ALL) connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) - _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES) + _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=["gidNumber"]) for group in response: ldap_group_osggids.add(group["attributes"]["gidNumber"]) return ldap_group_osggids +def get_ldap_group_members(ldap_gid, ldap_server, ldap_user, ldap_authtok): + ldap_group_members = set() + server = Server(ldap_server, get_info=ALL) + connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", f"(&(gidNumber={ldap_gid})(cn=*))", attributes=["hasMember"]) + for group in response: + ldap_group_members.update(group["attributes"]["hasMember"]) + return ldap_group_members + + +def get_ldap_active_users(ldap_server, ldap_user, ldap_authtok, filter_group_name=None): + ldap_active_users = set() + filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None + else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") + server = Server(ldap_server, get_info=ALL) + connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True) + _, _, response, _ = connection.search("ou=people,o=OSG,o=CO,dc=cilogon,dc=org", filter_str, attributes=["employeeNumber"]) + for person in response: + # the "employeeNumber" is the person's name in the first.last format + ldap_active_users.add(person["attributes"]["employeeNumber"]) + return ldap_active_users + + def identifier_from_list(id_list, id_type): id_type_list = [id["Type"] for id in id_list] try: diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index cd8e96f..7f6eb67 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,12 +1,9 @@ #!/usr/bin/env python3 -import re import os import sys -import json import time import getopt -import subprocess import urllib.error import urllib.request import comanage_scripts_utils as utils @@ -14,43 +11,12 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" +LDAP_SERVER = "ldaps://ldap-test.cilogon.org" +LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org" OSG_CO_ID = 8 -MINTIMEOUT = 5 -MAXTIMEOUT = 625 -TIMEOUTMULTIPLE = 5 CACHE_FILENAME = "COmanage_Projects_cache.txt" CACHE_LIFETIME_HOURS = 0.5 -LDAP_AUTH_COMMAND = [ - "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf", -] - -LDAP_GROUP_MEMBERS_COMMAND = [ - "ldapsearch", - "-H", - "ldaps://ldap.cilogon.org", - "-D", - "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", - "-w", "{auth}", - "-b", - "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", - "-s", - "one", - "(cn=*)", -] - -LDAP_ACTIVE_USERS_COMMAND = [ - "ldapsearch", - "-LLL", - "-H", "ldaps://ldap.cilogon.org", - "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org", - "-x", - "-w", "{auth}", - "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org", - "{filter}", "voPersonApplicationUID", - "|", "grep", "voPersonApplicationUID", - "|", "sort", -] _usage = f"""\ usage: [PASS=...] {SCRIPT} [OPTIONS] @@ -58,6 +24,9 @@ OPTIONS: -u USER[:PASS] specify USER and optionally PASS on command line -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) + -s LDAP_SERVER specify LDAP server to read data from + -l LDAP_USER specify LDAP user for reading data from LDAP server + -a ldap_authfile specify path to file to open and read LDAP authtok -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS -e ENDPOINT specify REST endpoint @@ -87,6 +56,9 @@ class Options: osg_co_id = OSG_CO_ID outfile = None authstr = None + ldap_server = LDAP_SERVER + ldap_user = LDAP_USER + ldap_authtok = None filtergrp = None @@ -110,14 +82,8 @@ def co_group_is_project(gid): def get_co_group_osggid(gid): - resp_data = get_co_group_identifiers(gid) - data = get_datalist(resp_data, "Identifiers") - return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"] - - -def get_co_group_osggid(gid): - resp_data = get_co_group_identifiers(gid) - data = get_datalist(resp_data, "Identifiers") + resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr) + data = utils.get_datalist(resp_data, "Identifiers") return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"] @@ -138,7 +104,7 @@ def get_co_person_osguser(pid): def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:h') + ops, args = getopt.getopt(args, 'u:c:s:l:a:d:f:g:e:o:h') except getopt.GetoptError: usage() @@ -147,11 +113,15 @@ def parse_options(args): passfd = None passfile = None + ldap_authfile = None for op, arg in ops: if op == '-h': usage() if op == '-u': options.user = arg if op == '-c': options.osg_co_id = int(arg) + if op == '-s': options.ldap_server= arg + if op == '-l': options.ldap_user = arg + if op == '-a': ldap_authfile = arg if op == '-d': passfd = int(arg) if op == '-f': passfile = arg if op == '-e': options.endpoint = arg @@ -161,69 +131,20 @@ def parse_options(args): try: user, passwd = utils.getpw(options.user, passfd, passfile) options.authstr = utils.mkauthstr(user, passwd) + options.ldap_authtok = utils.get_ldap_authtok(ldap_authfile) except PermissionError: usage("PASS required") -def get_ldap_group_members_data(): - gidNumber_str = "gidNumber: " - gidNumber_regex = re.compile(gidNumber_str) - member_str = "hasMember: " - member_regex = re.compile(member_str) - - auth_str = subprocess.run( - LDAP_AUTH_COMMAND, - stdout=subprocess.PIPE - ).stdout.decode('utf-8').strip() - - ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND - ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{auth}")] = auth_str - - data_file = subprocess.run( - ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') - - search_results = list(filter( - lambda x: not re.compile("#|dn:|cn:|objectClass:").match(x), - (line for line in data_file))) - - search_results.reverse() - +def get_ldap_group_members_dict(): group_data_dict = dict() - index = 0 - while index < len(search_results) - 1: - while not gidNumber_regex.match(search_results[index]): - index += 1 - gid = search_results[index].replace(gidNumber_str, "") - members_list = [] - while search_results[index] != "": - if member_regex.match(search_results[index]): - members_list.append(search_results[index].replace(member_str, "")) - index += 1 - group_data_dict[gid] = members_list - index += 1 + for group_gid in utils.get_ldap_groups(options.ldap_server, options.ldap_user, options.ldap_authtok): + group_members = utils.get_ldap_group_members(group_gid, options.ldap_server, options.ldap_user, options.ldap_authtok) + group_data_dict[group_gid] = group_members return group_data_dict -def get_ldap_active_users(filter_group_name): - auth_str = subprocess.run( - LDAP_AUTH_COMMAND, - stdout=subprocess.PIPE - ).stdout.decode('utf-8').strip() - - filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None - else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))") - - ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND - ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str - ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str - - active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n') - users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") - else "" for line in active_users) - return users - - def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names): users_to_projects_map = dict() for osggid in project_to_user_map: @@ -256,8 +177,10 @@ def get_co_api_data(): for entry in entries: osggid_name_pair = entry.split(":") if len(osggid_name_pair) == 2: - project_osggids_to_name[osggid_name_pair[0]] = osggid_name_pair[1] + project_osggids_to_name[int(osggid_name_pair[0])] = osggid_name_pair[1].strip() + r.close() else: + r.close() raise OSError except OSError: with open(CACHE_FILENAME, "w") as w: @@ -265,17 +188,14 @@ def get_co_api_data(): print(time.time(), file=w) for osggid, name in project_osggids_to_name.items(): print(f"{osggid}:{name}", file=w) - finally: - if r: - r.close() return project_osggids_to_name def get_osguser_groups(filter_group_name=None): project_osggids_to_name = get_co_api_data() - ldap_groups_members = get_ldap_group_members_data() - ldap_users = get_ldap_active_users(filter_group_name) + ldap_groups_members = get_ldap_group_members_dict() + ldap_users = utils.get_ldap_active_users(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name) active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys())) project_to_user_map = {