Update project_usermap creation script (INF-1060)

osg-comanage-project-usermap.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 
+import re
 import os
 import sys
 import json
+import time
 import getopt
-import collections
+import subprocess
 import urllib.error
 import urllib.request
 
@@ -15,7 +17,39 @@
 MINTIMEOUT = 5
 MAXTIMEOUT = 625
 TIMEOUTMULTIPLE = 5
-
+CACHE_FILENAME = "COmanage_Projects_cache.txt"
+CACHE_LIFETIME_HOURS = 0.5
+
+LDAP_AUTH_COMMAND = [
+    "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf",
+]
+
+LDAP_GROUP_MEMBERS_COMMAND = [
+    "ldapsearch",
+    "-H",
+    "ldaps://ldap.cilogon.org",
+    "-D",
+    "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org",
+    "-w", "{auth}",
+    "-b",
+    "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org",
+    "-s",
+    "one",
+    "(cn=*)",
+]
+
+LDAP_ACTIVE_USERS_COMMAND = [
+    "ldapsearch",
+    "-LLL",
+    "-H", "ldaps://ldap.cilogon.org",
+    "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org",
+    "-x",
+    "-w",  "{auth}",
+    "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org",
+    "{filter}", "voPersonApplicationUID",
+    "|", "grep", "voPersonApplicationUID",
+    "|", "sort",
+]
 
 _usage = f"""\
 usage: [PASS=...] {SCRIPT} [OPTIONS]
@@ -142,13 +176,19 @@ def get_osg_co_groups__map():
     return { g["Id"]: g["Name"] for g in data }
 
 
-def co_group_is_ospool(gid):
+def co_group_is_project(gid):
     #print(f"co_group_is_ospool({gid})")
     resp_data = get_co_group_identifiers(gid)
     data = get_datalist(resp_data, "Identifiers")
     return any( i["Type"] == "ospoolproject" for i in data )
 
 
+def get_co_group_osggid(gid):
+    resp_data = get_co_group_identifiers(gid)
+    data = get_datalist(resp_data, "Identifiers")
+    return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"]
+
+
 def get_co_group_members__pids(gid):
     #print(f"get_co_group_members__pids({gid})")
     resp_data = get_co_group_members(gid)
@@ -192,36 +232,134 @@ def parse_options(args):
     options.authstr = mkauthstr(user, passwd)
 
 
-def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser):
-    pid_gids = collections.defaultdict(set)
+def get_ldap_group_members_data():
+    gidNumber_str = "gidNumber: "
+    gidNumber_regex = re.compile(gidNumber_str)
+    member_str = "hasMember: "
+    member_regex = re.compile(member_str)
 
-    for gid in gid_pids:
-        for pid in gid_pids[gid]:
-            if pid_osguser[pid] is not None:
-                pid_gids[pid].add(gid)
+    auth_str = subprocess.run(
+            LDAP_AUTH_COMMAND,
+            stdout=subprocess.PIPE
+            ).stdout.decode('utf-8').strip()
+
+    ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND
+    ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{auth}")] = auth_str
+
+    data_file = subprocess.run(
+        ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n')
+
+    search_results = list(filter( 
+        lambda x: not re.compile("#|dn:|cn:|objectClass:").match(x),
+        (line for line in data_file)))
+
+    search_results.reverse()
+
+    group_data_dict = dict()
+    index = 0
+    while index < len(search_results) - 1:
+        while not gidNumber_regex.match(search_results[index]):
+            index += 1
+        gid = search_results[index].replace(gidNumber_str, "")
+        members_list = []
+        while search_results[index] != "":
+            if member_regex.match(search_results[index]):
+                members_list.append(search_results[index].replace(member_str, ""))
+            index += 1
+        group_data_dict[gid] = members_list
+        index += 1
+
+    return group_data_dict
+
+
+def get_ldap_active_users(filter_group_name):
+    auth_str = subprocess.run(
+            LDAP_AUTH_COMMAND,
+            stdout=subprocess.PIPE
+            ).stdout.decode('utf-8').strip()
+
+    filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None 
+                  else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))")
+
+    ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND
+    ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str
+    ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str
 
-    return pid_gids
+    active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n')
+    users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") 
+                else "" for line in active_users)
+    return users
 
 
-def filter_by_group(pid_gids, groups, filter_group_name):
-    groups_idx = { v: k for k,v in groups.items() }
-    filter_gid = groups_idx[filter_group_name]  # raises KeyError if missing
-    filter_group_pids = set(get_co_group_members__pids(filter_gid))
-    return { p: g for p,g in pid_gids.items() if p in filter_group_pids }
+def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names):
+    users_to_projects_map = dict()
+    for osggid in project_to_user_map:
+        for user in project_to_user_map[osggid]:
+            if user in active_users:
+                if user not in users_to_projects_map:
+                    users_to_projects_map[user] = [osggids_to_names[osggid]]
+                else:
+                    users_to_projects_map[user].append(osggids_to_names[osggid])
 
+    return users_to_projects_map
 
-def get_osguser_groups(filter_group_name=None):
+
+def get_groups_data_from_api():
     groups = get_osg_co_groups__map()
-    ospool_gids = filter(co_group_is_ospool, groups)
-    gid_pids = { gid: get_co_group_members__pids(gid) for gid in ospool_gids }
-    all_pids = set( pid for gid in gid_pids for pid in gid_pids[gid] )
-    pid_osguser = { pid: get_co_person_osguser(pid) for pid in all_pids }
-    pid_gids = gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser)
-    if filter_group_name is not None:
-        pid_gids = filter_by_group(pid_gids, groups, filter_group_name)
-
-    return { pid_osguser[pid]: sorted(map(groups.get, gids))
-             for pid, gids in pid_gids.items() }
+    project_osggids_to_name = dict()
+    for id,name in groups.items():
+        if co_group_is_project(id):
+            project_osggids_to_name[get_co_group_osggid(id)] = name
+    return project_osggids_to_name
+
+
+def get_co_api_data():
+    try:
+        r = open(CACHE_FILENAME, "r")
+        lines = r.readlines()
+        if float(lines[0]) >= (time.time() - (60 * 60 * CACHE_LIFETIME_HOURS)):
+            entries = lines[1:len(lines)]
+            project_osggids_to_name = dict()
+            for entry in entries:
+                osggid_name_pair = entry.split(":")
+                if len(osggid_name_pair) == 2:
+                    project_osggids_to_name[osggid_name_pair[0]] = osggid_name_pair[1]
+        else:
+            raise OSError
+    except OSError:
+        with open(CACHE_FILENAME, "w") as w:
+            project_osggids_to_name = get_groups_data_from_api()
+            print(time.time(), file=w)
+            for osggid, name in project_osggids_to_name.items():
+                print(f"{osggid}:{name}", file=w)
+    finally:
+        if r:
+            r.close()
+
+    return project_osggids_to_name
+
+
+def get_osguser_groups(filter_group_name=None):
+    project_osggids_to_name = get_co_api_data()
+    ldap_groups_members = get_ldap_group_members_data()
+    ldap_users = get_ldap_active_users(filter_group_name)
+
+    active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys()))
+    project_to_user_map = {
+        osggid : ldap_groups_members[osggid]
+        for osggid in active_project_osggids
+        }
+    all_project_users = set(
+        username for osggid in project_to_user_map for username in project_to_user_map[osggid]
+        )
+    all_active_project_users = all_project_users.intersection(ldap_users)
+    usernames_to_project_map = create_user_to_projects_map(
+                                project_to_user_map,  
+                                all_active_project_users,
+                                project_osggids_to_name,
+                                )
+
+    return usernames_to_project_map
 
 
 def print_usermap_to_file(osguser_groups, file):