From 72a9cfc2fddfbae1223b54d68b58382eea457439 Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 5 Sep 2024 16:26:52 -0500 Subject: [PATCH 1/3] Add local mapfile merging to osg-comanage-project-usermap Initial pass at adding merging of local mapfiles into the mapfile generated by osg-comanage-project-usermap. Use option -i and provide a comma-separated list of paths/filenames for the local mapfiles you want to be merged in. --- osg-comanage-project-usermap.py | 37 +++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index f065cd7..56458db 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -24,6 +24,7 @@ (default = {ENDPOINT}) -o outfile specify output file (default: write to stdout) -g filter_group filter users by group name (eg, 'ap1-login') + -i inputfiles specify a comma-delineated list of local mapfiles to merge into outfile -h display this help text PASS for USER is taken from the first of: @@ -48,6 +49,7 @@ class Options: outfile = None authstr = None filtergrp = None + inputfiles = [] options = Options() @@ -87,7 +89,7 @@ def get_co_person_osguser(pid): def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:h') + ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:i:h') except getopt.GetoptError: usage() @@ -106,6 +108,7 @@ def parse_options(args): if op == '-e': options.endpoint = arg if op == '-o': options.outfile = arg if op == '-g': options.filtergrp = arg + if op == '-i': options.inputfiles = arg.split(",") try: user, passwd = utils.getpw(options.user, passfd, passfile) @@ -146,6 +149,35 @@ def get_osguser_groups(filter_group_name=None): for pid, gids in pid_gids.items() } +def parse_inputfile(inputfile: str): + user_groupmap = dict() + with open(inputfile) as file: + for line in file: + split_line = line.split() + if split_line[0] == "*" and len(split_line) >= 3: + user_groupmap[split_line[1]] = split_line[2].split(",") + return user_groupmap + + +def merge_maps(maps: list[dict[str, list]]): + while len(maps) > 1: + merge_target = maps[0] + merge_material = maps.pop(1) + for key in merge_material.keys(): + if key in merge_target: + merge_target[key].extend(merge_material[key]) + merge_target[key] = list(set(merge_target[key])) + else: + merge_target[key] = merge_material[key] + return maps[0] + + +def merge_inputfiles(osguser_groups): + maps = [osguser_groups] + for inputfile in options.inputfiles: + maps.append(parse_inputfile(inputfile)) + return merge_maps(maps) + def print_usermap_to_file(osguser_groups, file): for osguser, groups in sorted(osguser_groups.items()): print("* {} {}".format(osguser, ",".join(group.strip() for group in groups)), file=file) @@ -163,7 +195,8 @@ def main(args): parse_options(args) osguser_groups = get_osguser_groups(options.filtergrp) - print_usermap(osguser_groups) + osguser_groups_merged = merge_inputfiles(osguser_groups) + print_usermap(osguser_groups_merged) if __name__ == "__main__": From ffe8ece11e72078b96d1487896485a83c6e2295c Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 5 Sep 2024 16:33:52 -0500 Subject: [PATCH 2/3] Remove python type hints from local mapfile merging methods --- osg-comanage-project-usermap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 56458db..6bcfce5 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -149,7 +149,7 @@ def get_osguser_groups(filter_group_name=None): for pid, gids in pid_gids.items() } -def parse_inputfile(inputfile: str): +def parse_inputfile(inputfile): user_groupmap = dict() with open(inputfile) as file: for line in file: @@ -159,7 +159,7 @@ def parse_inputfile(inputfile: str): return user_groupmap -def merge_maps(maps: list[dict[str, list]]): +def merge_maps(maps): while len(maps) > 1: merge_target = maps[0] merge_material = maps.pop(1) From ccb46b31664b27c133f2b35fa419d743150000cb Mon Sep 17 00:00:00 2001 From: William Swanson Date: Fri, 6 Sep 2024 11:15:16 -0500 Subject: [PATCH 3/3] Feedback from Brian L. and cleanup --- osg-comanage-project-usermap.py | 55 +++++++++++++++++---------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 6bcfce5..295351e 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import sys import getopt import collections @@ -24,7 +25,7 @@ (default = {ENDPOINT}) -o outfile specify output file (default: write to stdout) -g filter_group filter users by group name (eg, 'ap1-login') - -i inputfiles specify a comma-delineated list of local mapfiles to merge into outfile + -l localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile -h display this help text PASS for USER is taken from the first of: @@ -49,7 +50,7 @@ class Options: outfile = None authstr = None filtergrp = None - inputfiles = [] + localmaps = [] options = Options() @@ -89,7 +90,7 @@ def get_co_person_osguser(pid): def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:i:h') + ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:l:h') except getopt.GetoptError: usage() @@ -108,7 +109,7 @@ def parse_options(args): if op == '-e': options.endpoint = arg if op == '-o': options.outfile = arg if op == '-g': options.filtergrp = arg - if op == '-i': options.inputfiles = arg.split(",") + if op == '-l': options.localmaps = arg.split(",") try: user, passwd = utils.getpw(options.user, passfd, passfile) @@ -145,39 +146,36 @@ def get_osguser_groups(filter_group_name=None): if filter_group_name is not None: pid_gids = filter_by_group(pid_gids, groups, filter_group_name) - return { pid_osguser[pid]: sorted(map(groups.get, gids)) + return { pid_osguser[pid]: set(map(groups.get, gids)) for pid, gids in pid_gids.items() } -def parse_inputfile(inputfile): +def parse_localmap(inputfile): user_groupmap = dict() - with open(inputfile) as file: + with open(inputfile, 'r', encoding='utf-8') as file: for line in file: - split_line = line.split() - if split_line[0] == "*" and len(split_line) >= 3: - user_groupmap[split_line[1]] = split_line[2].split(",") + # Split up 3 semantic columns + split_line = line.strip().split(maxsplit=2) + if split_line[0] == "*" and len(split_line) == 3: + line_groups = set(re.split(r'[ ,]+', split_line[2])) + if split_line[1] in user_groupmap: + user_groupmap[split_line[1]] |= line_groups + else: + user_groupmap[split_line[1]] = line_groups return user_groupmap def merge_maps(maps): - while len(maps) > 1: - merge_target = maps[0] - merge_material = maps.pop(1) - for key in merge_material.keys(): - if key in merge_target: - merge_target[key].extend(merge_material[key]) - merge_target[key] = list(set(merge_target[key])) + merged_map = dict() + for projectmap in maps: + for key in projectmap.keys(): + if key in merged_map: + merged_map[key] |= set(projectmap[key]) else: - merge_target[key] = merge_material[key] - return maps[0] + merged_map[key] = set(projectmap[key]) + return merged_map -def merge_inputfiles(osguser_groups): - maps = [osguser_groups] - for inputfile in options.inputfiles: - maps.append(parse_inputfile(inputfile)) - return merge_maps(maps) - def print_usermap_to_file(osguser_groups, file): for osguser, groups in sorted(osguser_groups.items()): print("* {} {}".format(osguser, ",".join(group.strip() for group in groups)), file=file) @@ -195,7 +193,12 @@ def main(args): parse_options(args) osguser_groups = get_osguser_groups(options.filtergrp) - osguser_groups_merged = merge_inputfiles(osguser_groups) + + maps = [osguser_groups] + for localmap in options.localmaps: + maps.append(parse_localmap(localmap)) + osguser_groups_merged = merge_maps(maps) + print_usermap(osguser_groups_merged)