From 06708b0bac2f2939a38f842121ee5c0140c50910 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 25 Sep 2023 21:56:34 +0200 Subject: [PATCH 1/4] initial script --- remove_setup.py | 174 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 remove_setup.py diff --git a/remove_setup.py b/remove_setup.py new file mode 100644 index 00000000000..1c38f137c69 --- /dev/null +++ b/remove_setup.py @@ -0,0 +1,174 @@ +from copy import deepcopy +import collections.abc +from diraccfg import CFG +import difflib +import pprint +import zlib +from DIRAC import initialize + +import typer +from pathlib import Path + + +def merge_json(d, u): + """ + Merge together two json + """ + for k, v in u.items(): + if isinstance(v, collections.abc.Mapping): + d[k] = merge_json(d.get(k, {}), v) + else: + d[k] = v + return d + + +def merge_cs_sections_json(root_section, default_setup, known_vos, name=None, merge_in=None): + """ + Recursively merge the Operation sections + :param root_section: start of the section we are looking at + :param name: The name of the section, just used for printing + :param merge_in: name of the section to merge in (if set, do not merge in the root). Only used for Defaults + """ + print(f"Working on {name}") + # Create a Defaults section if it doesn't exist + if merge_in: + if merge_in not in root_section: + root_section[merge_in] = {} + merge_root = root_section[merge_in] + else: + merge_root = root_section + for section in list(root_section): + # Can only happen when merge_in is Default + if section == merge_in: + continue + elif section == default_setup: + print(f"Merging {default_setup} to {merge_in}") + merge_json(merge_root, root_section[default_setup]) + elif section in known_vos: + print(f"{section} is a vo.") + merge_cs_sections_json(root_section[section], default_setup, known_vos, name=section) + else: + print(f"{section} is unknown, not touching it (other Setup, custom entry...)") + + +def merge_cs_sections_cfg(cs_cfg: CFG, default_setup, known_vos): + """Do the merge of the Operation sections using the CFG object directly + :param cs_cfg: root of the CS + """ + operation_section = cs_cfg.getAsCFG("/Operations") + for section in list(operation_section.getAsDict()): + # Can only happen when merge_in is Default + if section == "Defaults": + continue + if section == default_setup: + print(f"Merging {default_setup} to Defaults") + + merged_section = operation_section["Defaults"].mergeWith(operation_section[default_setup]) + operation_section.deleteKey("Defaults") + operation_section.createNewSection( + "Defaults", + comment="Automatic merging to remove Setups", + contents=merged_section, + ) + + # print(f"Removing {default_setup}") + # operation_section.deleteKey(default_setup) + elif section in known_vos: + vo_section = operation_section[section] + if default_setup in vo_section: + merged_section = vo_section.mergeWith(vo_section[default_setup]) + operation_section.deleteKey(section) + operation_section.createNewSection( + section, + comment="Automatic merging to remove Setups", + contents=merged_section, + ) + + print(f"{section} is a vo.") + + else: + print(f"{section} is unknown, not touching it (other Setup, custom entry...)") + + cs_cfg.deleteKey("/Operations") + cs_cfg.createNewSection("/Operations", comment="Merging", contents=operation_section) + + +def compare_dicts(d1, d2): + """Produces an HTML output of the diff between 2 dicts""" + return difflib.HtmlDiff().make_file( + pprint.pformat(d1).splitlines(), + pprint.pformat(d2).splitlines(), + ) + + +def main(diff_file: Path = "/tmp/diff.html", setup: str = "", execute_update: bool = False): + """ + Remove the Setup sections from the Operations section. + + If --diff-file is specified, dump the html comparison at this location + + If --setup is specified, use it instead of the default one + + If --execute-update is set, it will attempt to actually update the CS + """ + initialize() + from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + + cs_cfg = gConfigurationData.getRemoteCFG() + + original_cs_json = cs_cfg.getAsDict() + + cs_json = cs_cfg.getAsDict() + + default_setup = setup if setup else cs_json["DIRAC"]["DefaultSetup"] + typer.echo(f"Default setup is {default_setup}") + known_vos = cs_json["Registry"]["VO"] + typer.echo(f"Known VOs {known_vos}") + master_cs = cs_json["DIRAC"]["Configuration"]["MasterServer"] + typer.echo(f"MasterCS {master_cs}") + + # First do a merge using JSON + merge_cs_sections_json(cs_json["Operations"], default_setup, known_vos, "Operations", "Defaults") + first_pass_cs_json = deepcopy(cs_json) + # Do it twice, because runing over it twice shouldn't change the result + merge_cs_sections_json(cs_json["Operations"], default_setup, known_vos, "Operations", "Defaults") + # Make sure the output of the first and second pass are the same + assert first_pass_cs_json == cs_json + + ############## + + # Redo the exercise with the CFG object + + merge_cs_sections_cfg(cs_cfg, default_setup, known_vos) + first_pass_cs_cfg = cs_cfg.clone() + merge_cs_sections_cfg(cs_cfg, default_setup, known_vos) + assert first_pass_cs_cfg.getAsDict() == cs_cfg.getAsDict() + + ############## + + # Finally, make sure we get the same thing in json and CFG + + assert cs_cfg.getAsDict() == cs_json + + ############## + + # Produces diff output + + # print(compare_dicts(original_cs_json["Operations"], cs_cfg.getAsDict("/Operations"))) + with open(diff_file, "w") as f: + f.write(compare_dicts(original_cs_json["Operations"], cs_json["Operations"])) + + typer.echo(f"Diff written in {diff_file}") + + if execute_update: + compressed_data = zlib.compress(str(cs_cfg).encode(), 9) + update_res = ConfigurationClient(url=master_cs).commitNewData(compressed_data) + if update_res["OK"]: + typer.echo("Successfuly updated CS") + else: + typer.echo(f"Error updating CS: {update_res['Message']}") + + +if __name__ == "__main__": + typer.run(main) From 516c711752fa1580373744159620c4ac8a20d56f Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 14 Mar 2025 17:16:14 +0100 Subject: [PATCH 2/4] Also move systems conversion --- remove_setup.py | 112 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 95 insertions(+), 17 deletions(-) diff --git a/remove_setup.py b/remove_setup.py index 1c38f137c69..dc3146a848a 100644 --- a/remove_setup.py +++ b/remove_setup.py @@ -1,3 +1,4 @@ +import json from copy import deepcopy import collections.abc from diraccfg import CFG @@ -22,7 +23,7 @@ def merge_json(d, u): return d -def merge_cs_sections_json(root_section, default_setup, known_vos, name=None, merge_in=None): +def merge_cs_sections_operations_json(root_section, default_setup, known_vos, name=None, merge_in=None): """ Recursively merge the Operation sections :param root_section: start of the section we are looking at @@ -46,12 +47,12 @@ def merge_cs_sections_json(root_section, default_setup, known_vos, name=None, me merge_json(merge_root, root_section[default_setup]) elif section in known_vos: print(f"{section} is a vo.") - merge_cs_sections_json(root_section[section], default_setup, known_vos, name=section) + merge_cs_sections_operations_json(root_section[section], default_setup, known_vos, name=section) else: print(f"{section} is unknown, not touching it (other Setup, custom entry...)") -def merge_cs_sections_cfg(cs_cfg: CFG, default_setup, known_vos): +def merge_cs_sections_operations_cfg(cs_cfg: CFG, default_setup, known_vos): """Do the merge of the Operation sections using the CFG object directly :param cs_cfg: root of the CS """ @@ -93,6 +94,40 @@ def merge_cs_sections_cfg(cs_cfg: CFG, default_setup, known_vos): cs_cfg.createNewSection("/Operations", comment="Merging", contents=operation_section) +def merge_cs_sections_systems_json(cs_json, default_setup: str): + """ + Recursively merge the Systems sections + :param root_section: start of the section we are looking at + :param name: The name of the section, just used for printing + :param merge_in: name of the section to merge in (if set, do not merge in the root). Only used for Defaults + """ + for system in list(cs_json["Systems"]): + instance = cs_json["DIRAC"]["Setups"][default_setup][system] + + # print(f"Merging {default_setup} to {merge_in}") + merge_json(cs_json["Systems"][system], cs_json["Systems"][system][instance]) + + +def merge_cs_sections_systems_cfg(cs_cfg: CFG, default_setup: str): + """Do the merge of the System sections using the CFG object directly + :param cs_cfg: root of the CS + """ + all_systems_section = cs_cfg.getAsCFG("/Systems") + for system in list(all_systems_section.getAsDict()): + instance = cs_cfg["DIRAC"]["Setups"][default_setup][system] + + merged_section = all_systems_section[system].mergeWith(all_systems_section[system][instance]) + all_systems_section.deleteKey(system) + all_systems_section.createNewSection( + system, + comment="Automatic merging to remove Setups", + contents=merged_section, + ) + + cs_cfg.deleteKey("/Systems") + cs_cfg.createNewSection("/Systems", comment="Merging", contents=all_systems_section) + + def compare_dicts(d1, d2): """Produces an HTML output of the diff between 2 dicts""" return difflib.HtmlDiff().make_file( @@ -101,21 +136,28 @@ def compare_dicts(d1, d2): ) -def main(diff_file: Path = "/tmp/diff.html", setup: str = "", execute_update: bool = False): +def main(diff_folder: Path = "/tmp/", master_cs_file: Path = "", setup: str = "", execute_update: bool = False): """ - Remove the Setup sections from the Operations section. + Copy the Setup sections from the Operations section into the Default part. + Copy the Instance sections of the Systems in the System directly. - If --diff-file is specified, dump the html comparison at this location + If --diff-folder is specified, dump the html comparison at this location If --setup is specified, use it instead of the default one - If --execute-update is set, it will attempt to actually update the CS + If --execute-update is set, it will attempt to actually update the CS. This does not work if master_cs_file is specified """ - initialize() - from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient - from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData - cs_cfg = gConfigurationData.getRemoteCFG() + if master_cs_file.is_file(): + if execute_update: + raise NotImplementedError("Cannot update the Master CS if starting from a config file !") + cs_cfg = CFG().loadFromFile(master_cs_file) + else: + initialize() + from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + + cs_cfg = gConfigurationData.getRemoteCFG() original_cs_json = cs_cfg.getAsDict() @@ -128,11 +170,13 @@ def main(diff_file: Path = "/tmp/diff.html", setup: str = "", execute_update: bo master_cs = cs_json["DIRAC"]["Configuration"]["MasterServer"] typer.echo(f"MasterCS {master_cs}") + ######### Update the Operation section + # First do a merge using JSON - merge_cs_sections_json(cs_json["Operations"], default_setup, known_vos, "Operations", "Defaults") + merge_cs_sections_operations_json(cs_json["Operations"], default_setup, known_vos, "Operations", "Defaults") first_pass_cs_json = deepcopy(cs_json) # Do it twice, because runing over it twice shouldn't change the result - merge_cs_sections_json(cs_json["Operations"], default_setup, known_vos, "Operations", "Defaults") + merge_cs_sections_operations_json(cs_json["Operations"], default_setup, known_vos, "Operations", "Defaults") # Make sure the output of the first and second pass are the same assert first_pass_cs_json == cs_json @@ -140,9 +184,9 @@ def main(diff_file: Path = "/tmp/diff.html", setup: str = "", execute_update: bo # Redo the exercise with the CFG object - merge_cs_sections_cfg(cs_cfg, default_setup, known_vos) + merge_cs_sections_operations_cfg(cs_cfg, default_setup, known_vos) first_pass_cs_cfg = cs_cfg.clone() - merge_cs_sections_cfg(cs_cfg, default_setup, known_vos) + merge_cs_sections_operations_cfg(cs_cfg, default_setup, known_vos) assert first_pass_cs_cfg.getAsDict() == cs_cfg.getAsDict() ############## @@ -153,13 +197,47 @@ def main(diff_file: Path = "/tmp/diff.html", setup: str = "", execute_update: bo ############## + ######### END Update the Operation section + + ######### Update the System section + + # First do a merge using JSON + merge_cs_sections_systems_json(cs_json, default_setup) + first_pass_cs_json = deepcopy(cs_json) + # Do it twice, because runing over it twice shouldn't change the result + merge_cs_sections_systems_json(cs_json, default_setup) + # Make sure the output of the first and second pass are the same + assert first_pass_cs_json == cs_json + + # Redo the exercise with the CFG object + merge_cs_sections_systems_cfg(cs_cfg, default_setup) + first_pass_cs_cfg = cs_cfg.clone() + merge_cs_sections_systems_cfg(cs_cfg, default_setup) + assert first_pass_cs_cfg.getAsDict() == cs_cfg.getAsDict() + + ############## + + # Finally, make sure we get the same thing in json and CFG + + assert cs_cfg.getAsDict() == cs_json + + ############## + + ######### END Update the System section + # Produces diff output # print(compare_dicts(original_cs_json["Operations"], cs_cfg.getAsDict("/Operations"))) - with open(diff_file, "w") as f: + with open(diff_folder / "diff_operations.html", "w") as f: f.write(compare_dicts(original_cs_json["Operations"], cs_json["Operations"])) - typer.echo(f"Diff written in {diff_file}") + with open(diff_folder / "diff_systems.html", "w") as f: + f.write(compare_dicts(original_cs_json["Systems"], cs_json["Systems"])) + + typer.echo(f"Diff written in {diff_folder}") + cs_cfg.writeToFile(diff_folder / "modifed_cs.cfg") + with open(diff_folder / "modifed_cs.json", "w") as f: + json.dump(cs_json, f, indent=3) if execute_update: compressed_data = zlib.compress(str(cs_cfg).encode(), 9) From aa83c9572561cf94d78c6b1289615dc096987197 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 18 Mar 2025 09:48:07 +0100 Subject: [PATCH 3/4] Merge Andreis script --- remove_setup.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/remove_setup.py b/remove_setup.py index dc3146a848a..1c22c57ad8d 100644 --- a/remove_setup.py +++ b/remove_setup.py @@ -72,12 +72,11 @@ def merge_cs_sections_operations_cfg(cs_cfg: CFG, default_setup, known_vos): contents=merged_section, ) - # print(f"Removing {default_setup}") - # operation_section.deleteKey(default_setup) elif section in known_vos: vo_section = operation_section[section] if default_setup in vo_section: merged_section = vo_section.mergeWith(vo_section[default_setup]) + operation_section.deleteKey(section) operation_section.createNewSection( section, @@ -136,7 +135,12 @@ def compare_dicts(d1, d2): ) -def main(diff_folder: Path = "/tmp/", master_cs_file: Path = "", setup: str = "", execute_update: bool = False): +def main( + diff_folder: Path = "/tmp/", + master_cs_file: Path = "", + setup: str = "", + execute_update: bool = False, +): """ Copy the Setup sections from the Operations section into the Default part. Copy the Instance sections of the Systems in the System directly. From 54ee91024aa87c08ea082a61d7d80ec55c3d4801 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 31 Mar 2025 08:58:01 +0200 Subject: [PATCH 4/4] commit safegard --- remove_setup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/remove_setup.py b/remove_setup.py index 1c22c57ad8d..fa62a7a0c27 100644 --- a/remove_setup.py +++ b/remove_setup.py @@ -171,8 +171,11 @@ def main( typer.echo(f"Default setup is {default_setup}") known_vos = cs_json["Registry"]["VO"] typer.echo(f"Known VOs {known_vos}") - master_cs = cs_json["DIRAC"]["Configuration"]["MasterServer"] - typer.echo(f"MasterCS {master_cs}") + try: + master_cs = cs_json["DIRAC"]["Configuration"]["MasterServer"] + typer.echo(f"MasterCS {master_cs}") + except: + typer.echo(f"No MasterCS, will not be able to commit") ######### Update the Operation section @@ -244,6 +247,8 @@ def main( json.dump(cs_json, f, indent=3) if execute_update: + if not master_cs: + typer.echo("No master CS found, not updating", error=True) compressed_data = zlib.compress(str(cs_cfg).encode(), 9) update_res = ConfigurationClient(url=master_cs).commitNewData(compressed_data) if update_res["OK"]: