|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# Copyright (c) Facebook, Inc. and its affiliates. |
| 3 | +# All rights reserved. |
| 4 | +# |
| 5 | +# This source code is licensed under the BSD-style license found in the |
| 6 | +# LICENSE file in the root directory of this source tree. |
| 7 | + |
| 8 | +import configparser as configparser |
| 9 | +import logging |
| 10 | +from pathlib import Path |
| 11 | +from typing import List, Optional, TextIO |
| 12 | + |
| 13 | +from torchx.schedulers import Scheduler, get_schedulers |
| 14 | +from torchx.specs import RunConfig, get_type_name |
| 15 | + |
| 16 | + |
| 17 | +_NONE = "None" |
| 18 | + |
| 19 | +log: logging.Logger = logging.getLogger(__name__) |
| 20 | + |
| 21 | + |
| 22 | +def _configparser() -> configparser.ConfigParser: |
| 23 | + """ |
| 24 | + Sets up the configparser and returns it. The same config parser |
| 25 | + should be used between dumps() and loads() methods for ser/de compatibility |
| 26 | + """ |
| 27 | + |
| 28 | + config = configparser.ConfigParser() |
| 29 | + # if optionxform is not overridden, configparser will by default lowercase |
| 30 | + # the option keys because it is compatible with Windows INI files |
| 31 | + # which are expected to be parsed case insensitive. |
| 32 | + # override since torchx's runopts are case-sensitive |
| 33 | + # see: https://stackoverflow.com/questions/19359556/configparser-reads-capital-keys-and-make-them-lower-case |
| 34 | + # pyre-ignore[8] |
| 35 | + config.optionxform = lambda option: option |
| 36 | + |
| 37 | + return config |
| 38 | + |
| 39 | + |
| 40 | +def _get_scheduler(name: str) -> Scheduler: |
| 41 | + schedulers = get_schedulers(session_name="_") |
| 42 | + sched = schedulers.get(name) |
| 43 | + if not sched: |
| 44 | + raise ValueError( |
| 45 | + f"`{name}` is not a registered scheduler. Valid scheduler names: {schedulers.keys()}" |
| 46 | + ) |
| 47 | + return sched |
| 48 | + |
| 49 | + |
| 50 | +def dump( |
| 51 | + f: TextIO, schedulers: Optional[List[str]] = None, required_only: bool = False |
| 52 | +) -> None: |
| 53 | + """ |
| 54 | + Dumps a default INI-style config template containing the runopts for the |
| 55 | + given scheduler names into ``f``. If no ``schedulers`` are specified |
| 56 | + dumps all known registered schedulers. |
| 57 | +
|
| 58 | + Optional runopts are pre-filled with their default values. |
| 59 | + Required runopts are set with a ``<FIXME_...>`` placeholder. |
| 60 | + Each scheduler's runopts are written in the section called |
| 61 | + ``[default.scheduler_args.{scheduler_name}]`` (e.g. ``[default.scheduler_args.kubernetes]``) |
| 62 | +
|
| 63 | + To only dump required runopts pass ``required_only=True``. |
| 64 | +
|
| 65 | + Raises a ``ValueError`` if given a scheduler name that is not known |
| 66 | + """ |
| 67 | + |
| 68 | + if schedulers: |
| 69 | + scheds = schedulers |
| 70 | + else: |
| 71 | + scheds = get_schedulers(session_name="_").keys() |
| 72 | + |
| 73 | + config = _configparser() |
| 74 | + for sched_name in scheds: |
| 75 | + sched = _get_scheduler(sched_name) |
| 76 | + |
| 77 | + section = f"default.scheduler_args.{sched_name}" |
| 78 | + config.add_section(section) |
| 79 | + |
| 80 | + for opt_name, opt in sched.run_opts(): |
| 81 | + if opt.is_required: |
| 82 | + val = f"<FIXME_WITH_A_{get_type_name(opt.opt_type)}_VALUE>" |
| 83 | + else: # not required runopts MUST have a default |
| 84 | + if required_only: |
| 85 | + continue |
| 86 | + |
| 87 | + # serialize list elements with `;` delimiter (consistent with torchx cli) |
| 88 | + if opt.opt_type == List[str]: |
| 89 | + # deal with empty or None default lists |
| 90 | + if opt.default: |
| 91 | + # pyre-ignore[6] opt.default type checked already as List[str] |
| 92 | + val = ";".join(opt.default) |
| 93 | + else: |
| 94 | + val = _NONE |
| 95 | + else: |
| 96 | + val = f"{opt.default}" |
| 97 | + |
| 98 | + config.set(section, opt_name, val) |
| 99 | + |
| 100 | + config.write(f, space_around_delimiters=True) |
| 101 | + |
| 102 | + |
| 103 | +def apply(profile: str, scheduler: str, runcfg: RunConfig) -> None: |
| 104 | + """ |
| 105 | + Loads .torchxconfig files from predefined locations according |
| 106 | + to a load hierarchy and applies the loaded configs into the |
| 107 | + given ``runcfg``. The load hierarchy is as follows (in order of precedence): |
| 108 | +
|
| 109 | + #. ``runcfg`` given to this function |
| 110 | + #. configs loaded from ``$HOME/.torchxconfig`` |
| 111 | + #. configs loaded from ``$CWD/.torchxconfig`` |
| 112 | +
|
| 113 | + Note that load hierarchy does NOT overwrite, but rather adds. |
| 114 | + That is, the configs already present in ``runcfg`` are not |
| 115 | + overridden during the load. |
| 116 | + """ |
| 117 | + lookup_dirs = [Path.home(), Path.cwd()] |
| 118 | + |
| 119 | + for d in lookup_dirs: |
| 120 | + configfile = d / ".torchxconfig" |
| 121 | + if configfile.exists(): |
| 122 | + log.info(f"loading configs from {configfile}") |
| 123 | + with open(str(configfile), "r") as f: |
| 124 | + load(profile, scheduler, f, runcfg) |
| 125 | + |
| 126 | + |
| 127 | +def load(profile: str, scheduler: str, f: TextIO, runcfg: RunConfig) -> None: |
| 128 | + """ |
| 129 | + loads the section ``[{profile}.scheduler_args.{scheduler}]`` from the given |
| 130 | + configfile ``f`` (in .INI format) into the provided ``runcfg``, only adding |
| 131 | + configs that are NOT currently in the given ``runcfg`` (e.g. does not |
| 132 | + override existing values in ``runcfg``). If no section is found, does nothing. |
| 133 | + """ |
| 134 | + |
| 135 | + config = _configparser() |
| 136 | + config.read_file(f) |
| 137 | + |
| 138 | + runopts = _get_scheduler(scheduler).run_opts() |
| 139 | + |
| 140 | + section = f"{profile}.scheduler_args.{scheduler}" |
| 141 | + if config.has_section(section): |
| 142 | + for name, value in config.items(section): |
| 143 | + if name in runcfg.cfgs: |
| 144 | + # DO NOT OVERRIDE existing configs |
| 145 | + continue |
| 146 | + |
| 147 | + if value == _NONE: |
| 148 | + # should map to None (not str 'None') |
| 149 | + # this also handles empty or None lists |
| 150 | + runcfg.set(name, None) |
| 151 | + else: |
| 152 | + runopt = runopts.get(name) |
| 153 | + |
| 154 | + if runopt is None: |
| 155 | + log.warning( |
| 156 | + f"`{name} = {value}` was declared in the [{section}] section " |
| 157 | + f" of the config file but is not a runopt of `{scheduler}` scheduler." |
| 158 | + f" Remove the entry from the config file to no longer see this warning" |
| 159 | + ) |
| 160 | + else: |
| 161 | + if runopt.opt_type is bool: |
| 162 | + # need to handle bool specially since str -> bool is based on |
| 163 | + # str emptiness not value (e.g. bool("False") == True) |
| 164 | + runcfg.set(name, config.getboolean(section, name)) |
| 165 | + elif runopt.opt_type is List[str]: |
| 166 | + runcfg.set(name, value.split(";")) |
| 167 | + else: |
| 168 | + # pyre-ignore[29] |
| 169 | + runcfg.set(name, runopt.opt_type(value)) |
0 commit comments