Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
"numpy>=1.17.0,<2.0",
"requests>=2.0.0",
"tqdm>=4.0.0",
"click>=7.1.2,!=8.0.0", # 8.0.0 blocked due to reported bug
"torch>=1.7.0",
"transformers>4.0,<5.0",
"datasets",
Expand Down
79 changes: 0 additions & 79 deletions src/llmcompressor/utils/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
Common functions for interfacing with python primitives and directories/files.
"""

import ast
import contextlib
import errno
import fnmatch
Expand Down Expand Up @@ -59,7 +58,6 @@
"tensors_export",
"json_to_jsonl",
"deprecation_warning",
"parse_kwarg_tuples",
"is_package_available",
"import_from_path",
"getattr_chain",
Expand Down Expand Up @@ -882,83 +880,6 @@ def deprecation_warning(message: str):
)


def parse_kwarg_tuples(kwargs: tuple) -> Dict:
"""
Convert a tuple of kwargs to a dict of kwargs.
This function is used to enable the click parsing of kwargs.

Example use:
```
@click.command(
context_settings=dict(
ignore_unknown_options=True)
)
@click.argument(...)
@click.option(...)
...
@click.argument("kwargs", nargs=-1, type=click.UNPROCESSED)
def main(..., kwargs):
...
kwargs: Dict[str, Any] = parse_kwarg_tuples(kwargs: Tuple)
```

Example inputs, outputs:
```
input = ('--arg1', 1, 'arg2', 2, '-arg3', 3)
output = parse_kwarg_tuples(input)
output = {'arg1': 1, 'arg2': 2, 'arg3': 3}
```

```
input = ('--arg1', 1, '--args1', 2 , 'arg2', 2, '-arg3', 3)
output = parse_kwarg_tuples(input)
output = {'arg1': [1, 2], 'arg2': 2, 'arg3': 3}
```

:param kwargs: The kwargs to convert. Should be a tuple of alternating
kwargs names and kwargs values e.g.('--arg1', 1, 'arg2', 2, -arg3', 3).
The names can optionally have a '-' or `--` in front of them.
:return: The converted kwargs as a dict.
"""
if len(kwargs) == 0:
return {}
if len(kwargs) % 2 != 0:
raise ValueError(
"kwargs must be a tuple of alternating names and values "
"i.e. the length of kwargs tuple must be even. Received "
f"kwargs: {kwargs}"
)
# names are uneven indices, values are even indices
kwargs_names = kwargs[0::2]
kwargs_values = kwargs[1::2]
# by default kwargs values are strings, so convert them
# to the appropriate type if possible
kwargs_values = list(kwargs_values)
for i, value in enumerate(kwargs_values):
try:
kwargs_values[i] = ast.literal_eval(value)
except Exception as e: # noqa E841
logger.debug(
f"Failed to infer non-string type"
f"from kwarg value: {value}. It will"
f"be left as a string."
)
pass
# remove any '-' or '--' from the names
kwargs_names = [name.lstrip("-") for name in kwargs_names]
processed_kwargs = {}
for kwarg_name, kwarg_value in zip(kwargs_names, kwargs_values):
if kwarg_name in processed_kwargs:
# if the kwarg name is already in the processed kwargs,
# then we should convert the value to a list
if not isinstance(processed_kwargs[kwarg_name], list):
processed_kwargs[kwarg_name] = [processed_kwargs[kwarg_name]]
processed_kwargs[kwarg_name].append(kwarg_value)
else:
processed_kwargs[kwarg_name] = kwarg_value
return processed_kwargs


def is_package_available(
package_name: str,
return_version: bool = False,
Expand Down
6 changes: 0 additions & 6 deletions tests/llmcompressor/utils/test_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
flatten_iterable,
getattr_chain,
interpolate,
parse_kwarg_tuples,
validate_str_iterable,
)

Expand Down Expand Up @@ -93,11 +92,6 @@ def test_interpolate(x_cur, x0, x1, y0, y1, inter_func, out):
assert abs(out - interpolated) < 0.01


def test_pass_kwargs_tuples():
kwargs = parse_kwarg_tuples(("--input_1", 1, "--input_2", "two", "--input_3", "2"))
assert kwargs == dict(input_1=1, input_2="two", input_3=2)


def test_getattr_chain():
base = SimpleNamespace()
base.a = None
Expand Down