Add support for locales

Support numeric data using locale-specific numeric separators and formating.

Author: silvexis

.github/workflows/test.developer.branches.yaml

@@ -23,7 +23,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
+          python-version: "3.11"
           cache: 'pip'
           cache-dependency-path: |
             **/pyproject.toml
@@ -42,7 +42,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: [ "3.11"]
+        python-version: [ "3.9", "3.10", "3.11"]
 
     needs: [lint]
     steps:

tests/common/test_cli.py

@@ -1,7 +1,7 @@
 #  Copyright (c) 2021-2024 CloudZero, Inc. or its affiliates. All Rights Reserved.
 #  SPDX-License-Identifier: Apache-2.0
 #  Direct all questions to support@cloudzero.com
-import os
+from pathlib import Path
 
 from uca.common.cli import print_uca_sample
 from uca.common.files import load_jsonl
@@ -17,7 +17,7 @@ def test_print_uca_sample():
 
     """
     # load sample uca data
-    uca_sample_data_path = os.path.join(os.path.dirname(__file__), "../data/sample_uca_data.jsonl")
+    uca_sample_data_path = str((Path(__file__).parent / "../data/sample_uca_data.jsonl").resolve())
     uca_data = load_jsonl(uca_sample_data_path)
 
     print("Running test\n\n")

tests/common/test_files.py

@@ -0,0 +1,18 @@
+#  Copyright (c) 2024 CloudZero, Inc. or its affiliates. All Rights Reserved.
+#  SPDX-License-Identifier: Apache-2.0
+#  Direct all questions to support@cloudzero.com
+from pathlib import Path
+
+from uca.common.files import load_data_files
+
+
+def test_load_data_files():
+    """
+    Test to load data files
+    """
+    test_data_file = str((Path(__file__).parent / "../data/test_data.csv").resolve())
+    data_files = load_data_files(test_data_file, "TEXT")
+    assert len(data_files) == 13
+
+    data_files = load_data_files(test_data_file, "CSV")
+    assert len(data_files) == 12

tests/data/test_data.csv

@@ -0,0 +1,13 @@
+timestamp,unit_id,unit_value
+2021-08-01,Sunbank,37
+2021-08-01,SoftwareCorp,17
+2021-08-01,"Parts, Inc.",140
+2021-08-01,Transport Co.,25
+2021-08-01,"WeShipit, Inc.",124
+2021-08-01,CapitalTwo,90
+2021-08-02,Sunbank,45
+2021-08-02,SoftwareCorp,"1,234.03"
+2021-08-02,"Parts, Inc.",12
+2021-08-02,Transport Co.,89
+2021-08-02,"WeShipit, Inc.",77
+2021-08-02,CapitalTwo,934

tests/features/test_generate.py

@@ -3,8 +3,10 @@
 #  Direct all questions to support@cloudzero.com
 
 import datetime
+from pathlib import Path
 
-from uca.features.generate import _render_uca_data
+from uca.common.files import load_data_files
+from uca.features.generate import _render_uca_data, generate_uca
 
 
 def test_filter_nil_unit_values_exact_mode(input_uca_data, input_settings, input_template):
@@ -47,3 +49,28 @@ def test_filter_nil_unit_values_random_mode(input_uca_data, input_settings, inpu
 
     result = _render_uca_data(input_uca_data, input_settings, input_template, input_timestamp)
     assert len(result) == 1
+
+
+def test_generate_uca_data_from_CSV(input_settings, input_template):
+    """
+    Test to generate UCA data
+
+    Args:
+    ----
+        input_uca_data:
+        input_settings:
+        input_template:
+
+    Returns:
+    -------
+        None
+
+    """
+    test_data_file = str((Path(__file__).parent / "../data/test_data.csv").resolve())
+    test_data = load_data_files(test_data_file, "CSV")
+    uca_to_send = generate_uca(None, input_template, input_settings, test_data)
+    assert len(uca_to_send) == 12
+    for x, li in enumerate(uca_to_send):
+        assert test_data[x]["unit_value"] == li["value"]
+        assert test_data[x]["timestamp"] == li["timestamp"]
+        assert test_data[x]["unit_id"] == li["id"]

uca/__version__.py

@@ -2,4 +2,4 @@
 #  SPDX-License-Identifier: Apache-2.0
 #  Direct all questions to support@cloudzero.com
 
-__version__ = "0.7.10"
+__version__ = "0.7.11"

uca/data/test_data.csv

@@ -6,7 +6,7 @@ timestamp,unit_id,unit_value
 2021-08-01,"WeShipit, Inc.",124
 2021-08-01,CapitalTwo,90
 2021-08-02,Sunbank,45
-2021-08-02,SoftwareCorp,234
+2021-08-02,SoftwareCorp,"1,234.03"
 2021-08-02,"Parts, Inc.",12
 2021-08-02,Transport Co.,89
 2021-08-02,"WeShipit, Inc.",77

uca/features/generate.py

@@ -4,6 +4,7 @@
 
 import calendar
 import copy
+import locale
 import sys
 from datetime import timedelta
 from decimal import Decimal
@@ -115,14 +116,32 @@ def _render_uca_data(uca_data, settings, uca_template, timestamp=None):
 
     uca_events = []
     # skipped = 0
+    row_count = 1
+    locale.setlocale(locale.LC_ALL, locale.getlocale())
     for row in uca_data:
         try:
-            if not row[unit_value_header] or Decimal(row[unit_value_header]) <= 0:
+            if not row[unit_value_header]:
                 continue
+
+            # support localized numbers
+            unit_value = locale.delocalize(row[unit_value_header])
+
+            # skip zero or negative values
+            if Decimal(unit_value) <= 0:
+                continue
+
+            row[unit_value_header] = unit_value
+
         except Exception as err:
-            print(f"Error: {err}")
-            print(f"{row[unit_value_header]}")
-            sys.exit(-1)
+            if "Conversion" in str(err):
+                eprint(f"Error processing input data in row {row_count}")
+                eprint(f"  Column: {unit_value_header}")
+                eprint(f"   Value: {row[unit_value_header]}")
+                sys.exit(-1)
+            else:
+                eprint(f"Unexpected error: {err}")
+                eprint(f"{row[unit_value_header]}")
+                sys.exit(-1)
 
         if generate_settings.get("mode") == "random":
             unit_value = preserve_precision(row[unit_value_header], precision)
@@ -182,6 +201,8 @@ def _render_uca_data(uca_data, settings, uca_template, timestamp=None):
             print(row)
             sys.exit(-1)
 
+        row_count += 1
+
     return uca_events
 
 
@@ -224,7 +245,7 @@ def preserve_precision(input_number: (str, int, Decimal), precision: int) -> int
     return int(round_decimal(Decimal(input_number), precision) * precision)
 
 
-def restore_precision(input_number: (str, int, Decimal), precision: int) -> int:
+def restore_precision(input_number: (str, int, Decimal), precision: int) -> Decimal:
     """
     Restore the precision of a number