From b9d88e61a82457b88a54a996ff0ece7f7e368fca Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Sat, 20 Dec 2025 15:55:45 +0000 Subject: [PATCH 01/28] fileutils unit tests --- syscore/tests/test_fileutils.py | 130 ++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 syscore/tests/test_fileutils.py diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py new file mode 100644 index 00000000000..b86d5a2a406 --- /dev/null +++ b/syscore/tests/test_fileutils.py @@ -0,0 +1,130 @@ +import pytest +import sys +from pathlib import Path +from syscore.fileutils import ( + resolve_path_and_filename_for_package, + get_resolved_pathname, +) + + +@pytest.fixture() +def project_dir(request): + module_path = Path(request.module.__file__) + return str(module_path.parent.parent.parent.absolute()) + + +@pytest.mark.skipif(sys.platform.startswith("win"), reason="Only runs on unix") +class TestFileUtilsUnix: + def test_resolve_path_absolute(self): + actual = get_resolved_pathname("/home/rob") + assert actual == "/home/rob" + + def test_resolve_path_absolute_trailing(self): + actual = get_resolved_pathname("/home/rob/") + assert actual == "/home/rob" + + def test_resolve_path_absolute_dotted(self): + actual = get_resolved_pathname(".home.rob") + assert actual == "/home/rob" + + def test_resolve_path_relative(self, project_dir): + actual = get_resolved_pathname("syscore.tests") + assert actual == f"{project_dir}/syscore/tests" + + def test_resolve_path_non_existent(self, project_dir): + actual = get_resolved_pathname("syscore.testz") + assert actual == f"{project_dir}/syscore/testz" + + def test_resolve_path_and_filename_for_package(self): + actual = resolve_path_and_filename_for_package("/home/rob/", "file.csv") + assert actual == "/home/rob/file.csv" + + actual = resolve_path_and_filename_for_package("/home/rob/file.csv") + assert actual == "/home/rob/file.csv" + + actual = resolve_path_and_filename_for_package(".home.rob.file.csv") + assert actual == "/home/rob/file.csv" + + def test_path_and_filename_for_package_modules(self, project_dir): + actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv") + assert actual == f"{project_dir}/syscore/tests/file.csv" + + actual = resolve_path_and_filename_for_package("syscore.tests.file.csv") + assert actual == f"{project_dir}/syscore/tests/file.csv" + + @pytest.mark.xfail(reason="Cannot work with current implementation") + def test_resolve_dotted_dir_name(self, tmp_path): + directory = tmp_path / "dir.name.with.dots" + directory.mkdir() + file = directory / "hello.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(str(file)) + assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" + + @pytest.mark.xfail(reason="Cannot work with current implementation") + def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( + self, tmp_path + ): + directory = tmp_path / "dir.name.with.dots" + directory.mkdir() + file = directory / "hello.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}/dir.name.with.dots", "hello.txt" + ) + assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" + + @pytest.mark.xfail(reason="Cannot work with current implementation") + def test_resolve_dotted_file_name(self, tmp_path): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(str(file)) + assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + + @pytest.mark.xfail(reason="Cannot work with current implementation") + def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( + self, tmp_path + ): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}/dir_name/dotted.filename.txt" + ) + assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + + +@pytest.mark.skipif(sys.platform in ["linux", "darwin"], reason="Only runs on windows") +class TestFileUtilsWindoze: + def test_resolve_path_absolute(self): + actual = get_resolved_pathname("C:\\home\\rob\\") + assert actual == "C:\\home\\rob" + + def test_resolve_path_absolute_trailing(self): + actual = get_resolved_pathname("C:\\home\\rob\\") + assert actual == "C:\\home\\rob" + + def test_resolve_path_absolute_dotted(self): + actual = get_resolved_pathname(".home.rob") + assert actual == "\\home\\rob" + + def test_resolve_path_relative(self, project_dir): + actual = get_resolved_pathname("syscore.tests") + assert actual == f"{project_dir}\\syscore\\tests" + + def test_resolve_path_and_filename_for_package(self): + actual = resolve_path_and_filename_for_package("C:\\home\\rob\\", "file.csv") + assert actual == "C:\\home\\rob\\file.csv" + + actual = resolve_path_and_filename_for_package("C:\\home\\rob\\file.csv") + assert actual == "C:\\home\\rob\\file.csv" + + def test_path_and_filename_for_package_modules(self, project_dir): + actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv") + assert actual == f"{project_dir}\\syscore\\tests\\file.csv" + + actual = resolve_path_and_filename_for_package("syscore.tests.file.csv") + assert actual == f"{project_dir}\\syscore\\tests\\file.csv" From a4541c0d9e19d969bb9d9a13a667f426ff90548c Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Sat, 20 Dec 2025 16:01:56 +0000 Subject: [PATCH 02/28] removing unused import --- sysdata/config/configdata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sysdata/config/configdata.py b/sysdata/config/configdata.py index e3c7b6718d9..70c9be923b9 100644 --- a/sysdata/config/configdata.py +++ b/sysdata/config/configdata.py @@ -13,7 +13,6 @@ """ from pathlib import Path -import os import yaml From f4e791458212296c727ba59ff854e85c79d465ce Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Tue, 23 Dec 2025 10:12:03 +0000 Subject: [PATCH 03/28] replace os.path with pathlib --- sysdata/parquet/parquet_access.py | 14 ++++++-------- .../parquet_futures_per_contract_prices.py | 2 +- .../futures/barchart_futures_contract_prices.py | 8 ++++---- syslogging/logger.py | 3 ++- sysproduction/backup_db_to_csv.py | 8 ++++---- sysproduction/data/backtest.py | 15 ++++++--------- sysproduction/reporting/reporting_functions.py | 11 ++++++----- 7 files changed, 29 insertions(+), 32 deletions(-) diff --git a/sysdata/parquet/parquet_access.py b/sysdata/parquet/parquet_access.py index 990d7d34f9f..c16e3cc8f8b 100644 --- a/sysdata/parquet/parquet_access.py +++ b/sysdata/parquet/parquet_access.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path import pandas as pd from syscore.exceptions import missingFile from syscore.fileutils import ( @@ -19,13 +19,13 @@ def get_all_identifiers_with_data_type(self, data_type: str): path = self._get_pathname_given_data_type(data_type) return files_with_extension_in_pathname(path, extension=EXTENSION) - def does_idenitifier_with_data_type_exist( + def does_identifier_with_data_type_exist( self, data_type: str, identifier: str ) -> bool: filename = self._get_filename_given_data_type_and_identifier( data_type=data_type, identifier=identifier ) - return os.path.isfile(filename) + return Path(filename).exists() def delete_data_given_data_type_and_identifier( self, data_type: str, identifier: str @@ -34,7 +34,7 @@ def delete_data_given_data_type_and_identifier( data_type=data_type, identifier=identifier ) try: - os.remove(filename) + Path(filename).unlink() except FileNotFoundError: raise missingFile(f"File '{filename}' does not exist") @@ -66,7 +66,5 @@ def _get_filename_given_data_type_and_identifier( def _get_pathname_given_data_type(self, data_type: str): root = self.parquet_store - path = os.path.join(root, data_type) - Path(path).mkdir(parents=True, exist_ok=True) - - return path + path = Path(root, data_type).mkdir(parents=True, exist_ok=True) + return str(path) diff --git a/sysdata/parquet/parquet_futures_per_contract_prices.py b/sysdata/parquet/parquet_futures_per_contract_prices.py index 6fa5566e768..01c595f5d01 100644 --- a/sysdata/parquet/parquet_futures_per_contract_prices.py +++ b/sysdata/parquet/parquet_futures_per_contract_prices.py @@ -148,7 +148,7 @@ def has_price_data_for_contract_at_frequency( self, contract_object: futuresContract, frequency: Frequency ) -> bool: ident = from_contract_and_freq_to_key(contract_object, frequency=frequency) - return self.parquet.does_idenitifier_with_data_type_exist( + return self.parquet.does_identifier_with_data_type_exist( data_type=CONTRACT_COLLECTION, identifier=ident ) diff --git a/sysinit/futures/barchart_futures_contract_prices.py b/sysinit/futures/barchart_futures_contract_prices.py index 4147f7f2f6a..62c4a4cc0e3 100644 --- a/sysinit/futures/barchart_futures_contract_prices.py +++ b/sysinit/futures/barchart_futures_contract_prices.py @@ -1,5 +1,5 @@ from sysdata.csv.csv_futures_contract_prices import ConfigCsvFuturesPrices -import os +from pathlib import Path from syscore.fileutils import ( get_resolved_pathname, files_with_extension_in_resolved_pathname, @@ -30,11 +30,11 @@ def strip_file_names(pathname): datecode = str(year) + "{0:02d}".format(month) new_file_name = "%s_%s00.csv" % (instrument, datecode) - new_full_name = os.path.join(resolved_pathname, new_file_name) - old_full_name = os.path.join(resolved_pathname, filename + ".csv") + new_full_name = Path(resolved_pathname, new_file_name) + old_full_name = Path(resolved_pathname, filename + ".csv") print("Rename %s to\n %s" % (old_full_name, new_full_name)) - os.rename(old_full_name, new_full_name) + old_full_name.rename(new_full_name) return None diff --git a/syslogging/logger.py b/syslogging/logger.py index a1ec0e550d5..f046513073f 100644 --- a/syslogging/logger.py +++ b/syslogging/logger.py @@ -1,3 +1,4 @@ +from pathlib import Path import os import sys import socket @@ -98,7 +99,7 @@ def _configure_sim(): def _configure_prod(logging_config_file): print(f"Attempting to configure prod logging from {logging_config_file}") config_path = resolve_path_and_filename_for_package(logging_config_file) - if os.path.exists(config_path): + if Path(config_path).exists(): try: config = parse_config(path=config_path) host, port = _get_log_server_config(config) diff --git a/sysproduction/backup_db_to_csv.py b/sysproduction/backup_db_to_csv.py index 0ff4e8e694f..7b2e233d1e8 100644 --- a/sysproduction/backup_db_to_csv.py +++ b/sysproduction/backup_db_to_csv.py @@ -1,3 +1,4 @@ +from pathlib import Path import os import pandas as pd @@ -98,10 +99,9 @@ def get_data_and_create_csv_directories(logname): ) for class_name, path in class_paths.items(): - dir_name = os.path.join(csv_dump_dir, path) - class_paths[class_name] = dir_name - if not os.path.exists(dir_name): - os.makedirs(dir_name) + dir_name = Path(csv_dump_dir, path) + class_paths[class_name] = str(dir_name) + Path(dir_name).mkdir(exist_ok=True) data = dataBlob(csv_data_paths=class_paths, log_name=logname) diff --git a/sysproduction/data/backtest.py b/sysproduction/data/backtest.py index 105d440d292..ab3361f5241 100644 --- a/sysproduction/data/backtest.py +++ b/sysproduction/data/backtest.py @@ -1,5 +1,5 @@ from copy import copy -import os +from pathlib import Path from shutil import copyfile from syscore.dateutils import create_datetime_marker_string @@ -218,10 +218,7 @@ def store_backtest_state(data, system, strategy_name="default_strategy"): def ensure_backtest_directory_exists(strategy_name): full_directory = get_backtest_directory_for_strategy(strategy_name) - try: - os.makedirs(full_directory) - except FileExistsError: - pass + Path(full_directory).mkdir(exist_ok=True) def rchop(s, suffix): @@ -262,9 +259,9 @@ def get_backtest_config_filename(strategy_name, datetime_marker): def get_backtest_filename_prefix(strategy_name, datetime_marker): # eg '/home/rob/data/backtests/medium_speed_TF_carry/20200622_102913' full_directory = get_backtest_directory_for_strategy(strategy_name) - full_filename_prefix = os.path.join(full_directory, datetime_marker) + full_filename_prefix = Path(full_directory, datetime_marker) - return full_filename_prefix + return str(full_filename_prefix) def get_backtest_directory_for_strategy(strategy_name): @@ -272,9 +269,9 @@ def get_backtest_directory_for_strategy(strategy_name): directory_store_backtests = get_directory_store_backtests() directory_store_backtests = get_resolved_pathname(directory_store_backtests) - full_directory = os.path.join(directory_store_backtests, strategy_name) + full_directory = Path(directory_store_backtests, strategy_name) - return full_directory + return str(full_directory) def get_directory_store_backtests(): diff --git a/sysproduction/reporting/reporting_functions.py b/sysproduction/reporting/reporting_functions.py index 82607834ff8..4410e78e915 100644 --- a/sysproduction/reporting/reporting_functions.py +++ b/sysproduction/reporting/reporting_functions.py @@ -3,6 +3,7 @@ from PyPDF2 import PdfMerger import datetime import pandas as pd +from pathlib import Path import os import shutil import matplotlib.pyplot as plt @@ -274,14 +275,14 @@ def output_file_report( data.log.debug("Written report to %s" % full_filename) -def resolve_report_filename(report_config, data: dataBlob): +def resolve_report_filename(report_config, data: dataBlob) -> str: filename_with_spaces = report_config.title filename = filename_with_spaces.replace(" ", "_") use_directory = get_directory_for_reporting(data) use_directory_resolved = get_resolved_pathname(use_directory) - full_filename = os.path.join(use_directory_resolved, filename) + full_filename = Path(use_directory_resolved, filename) - return full_filename + return str(full_filename) def get_directory_for_reporting(data): @@ -337,6 +338,6 @@ def _generate_temp_pdf_filename( TEMPFILE_PATTERN, str(datetime_to_long(datetime.datetime.now())), ) - full_filename = os.path.join(use_directory_resolved, filename) + full_filename = Path(use_directory_resolved, filename) - return full_filename + return str(full_filename) From 1df39086a240225f4beb37dec2207ac98300b612 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Tue, 23 Dec 2025 10:12:35 +0000 Subject: [PATCH 04/28] start refactor of fileutils --- syscore/fileutils.py | 239 ++++---------------------------- syscore/tests/test_fileutils.py | 1 + 2 files changed, 30 insertions(+), 210 deletions(-) diff --git a/syscore/fileutils.py b/syscore/fileutils.py index d5bf79c658b..ac28c9475ee 100755 --- a/syscore/fileutils.py +++ b/syscore/fileutils.py @@ -1,21 +1,18 @@ import glob import datetime import time -from importlib import import_module + +# from importlib import import_module +# from array import array +from importlib.util import find_spec + +# import inspect import os from pathlib import Path from typing import List, Tuple from syscore.dateutils import SECONDS_PER_DAY -# DO NOT DELETE: all these are unused: but are required to get the filename padding to work - - -""" - - FILES IN DIRECTORIES - -""" """ @@ -106,68 +103,22 @@ def resolve_path_and_filename_for_package( path_and_filename: str, separate_filename=None ) -> str: """ - A way of resolving relative and absolute filenames, and dealing with awkward OS specific things - - >>> resolve_path_and_filename_for_package("/home/rob/", "file.csv") - '/home/rob/file.csv' - - >>> resolve_path_and_filename_for_package(".home.rob", "file.csv") - '/home/rob/file.csv' - - >>> resolve_path_and_filename_for_package('C:\\home\\rob\\'', "file.csv") - 'C:\\home\\rob\\file.csv' - - >>> resolve_path_and_filename_for_package("syscore.tests", "file.csv") - '/home/rob/pysystemtrade/syscore/tests/file.csv' - - >>> resolve_path_and_filename_for_package("/home/rob/file.csv") - '/home/rob/file.csv' - - >>> resolve_path_and_filename_for_package(".home.rob.file.csv") - '/home/rob/file.csv' - - >>> resolve_path_and_filename_for_package("C:\\home\\rob\\file.csv") - 'C:\\home\\rob\\file.csv' - - >>> resolve_path_and_filename_for_package("syscore.tests.file.csv") - '/home/rob/pysystemtrade/syscore/tests/file.csv' - + A way of resolving relative and absolute filenames """ - path_and_filename_as_list = transform_path_into_list(path_and_filename) + resolved = get_resolved_pathname(path_and_filename) if separate_filename is None: - ( - path_as_list, - separate_filename, - ) = extract_filename_from_combined_path_and_filename_list( - path_and_filename_as_list - ) + path = Path(resolved) else: - path_as_list = path_and_filename_as_list - - resolved_pathname = get_pathname_from_list(path_as_list) + path = Path(resolved, separate_filename) - resolved_path_and_filename = os.path.join(resolved_pathname, separate_filename) - - return resolved_path_and_filename + return str(path) def get_resolved_pathname(pathname: str) -> str: """ - >>> get_resolved_pathname("/home/rob/") - '/home/rob' - - >>> get_resolved_pathname(".home.rob") - '/home/rob' - - >>> get_resolved_pathname('C:\\home\\rob\\'') - 'C:\\home\\rob' - - >>> get_resolved_pathname("syscore.tests") - '/home/rob/pysystemtrade/syscore/tests' - + TODO """ - if isinstance(pathname, Path): # special case when already a Path pathname = str(pathname.absolute()) @@ -176,158 +127,19 @@ def get_resolved_pathname(pathname: str) -> str: # This is an ssh address for rsync - don't change return pathname - # Turn /,\ into . so system independent - path_as_list = transform_path_into_list(pathname) - resolved_pathname = get_pathname_from_list(path_as_list) - - return resolved_pathname - - -## something unlikely to occur naturally in a pathname -RESERVED_CHARACTERS = "&!*" - - -def transform_path_into_list(pathname: str) -> List[str]: - """ - >>> path_as_list("/home/rob/test.csv") - ['', 'home', 'rob', 'test', 'csv'] - - >>> path_as_list("/home/rob/") - ['', 'home', 'rob'] - - >>> path_as_list(".home.rob") - ['', 'home', 'rob'] - - >>> path_as_list('C:\\home\\rob\\'') - ['C:', 'home', 'rob'] - - >>> path_as_list('C:\\home\\rob\\test.csv') - ['C:', 'home', 'rob', 'test', 'csv'] - - >>> path_as_list("syscore.tests.fileutils.csv") - ['syscore', 'tests', 'fileutils', 'csv'] - - >>> path_as_list("syscore.tests") - ['syscore', 'tests'] - - """ - - pathname_replace = add_reserved_characters_to_pathname(pathname) - path_as_list = pathname_replace.rsplit(RESERVED_CHARACTERS) - - if path_as_list[-1] == "": - path_as_list.pop() - - return path_as_list - - -def add_reserved_characters_to_pathname(pathname: str) -> str: - pathname_replace = pathname.replace(".", RESERVED_CHARACTERS) - pathname_replace = pathname_replace.replace("/", RESERVED_CHARACTERS) - pathname_replace = pathname_replace.replace("\\", RESERVED_CHARACTERS) - - return pathname_replace - - -def extract_filename_from_combined_path_and_filename_list( - path_and_filename_as_list: list, -) -> Tuple[list, str]: - """ - >>> extract_filename_from_combined_path_and_filename_list(['home', 'rob','file', 'csv']) - (['home', 'rob'], 'file.csv') - """ - ## need -2 because want extension - extension = path_and_filename_as_list.pop() - filename = path_and_filename_as_list.pop() - - separate_filename = ".".join([filename, extension]) - - return path_and_filename_as_list, separate_filename - - -def get_pathname_from_list(path_as_list: List[str]) -> str: - """ - >>> get_pathname_from_list(['C:', 'home', 'rob']) - 'C:\\home\\rob' - >>> get_pathname_from_list(['','home','rob']) - '/home/rob' - >>> get_pathname_from_list(['syscore','tests']) - '/home/rob/pysystemtrade/syscore/tests' - """ - if path_as_list[0] == "": - # path_type_absolute - resolved_pathname = get_absolute_linux_pathname_from_list(path_as_list[1:]) - elif is_windoze_path_list(path_as_list): - # windoze - resolved_pathname = get_absolute_windows_pathname_from_list(path_as_list) + path = Path(pathname) + if path.is_absolute(): + return str(path) else: - # relative - resolved_pathname = get_relative_pathname_from_list(path_as_list) - - return resolved_pathname - - -def is_windoze_path_list(path_as_list: List[str]) -> bool: - """ - >>> is_windoze_path_list(['C:']) - True - >>> is_windoze_path_list(['wibble']) - False - """ - return path_as_list[0].endswith(":") - - -def get_relative_pathname_from_list(path_as_list: List[str]) -> str: - """ - - >>> get_relative_pathname_from_list(['syscore','tests']) - '/home/rob/pysystemtrade/syscore/tests' - """ - package_name = path_as_list[0] - paths_or_files = path_as_list[1:] - - if len(paths_or_files) == 0: - directory_name_of_package = os.path.dirname( - import_module(package_name).__file__ - ) - return directory_name_of_package - - last_item_in_list = path_as_list.pop() - pathname = os.path.join( - get_relative_pathname_from_list(path_as_list), last_item_in_list - ) - - return pathname - - -def get_absolute_linux_pathname_from_list(path_as_list: List[str]) -> str: - """ - Returns the absolute pathname from a list - - >>> get_absolute_linux_pathname_from_list(['home', 'rob']) - '/home/rob' - """ - pathname = os.path.join(*path_as_list) - pathname = os.path.sep + pathname - - return pathname + return _get_path_for_module(pathname) -def get_absolute_windows_pathname_from_list(path_as_list: list) -> str: - """ - Test will fail on linux - >>> get_absolute_windows_pathname_from_list(['C:','home','rob']) - 'C:\\home\\rob' - """ - drive_part_of_path = path_as_list[0] - if drive_part_of_path.endswith(":"): - ## add back backslash - drive_part_of_path = drive_part_of_path.replace(":", ":\\") - path_as_list[0] = drive_part_of_path - - pathname = os.path.join(*path_as_list) - - return pathname +def _get_path_for_module(module_name: str) -> str: + module_spec = find_spec(module_name) + if module_spec is None: + raise ModuleNotFoundError(f"Module {module_name} not found") + path = Path(module_spec.origin) + return str(path.parent) """ @@ -347,6 +159,13 @@ def write_list_of_lists_as_html_table_in_file(file, list_of_lists: list): file.write("") +""" + + FILES IN DIRECTORIES + +""" + + def files_with_extension_in_pathname(pathname: str, extension=".csv") -> List[str]: """ Find all the files with a particular extension in a directory diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py index b86d5a2a406..5387aead387 100644 --- a/syscore/tests/test_fileutils.py +++ b/syscore/tests/test_fileutils.py @@ -42,6 +42,7 @@ def test_resolve_path_and_filename_for_package(self): actual = resolve_path_and_filename_for_package("/home/rob/file.csv") assert actual == "/home/rob/file.csv" + # old: works, new: should not work actual = resolve_path_and_filename_for_package(".home.rob.file.csv") assert actual == "/home/rob/file.csv" From 4731282ccfdd6d141ac326b58239bb6fda2d735c Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Mon, 29 Dec 2025 11:17:13 +0000 Subject: [PATCH 05/28] tests refactored --- syscore/tests/test_fileutils.py | 43 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py index 5387aead387..973c997975a 100644 --- a/syscore/tests/test_fileutils.py +++ b/syscore/tests/test_fileutils.py @@ -35,33 +35,45 @@ def test_resolve_path_non_existent(self, project_dir): actual = get_resolved_pathname("syscore.testz") assert actual == f"{project_dir}/syscore/testz" - def test_resolve_path_and_filename_for_package(self): + @pytest.mark.xfail(reason="Cannot work with current implementation") + def test_resolve_dotted_dir_name(self, tmp_path): + directory = tmp_path / "dir.name.with.dots" + directory.mkdir() + file = directory / "hello.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(str(file)) + assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" + + @pytest.mark.xfail(reason="Cannot work with current implementation") + def test_resolve_dotted_file_name(self, tmp_path): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(str(file)) + assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + + def test_resolve_package_separate(self): actual = resolve_path_and_filename_for_package("/home/rob/", "file.csv") assert actual == "/home/rob/file.csv" + def test_resolve_package_combined(self): actual = resolve_path_and_filename_for_package("/home/rob/file.csv") assert actual == "/home/rob/file.csv" + def test_resolve_package_combined_dotted(self): # old: works, new: should not work actual = resolve_path_and_filename_for_package(".home.rob.file.csv") assert actual == "/home/rob/file.csv" - def test_path_and_filename_for_package_modules(self, project_dir): + def test_resolve_package_module_separate(self, project_dir): actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv") assert actual == f"{project_dir}/syscore/tests/file.csv" + def test_resolve_package_module_combined(self, project_dir): actual = resolve_path_and_filename_for_package("syscore.tests.file.csv") assert actual == f"{project_dir}/syscore/tests/file.csv" - @pytest.mark.xfail(reason="Cannot work with current implementation") - def test_resolve_dotted_dir_name(self, tmp_path): - directory = tmp_path / "dir.name.with.dots" - directory.mkdir() - file = directory / "hello.txt" - file.write_text("content", encoding="utf-8") - resolved_path = get_resolved_pathname(str(file)) - assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" - @pytest.mark.xfail(reason="Cannot work with current implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( self, tmp_path @@ -75,15 +87,6 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( ) assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" - @pytest.mark.xfail(reason="Cannot work with current implementation") - def test_resolve_dotted_file_name(self, tmp_path): - directory = tmp_path / "dir_name" - directory.mkdir() - file = directory / "dotted.filename.txt" - file.write_text("content", encoding="utf-8") - resolved_path = get_resolved_pathname(str(file)) - assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" - @pytest.mark.xfail(reason="Cannot work with current implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( self, tmp_path From e319e6ffda6e4711ae7e0af4e8c41271a5c6b1ef Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Mon, 29 Dec 2025 16:42:15 +0000 Subject: [PATCH 06/28] handle mixed module and file names --- syscore/fileutils.py | 36 +- syscore/tests/price.test.data.csv | 529 ++++++++++++++++++++++++++++++ syscore/tests/test_fileutils.py | 91 ++++- 3 files changed, 636 insertions(+), 20 deletions(-) create mode 100644 syscore/tests/price.test.data.csv diff --git a/syscore/fileutils.py b/syscore/fileutils.py index ac28c9475ee..51090599642 100755 --- a/syscore/fileutils.py +++ b/syscore/fileutils.py @@ -131,15 +131,37 @@ def get_resolved_pathname(pathname: str) -> str: if path.is_absolute(): return str(path) else: - return _get_path_for_module(pathname) + return _resolve_path(pathname) -def _get_path_for_module(module_name: str) -> str: - module_spec = find_spec(module_name) - if module_spec is None: - raise ModuleNotFoundError(f"Module {module_name} not found") - path = Path(module_spec.origin) - return str(path.parent) +def _resolve_path(path: str) -> str: + """ + Resolve a dotted path by iteratively treating trailing parts as a filename. + Returns the full resolved path (module directory + filename if present). + """ + parts = path.split(".") + + for i in range(len(parts), 0, -1): + module_parts = parts[:i] + file_parts = parts[i:] + + candidate_module = ".".join(module_parts) + candidate_file = ".".join(file_parts) if file_parts else None + + try: + module_spec = find_spec(candidate_module) + if module_spec is not None and module_spec.origin: + module_path = Path(module_spec.origin).parent + + if candidate_file: + if "." in candidate_file: + return str(module_path / candidate_file) + else: + return str(module_path) + except (ModuleNotFoundError, ValueError, AttributeError): + continue + + raise ModuleNotFoundError(f"Could not resolve path: {path}") """ diff --git a/syscore/tests/price.test.data.csv b/syscore/tests/price.test.data.csv new file mode 100644 index 00000000000..b4c9c312a5b --- /dev/null +++ b/syscore/tests/price.test.data.csv @@ -0,0 +1,529 @@ +DATETIME,ADJ +2013-04-15,139.0825 +2013-04-16, +2013-04-17, +2013-04-18, +2013-04-19, +2013-04-22, +2013-04-23, +2013-04-24, +2013-04-25, +2013-04-26, +2013-04-29, +2013-04-30, +2013-05-01, +2013-05-02, +2013-05-03,138.7825 +2013-05-06,137.0525 +2013-05-07, +2013-05-08, +2013-05-09, +2013-05-10, +2013-05-13, +2013-05-14, +2013-05-15, +2013-05-16, +2013-05-17, +2013-05-20, +2013-05-21, +2013-05-22, +2013-05-23, +2013-05-24, +2013-05-27, +2013-05-28,134.4125 +2013-05-29, +2013-05-30, +2013-05-31, +2013-06-03,134.9825 +2013-06-04,134.8025 +2013-06-05,135.1225 +2013-06-06,135.2825 +2013-06-07,134.9225 +2013-06-10,134.1625 +2013-06-11,133.9425 +2013-06-12,134.1025 +2013-06-13,134.4325 +2013-06-14,135.1625 +2013-06-17,135.0225 +2013-06-18,134.7025 +2013-06-19,133.8925 +2013-06-20,133.5725 +2013-06-21,132.5025 +2013-06-24,132.1325 +2013-06-25,131.8025 +2013-06-26,132.4325 +2013-06-27,132.9925 +2013-06-28,132.9025 +2013-07-01,132.9625 +2013-07-02,133.5125 +2013-07-03,133.5825 +2013-07-04,133.7125 +2013-07-05,132.9225 +2013-07-08,133.3625 +2013-07-09,134.0725 +2013-07-10,134.0025 +2013-07-11,134.1825 +2013-07-12,135.1125 +2013-07-15,134.7825 +2013-07-16,135.1225 +2013-07-17,135.2825 +2013-07-18,135.4425 +2013-07-19,135.4825 +2013-07-22,135.5325 +2013-07-23,135.1025 +2013-07-24,133.6825 +2013-07-25,133.6925 +2013-07-26,133.8425 +2013-07-29,133.7025 +2013-07-30,133.7125 +2013-07-31,134.2825 +2013-08-01,133.6025 +2013-08-02,134.1125 +2013-08-05,133.5625 +2013-08-06,133.3825 +2013-08-07,133.5225 +2013-08-08,133.6625 +2013-08-09,133.7125 +2013-08-12,133.3725 +2013-08-13,132.1325 +2013-08-14,132.1725 +2013-08-15,131.5325 +2013-08-16,131.0825 +2013-08-19,131.2825 +2013-08-20,131.9625 +2013-08-21,131.3125 +2013-08-22,131.0525 +2013-08-23,130.8825 +2013-08-26,131.3525 +2013-08-27,132.2625 +2013-08-28,131.5825 +2013-08-29,132.0625 +2013-08-30,131.7525 +2013-09-02,131.3225 +2013-09-03,131.1525 +2013-09-04,130.8325 +2013-09-05,129.7325 +2013-09-06,130.8125 +2013-09-09,130.7525 +2013-09-10,129.9325 +2013-09-11,130.6125 +2013-09-12,130.8725 +2013-09-13,131.2225 +2013-09-16,131.4425 +2013-09-17,131.3425 +2013-09-18,132.3225 +2013-09-19,131.7725 +2013-09-20,131.7925 +2013-09-23,132.0725 +2013-09-24,133.0025 +2013-09-25,133.2425 +2013-09-26,133.1025 +2013-09-27,133.6425 +2013-09-30,133.7825 +2013-10-01,133.5425 +2013-10-02,133.6025 +2013-10-03,133.5125 +2013-10-04,133.2425 +2013-10-07,133.5925 +2013-10-08,133.5025 +2013-10-09,133.5225 +2013-10-10,132.9325 +2013-10-11,133.0725 +2013-10-14,133.0325 +2013-10-15,132.5225 +2013-10-16,132.1925 +2013-10-17,132.9625 +2013-10-18,133.3325 +2013-10-21,133.1925 +2013-10-22,133.8225 +2013-10-23,134.1425 +2013-10-24,134.1525 +2013-10-25,134.3425 +2013-10-28,134.4325 +2013-10-29,134.5525 +2013-10-30,134.7325 +2013-10-31,135.2825 +2013-11-01,135.1325 +2013-11-04,135.1725 +2013-11-05,134.4425 +2013-11-06,134.4325 +2013-11-07,135.1125 +2013-11-08,134.3025 +2013-11-11,134.2925 +2013-11-12,133.9825 +2013-11-13,134.6125 +2013-11-14,134.9825 +2013-11-15,134.9125 +2013-11-18,135.1325 +2013-11-19,134.6925 +2013-11-20,134.7225 +2013-11-21,134.2625 +2013-11-22,134.2325 +2013-11-25,134.6025 +2013-11-26,134.9825 +2013-11-27,134.9125 +2013-11-28,134.9325 +2013-11-29,134.9925 +2013-12-02,134.4025 +2013-12-03,134.5625 +2013-12-04,133.6625 +2013-12-05,133.2425 +2013-12-06,133.5575 +2013-12-09,133.5375 +2013-12-10,133.6175 +2013-12-11,133.7475 +2013-12-12,133.5475 +2013-12-13,133.6975 +2013-12-16,133.6975 +2013-12-17,133.7375 +2013-12-18,133.5875 +2013-12-19,133.2475 +2013-12-20,133.3675 +2013-12-23,133.1975 +2013-12-24, +2013-12-25, +2013-12-26, +2013-12-27,132.4075 +2013-12-30,132.6175 +2013-12-31, +2014-01-01, +2014-01-02,132.5675 +2014-01-03,132.5375 +2014-01-06,132.9875 +2014-01-07,133.2075 +2014-01-08,132.9875 +2014-01-09,133.0075 +2014-01-10,133.6975 +2014-01-13,134.0975 +2014-01-14,134.1275 +2014-01-15,133.9875 +2014-01-16,134.6075 +2014-01-17,134.8475 +2014-01-20,135.0175 +2014-01-21,135.0975 +2014-01-22,134.9575 +2014-01-23,135.4875 +2014-01-24,136.1075 +2014-01-27,135.9975 +2014-01-28,135.9075 +2014-01-29,136.3475 +2014-01-30,136.6775 +2014-01-31,137.3275 +2014-02-03,137.4675 +2014-02-04,137.4275 +2014-02-05,137.5175 +2014-02-06,136.7775 +2014-02-07,137.2775 +2014-02-10,137.0975 +2014-02-11,137.0875 +2014-02-12,136.6175 +2014-02-13,137.2275 +2014-02-14,137.0475 +2014-02-17,137.0475 +2014-02-18,137.1675 +2014-02-19,137.3575 +2014-02-20,136.8975 +2014-02-21,137.3175 +2014-02-24,137.1275 +2014-02-25,137.4175 +2014-02-26,137.9075 +2014-02-27,138.5175 +2014-02-28,137.8275 +2014-03-03,138.5875 +2014-03-04,138.2 +2014-03-05,138.14 +2014-03-06,137.55 +2014-03-07,137.615 +2014-03-10,137.78 +2014-03-11,137.77 +2014-03-12,138.33 +2014-03-13,139.09 +2014-03-14,138.8 +2014-03-17,138.55 +2014-03-18,138.6275 +2014-03-19,137.86 +2014-03-20,137.76 +2014-03-21,137.93 +2014-03-24, +2014-03-25,138.57 +2014-03-26,138.9 +2014-03-27,139.09 +2014-03-28,139.01 +2014-03-31,138.84 +2014-04-01,138.65 +2014-04-02,138.18 +2014-04-03,138.3 +2014-04-04, +2014-04-07,139.01 +2014-04-08,138.8525 +2014-04-09, +2014-04-10,139.31 +2014-04-11,139.63 +2014-04-14,139.27 +2014-04-15,139.88 +2014-04-16,139.77 +2014-04-17,139.08 +2014-04-18, +2014-04-21, +2014-04-22,139.23 +2014-04-23,139.24 +2014-04-24,139.31 +2014-04-25,139.74 +2014-04-28,139.64 +2014-04-29,139.63 +2014-04-30,140.03 +2014-05-01, +2014-05-02,140.18 +2014-05-05,140.02 +2014-05-06,140.03 +2014-05-07,139.9 +2014-05-08,140.25 +2014-05-09,140.16 +2014-05-12,140.07 +2014-05-13,140.64 +2014-05-14,141.24 +2014-05-15,141.99 +2014-05-16,141.65 +2014-05-19,141.49 +2014-05-20,141.59 +2014-05-21,141.28 +2014-05-22,141.4 +2014-05-23,141.39 +2014-05-26,141.38 +2014-05-27,141.755 +2014-05-28,142.39 +2014-05-29,142.06 +2014-05-30,142.17 +2014-06-02,142.02 +2014-06-03,141.32 +2014-06-04,141.215 +2014-06-05,141.66 +2014-06-06,142.18 +2014-06-09,141.95 +2014-06-10,141.5 +2014-06-11,141.52 +2014-06-12,141.795 +2014-06-13,141.97 +2014-06-16,142.22 +2014-06-17,142.19 +2014-06-18,142.18 +2014-06-19,142.48 +2014-06-20,142.34 +2014-06-23,142.54 +2014-06-24,142.78 +2014-06-25,143.3 +2014-06-26,143.59 +2014-06-27,143.38 +2014-06-30,143.6 +2014-07-01,143.51 +2014-07-02,143.07 +2014-07-03,142.895 +2014-07-04,143.4 +2014-07-07,143.45 +2014-07-08,143.96 +2014-07-09,144.05 +2014-07-10,144.11 +2014-07-11,144.04 +2014-07-14,144.11 +2014-07-15,144.11 +2014-07-16,144.35 +2014-07-17,144.96 +2014-07-18,144.73 +2014-07-21,144.79 +2014-07-22,144.68 +2014-07-23,144.81 +2014-07-24,144.44 +2014-07-25,144.91 +2014-07-28,144.79 +2014-07-29,145.22 +2014-07-30,144.46 +2014-07-31,144.675 +2014-08-01, +2014-08-04, +2014-08-05, +2014-08-06,145.43 +2014-08-07,145.555 +2014-08-08,146.015 +2014-08-11,145.88 +2014-08-12,145.81 +2014-08-13,146.34 +2014-08-14,146.41 +2014-08-15,146.87 +2014-08-18,146.38 +2014-08-19,146.62 +2014-08-20,146.75 +2014-08-21,146.76 +2014-08-22,146.96 +2014-08-25,147.39 +2014-08-26,147.45 +2014-08-27,147.86 +2014-08-28,148.2 +2014-08-29,148.02 +2014-09-01,148.035 +2014-09-02,147.51 +2014-09-03,147.42 +2014-09-04,147.26 +2014-09-05,147.54 +2014-09-08,147.22 +2014-09-09,146.87 +2014-09-10,146.82 +2014-09-11,146.85 +2014-09-12,146.31 +2014-09-15,146.71 +2014-09-16,146.64 +2014-09-17,146.74 +2014-09-18,146.49 +2014-09-19,147.14 +2014-09-22,147.42 +2014-09-23,147.5 +2014-09-24,147.38 +2014-09-25,147.9 +2014-09-26,147.9 +2014-09-29,148.02 +2014-09-30,148.2 +2014-10-01,148.87 +2014-10-02,148.51 +2014-10-03,148.44 +2014-10-06,148.61 +2014-10-07,148.81 +2014-10-08,148.72 +2014-10-09,148.69 +2014-10-10,149.01 +2014-10-13,148.94 +2014-10-14,149.59 +2014-10-15,150.06 +2014-10-16,150.21 +2014-10-17,149.17 +2014-10-20,149.31 +2014-10-21,148.95 +2014-10-22,149.19 +2014-10-23,148.67 +2014-10-24,148.83 +2014-10-27,149.03 +2014-10-28,148.98 +2014-10-29,148.75 +2014-10-30,149.37 +2014-10-31,149.55 +2014-11-03,149.41 +2014-11-04,149.75 +2014-11-05,149.69 +2014-11-06,149.61 +2014-11-07,149.87 +2014-11-10,149.59 +2014-11-11,149.78 +2014-11-12,149.82 +2014-11-13,150.2 +2014-11-14,150.4 +2014-11-17,150.11 +2014-11-18,150.08 +2014-11-19,149.56 +2014-11-20,150.12 +2014-11-21,150.51 +2014-11-24,150.52 +2014-11-25,150.86 +2014-11-26,151.03 +2014-11-27,151.42 +2014-11-28,151.43 +2014-12-01,150.68 +2014-12-02,150.76 +2014-12-03,150.84 +2014-12-04,150.77 +2014-12-05,150.61 +2014-12-08,151.45 +2014-12-09,151.66 +2014-12-10,151.95 +2014-12-11,151.92 +2014-12-12,152.47 +2014-12-15,152.35 +2014-12-16,152.9 +2014-12-17,152.8 +2014-12-18,152.64 +2014-12-19,152.96 +2014-12-22,152.93 +2014-12-23,152.78 +2014-12-24, +2014-12-25, +2014-12-26, +2014-12-29,153.54 +2014-12-30,153.62 +2014-12-31, +2015-01-01, +2015-01-02,154.29 +2015-01-05,154.09 +2015-01-06,154.72 +2015-01-07,154.48 +2015-01-08,154.12 +2015-01-09,154.22 +2015-01-12,154.58 +2015-01-13,154.61 +2015-01-14,155.01 +2015-01-15,155.43 +2015-01-16,155.28 +2015-01-19,155.54 +2015-01-20,155.4 +2015-01-21,154.48 +2015-01-22,155.48 +2015-01-23,156.62 +2015-01-26,156.04 +2015-01-27,156 +2015-01-28,156.98 +2015-01-29,156.33 +2015-01-30,157.26 +2015-02-02,157.16 +2015-02-03,156.49 +2015-02-04,156.53 +2015-02-05,156.39 +2015-02-06,156.22 +2015-02-09,156.49 +2015-02-10,156.29 +2015-02-11,156.54 +2015-02-12,156.94 +2015-02-13,156.62 +2015-02-16,157.09 +2015-02-17,156.24 +2015-02-18,156.48 +2015-02-19,156.29 +2015-02-20,156.13 +2015-02-23,156.72 +2015-02-24,156.72 +2015-02-25,157.35 +2015-02-26,157.49 +2015-02-27,157.355 +2015-03-02,156.82 +2015-03-03,156.54 +2015-03-04,156.51 +2015-03-05,157.01 +2015-03-06,156.25 +2015-03-09,157.34 +2015-03-10,158.47 +2015-03-11,158.56 +2015-03-12,158 +2015-03-13,157.89 +2015-03-16,157.71 +2015-03-17,157.59 +2015-03-18,159.07 +2015-03-19,158.77 +2015-03-20,158.87 +2015-03-23,158.39 +2015-03-24,158.34 +2015-03-25,158.16 +2015-03-26,158.37 +2015-03-27,158.52 +2015-03-30,158.47 +2015-03-31,158.79 +2015-04-01,158.9 +2015-04-02,158.71 +2015-04-03, +2015-04-06, +2015-04-07,158.89 +2015-04-08,159.12 +2015-04-09,159.01 +2015-04-10,159.16 +2015-04-13,159.31 +2015-04-14,159.43 +2015-04-15,159.98 +2015-04-16,160.31 +2015-04-17,160.45 +2015-04-20,160.31 +2015-04-21,159.9 +2015-04-22,159.225 diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py index 973c997975a..95d965e803e 100644 --- a/syscore/tests/test_fileutils.py +++ b/syscore/tests/test_fileutils.py @@ -23,6 +23,7 @@ def test_resolve_path_absolute_trailing(self): actual = get_resolved_pathname("/home/rob/") assert actual == "/home/rob" + @pytest.mark.xfail(reason="Cannot work with new implementation") def test_resolve_path_absolute_dotted(self): actual = get_resolved_pathname(".home.rob") assert actual == "/home/rob" @@ -32,10 +33,9 @@ def test_resolve_path_relative(self, project_dir): assert actual == f"{project_dir}/syscore/tests" def test_resolve_path_non_existent(self, project_dir): - actual = get_resolved_pathname("syscore.testz") - assert actual == f"{project_dir}/syscore/testz" + with pytest.raises(ModuleNotFoundError): + get_resolved_pathname("syscore.testz") - @pytest.mark.xfail(reason="Cannot work with current implementation") def test_resolve_dotted_dir_name(self, tmp_path): directory = tmp_path / "dir.name.with.dots" directory.mkdir() @@ -44,7 +44,6 @@ def test_resolve_dotted_dir_name(self, tmp_path): resolved_path = get_resolved_pathname(str(file)) assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" - @pytest.mark.xfail(reason="Cannot work with current implementation") def test_resolve_dotted_file_name(self, tmp_path): directory = tmp_path / "dir_name" directory.mkdir() @@ -61,8 +60,8 @@ def test_resolve_package_combined(self): actual = resolve_path_and_filename_for_package("/home/rob/file.csv") assert actual == "/home/rob/file.csv" + @pytest.mark.xfail(reason="Cannot work with new implementation") def test_resolve_package_combined_dotted(self): - # old: works, new: should not work actual = resolve_path_and_filename_for_package(".home.rob.file.csv") assert actual == "/home/rob/file.csv" @@ -71,10 +70,17 @@ def test_resolve_package_module_separate(self, project_dir): assert actual == f"{project_dir}/syscore/tests/file.csv" def test_resolve_package_module_combined(self, project_dir): - actual = resolve_path_and_filename_for_package("syscore.tests.file.csv") - assert actual == f"{project_dir}/syscore/tests/file.csv" + actual = resolve_path_and_filename_for_package( + "syscore.tests.pricetestdata.csv" + ) + assert actual == f"{project_dir}/syscore/tests/pricetestdata.csv" + + def test_resolve_package_module_combined_dotted_filename(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests.price.test.data.csv" + ) + assert actual == f"{project_dir}/syscore/tests/price.test.data.csv" - @pytest.mark.xfail(reason="Cannot work with current implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( self, tmp_path ): @@ -87,7 +93,6 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( ) assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" - @pytest.mark.xfail(reason="Cannot work with current implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( self, tmp_path ): @@ -111,6 +116,7 @@ def test_resolve_path_absolute_trailing(self): actual = get_resolved_pathname("C:\\home\\rob\\") assert actual == "C:\\home\\rob" + @pytest.mark.xfail(reason="Cannot work with new implementation") def test_resolve_path_absolute_dotted(self): actual = get_resolved_pathname(".home.rob") assert actual == "\\home\\rob" @@ -119,16 +125,75 @@ def test_resolve_path_relative(self, project_dir): actual = get_resolved_pathname("syscore.tests") assert actual == f"{project_dir}\\syscore\\tests" - def test_resolve_path_and_filename_for_package(self): + def test_resolve_path_non_existent(self, project_dir): + with pytest.raises(ModuleNotFoundError): + get_resolved_pathname("syscore.testz") + + def test_resolve_dotted_dir_name(self, tmp_path): + directory = tmp_path / "dir.name.with.dots" + directory.mkdir() + file = directory / "hello.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(str(file)) + assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" + + def test_resolve_dotted_file_name(self, tmp_path): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(str(file)) + assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + + def test_resolve_package_separate(self): actual = resolve_path_and_filename_for_package("C:\\home\\rob\\", "file.csv") assert actual == "C:\\home\\rob\\file.csv" + def test_resolve_package_combined(self): actual = resolve_path_and_filename_for_package("C:\\home\\rob\\file.csv") assert actual == "C:\\home\\rob\\file.csv" - def test_path_and_filename_for_package_modules(self, project_dir): + @pytest.mark.xfail(reason="Cannot work with new implementation") + def test_resolve_package_combined_dotted(self): + actual = resolve_path_and_filename_for_package(".home.rob.file.csv") + assert actual == "/home/rob/file.csv" + + def test_resolve_package_module_separate(self, project_dir): actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv") assert actual == f"{project_dir}\\syscore\\tests\\file.csv" - actual = resolve_path_and_filename_for_package("syscore.tests.file.csv") - assert actual == f"{project_dir}\\syscore\\tests\\file.csv" + def test_resolve_package_module_combined(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests.pricetestdata.csv" + ) + assert actual == f"{project_dir}\\syscore\\tests\\pricetestdata.csv" + + def test_resolve_package_module_combined_dotted_filename(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests.price.test.data.csv" + ) + assert actual == f"{project_dir}\\syscore\\tests\\price.test.data.csv" + + def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( + self, tmp_path + ): + directory = tmp_path / "dir.name.with.dots" + directory.mkdir() + file = directory / "hello.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}\\dir.name.with.dots", "hello.txt" + ) + assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt" + + def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( + self, tmp_path + ): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}\\dir_name\\dotted.filename.txt" + ) + assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt" From ad54e9d3c90c6be1d5ed93c2631aa4da17a795c6 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Mon, 29 Dec 2025 17:29:44 +0000 Subject: [PATCH 07/28] run tests --- .github/workflows/os-test.yml | 6 +++++- syscore/tests/test_fileutils.py | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/os-test.yml b/.github/workflows/os-test.yml index c3253a50285..761c0dd1e58 100644 --- a/.github/workflows/os-test.yml +++ b/.github/workflows/os-test.yml @@ -25,4 +25,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - python -m pip install . + python -m pip install '.[dev]' + + - name: Test with pytest + run: | + pytest syscore/tests/test_fileutils.py diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py index 95d965e803e..47039a7a132 100644 --- a/syscore/tests/test_fileutils.py +++ b/syscore/tests/test_fileutils.py @@ -135,7 +135,7 @@ def test_resolve_dotted_dir_name(self, tmp_path): file = directory / "hello.txt" file.write_text("content", encoding="utf-8") resolved_path = get_resolved_pathname(str(file)) - assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" + assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt" def test_resolve_dotted_file_name(self, tmp_path): directory = tmp_path / "dir_name" @@ -143,7 +143,7 @@ def test_resolve_dotted_file_name(self, tmp_path): file = directory / "dotted.filename.txt" file.write_text("content", encoding="utf-8") resolved_path = get_resolved_pathname(str(file)) - assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt" def test_resolve_package_separate(self): actual = resolve_path_and_filename_for_package("C:\\home\\rob\\", "file.csv") @@ -156,7 +156,7 @@ def test_resolve_package_combined(self): @pytest.mark.xfail(reason="Cannot work with new implementation") def test_resolve_package_combined_dotted(self): actual = resolve_path_and_filename_for_package(".home.rob.file.csv") - assert actual == "/home/rob/file.csv" + assert actual == "\\home\\rob\\file.csv" def test_resolve_package_module_separate(self, project_dir): actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv") From aa7c5470b796d0e1b066599fec9c8e1042aaea86 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Tue, 30 Dec 2025 17:38:18 +0000 Subject: [PATCH 08/28] fix parquet path --- sysdata/parquet/parquet_access.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sysdata/parquet/parquet_access.py b/sysdata/parquet/parquet_access.py index c16e3cc8f8b..48b7123061e 100644 --- a/sysdata/parquet/parquet_access.py +++ b/sysdata/parquet/parquet_access.py @@ -66,5 +66,6 @@ def _get_filename_given_data_type_and_identifier( def _get_pathname_given_data_type(self, data_type: str): root = self.parquet_store - path = Path(root, data_type).mkdir(parents=True, exist_ok=True) + path = Path(root, data_type) + path.mkdir(parents=True, exist_ok=True) return str(path) From 7f7e1715e4354e68a3b532d41a57be8be9360a06 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Tue, 30 Dec 2025 17:41:36 +0000 Subject: [PATCH 09/28] making data.futures.csvconfig a module --- data/futures/__init__.py | 0 data/futures/csvconfig/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 data/futures/__init__.py create mode 100644 data/futures/csvconfig/__init__.py diff --git a/data/futures/__init__.py b/data/futures/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/data/futures/csvconfig/__init__.py b/data/futures/csvconfig/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From ee90c5a40908584bd94bf6a281b4ba821fcaddbe Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Tue, 30 Dec 2025 17:42:22 +0000 Subject: [PATCH 10/28] adding a specific test for instrument config --- syscore/tests/test_fileutils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py index 47039a7a132..bc6e7055183 100644 --- a/syscore/tests/test_fileutils.py +++ b/syscore/tests/test_fileutils.py @@ -69,6 +69,12 @@ def test_resolve_package_module_separate(self, project_dir): actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv") assert actual == f"{project_dir}/syscore/tests/file.csv" + def test_resolve_package_module_instr_data_module(self, project_dir): + actual = resolve_path_and_filename_for_package( + "data.futures.csvconfig", "instrumentconfig.csv" + ) + assert actual == f"{project_dir}/data/futures/csvconfig/instrumentconfig.csv" + def test_resolve_package_module_combined(self, project_dir): actual = resolve_path_and_filename_for_package( "syscore.tests.pricetestdata.csv" From 7dc78770525a53adc61c18b67417d5a6f1fd05ec Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Tue, 30 Dec 2025 18:37:50 +0000 Subject: [PATCH 11/28] deal with private config --- sysdata/config/private_config.py | 25 +++++++++++-------------- sysdata/config/private_directory.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/sysdata/config/private_config.py b/sysdata/config/private_config.py index 791e592f128..51357bd5ced 100644 --- a/sysdata/config/private_config.py +++ b/sysdata/config/private_config.py @@ -1,25 +1,22 @@ from syscore.fileutils import resolve_path_and_filename_for_package, does_filename_exist from syscore.constants import arg_not_supplied -from sysdata.config.private_directory import get_full_path_for_private_config +from sysdata.config.private_directory import get_private_config_dir import yaml PRIVATE_CONFIG_FILE = "private_config.yaml" -def get_private_config_as_dict(filename: str = arg_not_supplied) -> dict: - if filename is arg_not_supplied: - filename = get_full_path_for_private_config(PRIVATE_CONFIG_FILE) - if not does_filename_exist(filename): +def get_private_config_as_dict() -> dict: + dir = get_private_config_dir() + try: + private_file = resolve_path_and_filename_for_package(dir, PRIVATE_CONFIG_FILE) + with open(private_file) as file_to_parse: + private_dict = yaml.load(file_to_parse, Loader=yaml.FullLoader) + return private_dict + + except (FileNotFoundError, ModuleNotFoundError): print( - "Private configuration %s does not exist; no problem if running in sim mode" - % filename + f"Private configuration '{dir}.{PRIVATE_CONFIG_FILE}' does not exist; no problem if running in sim mode" ) - return {} - - private_file = resolve_path_and_filename_for_package(filename) - with open(private_file) as file_to_parse: - private_dict = yaml.load(file_to_parse, Loader=yaml.FullLoader) - - return private_dict diff --git a/sysdata/config/private_directory.py b/sysdata/config/private_directory.py index a844f341d45..0b7b6ec400b 100644 --- a/sysdata/config/private_directory.py +++ b/sysdata/config/private_directory.py @@ -1,4 +1,5 @@ import os +from pathlib import Path DEFAULT_PRIVATE_DIR = "private" PRIVATE_CONFIG_DIR_ENV_VAR = "PYSYS_PRIVATE_CONFIG_DIR" @@ -13,3 +14,12 @@ def get_full_path_for_private_config(filename: str): private_config_path = f"{DEFAULT_PRIVATE_DIR}/{filename}" return private_config_path + + +def get_private_config_dir(): + if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR): + private_config_dir = Path(os.environ[PRIVATE_CONFIG_DIR_ENV_VAR]) + else: + private_config_dir = Path(DEFAULT_PRIVATE_DIR) + + return str(private_config_dir) From d83e296d6f3074ae9ecbae3f6237983f82f6a854 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Fri, 2 Jan 2026 13:05:49 +0000 Subject: [PATCH 12/28] revert --- data/futures/__init__.py | 0 data/futures/csvconfig/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 data/futures/__init__.py delete mode 100644 data/futures/csvconfig/__init__.py diff --git a/data/futures/__init__.py b/data/futures/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/data/futures/csvconfig/__init__.py b/data/futures/csvconfig/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 From 77cf1da7e47307561b305bca110657b9e9819aed Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Fri, 2 Jan 2026 13:06:05 +0000 Subject: [PATCH 13/28] revert --- syscore/fileutils.py | 247 ++++++++++++++++++++++++++------ syscore/tests/test_fileutils.py | 184 ++++++++++++++++++++++-- 2 files changed, 379 insertions(+), 52 deletions(-) diff --git a/syscore/fileutils.py b/syscore/fileutils.py index 51090599642..d5bf79c658b 100755 --- a/syscore/fileutils.py +++ b/syscore/fileutils.py @@ -1,18 +1,21 @@ import glob import datetime import time - -# from importlib import import_module -# from array import array -from importlib.util import find_spec - -# import inspect +from importlib import import_module import os from pathlib import Path from typing import List, Tuple from syscore.dateutils import SECONDS_PER_DAY +# DO NOT DELETE: all these are unused: but are required to get the filename padding to work + + +""" + + FILES IN DIRECTORIES + +""" """ @@ -103,22 +106,68 @@ def resolve_path_and_filename_for_package( path_and_filename: str, separate_filename=None ) -> str: """ - A way of resolving relative and absolute filenames + A way of resolving relative and absolute filenames, and dealing with awkward OS specific things + + >>> resolve_path_and_filename_for_package("/home/rob/", "file.csv") + '/home/rob/file.csv' + + >>> resolve_path_and_filename_for_package(".home.rob", "file.csv") + '/home/rob/file.csv' + + >>> resolve_path_and_filename_for_package('C:\\home\\rob\\'', "file.csv") + 'C:\\home\\rob\\file.csv' + + >>> resolve_path_and_filename_for_package("syscore.tests", "file.csv") + '/home/rob/pysystemtrade/syscore/tests/file.csv' + + >>> resolve_path_and_filename_for_package("/home/rob/file.csv") + '/home/rob/file.csv' + + >>> resolve_path_and_filename_for_package(".home.rob.file.csv") + '/home/rob/file.csv' + + >>> resolve_path_and_filename_for_package("C:\\home\\rob\\file.csv") + 'C:\\home\\rob\\file.csv' + + >>> resolve_path_and_filename_for_package("syscore.tests.file.csv") + '/home/rob/pysystemtrade/syscore/tests/file.csv' + """ - resolved = get_resolved_pathname(path_and_filename) + path_and_filename_as_list = transform_path_into_list(path_and_filename) if separate_filename is None: - path = Path(resolved) + ( + path_as_list, + separate_filename, + ) = extract_filename_from_combined_path_and_filename_list( + path_and_filename_as_list + ) else: - path = Path(resolved, separate_filename) + path_as_list = path_and_filename_as_list + + resolved_pathname = get_pathname_from_list(path_as_list) + + resolved_path_and_filename = os.path.join(resolved_pathname, separate_filename) - return str(path) + return resolved_path_and_filename def get_resolved_pathname(pathname: str) -> str: """ - TODO + >>> get_resolved_pathname("/home/rob/") + '/home/rob' + + >>> get_resolved_pathname(".home.rob") + '/home/rob' + + >>> get_resolved_pathname('C:\\home\\rob\\'') + 'C:\\home\\rob' + + >>> get_resolved_pathname("syscore.tests") + '/home/rob/pysystemtrade/syscore/tests' + """ + if isinstance(pathname, Path): # special case when already a Path pathname = str(pathname.absolute()) @@ -127,41 +176,158 @@ def get_resolved_pathname(pathname: str) -> str: # This is an ssh address for rsync - don't change return pathname - path = Path(pathname) - if path.is_absolute(): - return str(path) + # Turn /,\ into . so system independent + path_as_list = transform_path_into_list(pathname) + resolved_pathname = get_pathname_from_list(path_as_list) + + return resolved_pathname + + +## something unlikely to occur naturally in a pathname +RESERVED_CHARACTERS = "&!*" + + +def transform_path_into_list(pathname: str) -> List[str]: + """ + >>> path_as_list("/home/rob/test.csv") + ['', 'home', 'rob', 'test', 'csv'] + + >>> path_as_list("/home/rob/") + ['', 'home', 'rob'] + + >>> path_as_list(".home.rob") + ['', 'home', 'rob'] + + >>> path_as_list('C:\\home\\rob\\'') + ['C:', 'home', 'rob'] + + >>> path_as_list('C:\\home\\rob\\test.csv') + ['C:', 'home', 'rob', 'test', 'csv'] + + >>> path_as_list("syscore.tests.fileutils.csv") + ['syscore', 'tests', 'fileutils', 'csv'] + + >>> path_as_list("syscore.tests") + ['syscore', 'tests'] + + """ + + pathname_replace = add_reserved_characters_to_pathname(pathname) + path_as_list = pathname_replace.rsplit(RESERVED_CHARACTERS) + + if path_as_list[-1] == "": + path_as_list.pop() + + return path_as_list + + +def add_reserved_characters_to_pathname(pathname: str) -> str: + pathname_replace = pathname.replace(".", RESERVED_CHARACTERS) + pathname_replace = pathname_replace.replace("/", RESERVED_CHARACTERS) + pathname_replace = pathname_replace.replace("\\", RESERVED_CHARACTERS) + + return pathname_replace + + +def extract_filename_from_combined_path_and_filename_list( + path_and_filename_as_list: list, +) -> Tuple[list, str]: + """ + >>> extract_filename_from_combined_path_and_filename_list(['home', 'rob','file', 'csv']) + (['home', 'rob'], 'file.csv') + """ + ## need -2 because want extension + extension = path_and_filename_as_list.pop() + filename = path_and_filename_as_list.pop() + + separate_filename = ".".join([filename, extension]) + + return path_and_filename_as_list, separate_filename + + +def get_pathname_from_list(path_as_list: List[str]) -> str: + """ + >>> get_pathname_from_list(['C:', 'home', 'rob']) + 'C:\\home\\rob' + >>> get_pathname_from_list(['','home','rob']) + '/home/rob' + >>> get_pathname_from_list(['syscore','tests']) + '/home/rob/pysystemtrade/syscore/tests' + """ + if path_as_list[0] == "": + # path_type_absolute + resolved_pathname = get_absolute_linux_pathname_from_list(path_as_list[1:]) + elif is_windoze_path_list(path_as_list): + # windoze + resolved_pathname = get_absolute_windows_pathname_from_list(path_as_list) else: - return _resolve_path(pathname) + # relative + resolved_pathname = get_relative_pathname_from_list(path_as_list) + + return resolved_pathname + + +def is_windoze_path_list(path_as_list: List[str]) -> bool: + """ + >>> is_windoze_path_list(['C:']) + True + >>> is_windoze_path_list(['wibble']) + False + """ + return path_as_list[0].endswith(":") -def _resolve_path(path: str) -> str: +def get_relative_pathname_from_list(path_as_list: List[str]) -> str: """ - Resolve a dotted path by iteratively treating trailing parts as a filename. - Returns the full resolved path (module directory + filename if present). + + >>> get_relative_pathname_from_list(['syscore','tests']) + '/home/rob/pysystemtrade/syscore/tests' + """ + package_name = path_as_list[0] + paths_or_files = path_as_list[1:] + + if len(paths_or_files) == 0: + directory_name_of_package = os.path.dirname( + import_module(package_name).__file__ + ) + return directory_name_of_package + + last_item_in_list = path_as_list.pop() + pathname = os.path.join( + get_relative_pathname_from_list(path_as_list), last_item_in_list + ) + + return pathname + + +def get_absolute_linux_pathname_from_list(path_as_list: List[str]) -> str: + """ + Returns the absolute pathname from a list + + >>> get_absolute_linux_pathname_from_list(['home', 'rob']) + '/home/rob' """ - parts = path.split(".") + pathname = os.path.join(*path_as_list) + pathname = os.path.sep + pathname - for i in range(len(parts), 0, -1): - module_parts = parts[:i] - file_parts = parts[i:] + return pathname - candidate_module = ".".join(module_parts) - candidate_file = ".".join(file_parts) if file_parts else None - try: - module_spec = find_spec(candidate_module) - if module_spec is not None and module_spec.origin: - module_path = Path(module_spec.origin).parent +def get_absolute_windows_pathname_from_list(path_as_list: list) -> str: + """ + Test will fail on linux + >>> get_absolute_windows_pathname_from_list(['C:','home','rob']) + 'C:\\home\\rob' + """ + drive_part_of_path = path_as_list[0] + if drive_part_of_path.endswith(":"): + ## add back backslash + drive_part_of_path = drive_part_of_path.replace(":", ":\\") + path_as_list[0] = drive_part_of_path - if candidate_file: - if "." in candidate_file: - return str(module_path / candidate_file) - else: - return str(module_path) - except (ModuleNotFoundError, ValueError, AttributeError): - continue + pathname = os.path.join(*path_as_list) - raise ModuleNotFoundError(f"Could not resolve path: {path}") + return pathname """ @@ -181,13 +347,6 @@ def write_list_of_lists_as_html_table_in_file(file, list_of_lists: list): file.write("") -""" - - FILES IN DIRECTORIES - -""" - - def files_with_extension_in_pathname(pathname: str, extension=".csv") -> List[str]: """ Find all the files with a particular extension in a directory diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py index bc6e7055183..8cd250e2b37 100644 --- a/syscore/tests/test_fileutils.py +++ b/syscore/tests/test_fileutils.py @@ -23,7 +23,6 @@ def test_resolve_path_absolute_trailing(self): actual = get_resolved_pathname("/home/rob/") assert actual == "/home/rob" - @pytest.mark.xfail(reason="Cannot work with new implementation") def test_resolve_path_absolute_dotted(self): actual = get_resolved_pathname(".home.rob") assert actual == "/home/rob" @@ -33,9 +32,10 @@ def test_resolve_path_relative(self, project_dir): assert actual == f"{project_dir}/syscore/tests" def test_resolve_path_non_existent(self, project_dir): - with pytest.raises(ModuleNotFoundError): - get_resolved_pathname("syscore.testz") + actual = get_resolved_pathname("syscore.testz") + assert actual == f"{project_dir}/syscore/testz" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_dotted_dir_name(self, tmp_path): directory = tmp_path / "dir.name.with.dots" directory.mkdir() @@ -44,6 +44,7 @@ def test_resolve_dotted_dir_name(self, tmp_path): resolved_path = get_resolved_pathname(str(file)) assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_dotted_file_name(self, tmp_path): directory = tmp_path / "dir_name" directory.mkdir() @@ -60,7 +61,6 @@ def test_resolve_package_combined(self): actual = resolve_path_and_filename_for_package("/home/rob/file.csv") assert actual == "/home/rob/file.csv" - @pytest.mark.xfail(reason="Cannot work with new implementation") def test_resolve_package_combined_dotted(self): actual = resolve_path_and_filename_for_package(".home.rob.file.csv") assert actual == "/home/rob/file.csv" @@ -81,12 +81,14 @@ def test_resolve_package_module_combined(self, project_dir): ) assert actual == f"{project_dir}/syscore/tests/pricetestdata.csv" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_package_module_combined_dotted_filename(self, project_dir): actual = resolve_path_and_filename_for_package( "syscore.tests.price.test.data.csv" ) assert actual == f"{project_dir}/syscore/tests/price.test.data.csv" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( self, tmp_path ): @@ -99,6 +101,7 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( ) assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( self, tmp_path ): @@ -111,6 +114,83 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( ) assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + # No Separate filename + + def test_csv_data(self, project_dir): + actual = resolve_path_and_filename_for_package( + "data.futures.csvconfig.instrumentconfig.csv" + ) + assert actual == f"{project_dir}/data/futures/csvconfig/instrumentconfig.csv" + + def test_logging(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syslogging.logging_prod.yaml", + ) + assert actual == f"{project_dir}/syslogging/logging_prod.yaml" + + def test_config_defaults(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysdata.config.defaults.yaml", + ) + assert actual == f"{project_dir}/sysdata/config/defaults.yaml" + + def test_control_config_defaults(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscontrol.control_config.yaml", + ) + assert actual == f"{project_dir}/syscontrol/control_config.yaml" + + def test_private_config_dots(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private.private_config.yaml", + ) + assert actual == f"{project_dir}/private/private_config.yaml" + + def test_private_config_slash(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private/private_config.yaml", + ) + assert actual == f"{project_dir}/private/private_config.yaml" + + def test_strategy_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "systems.provided.rob_system.config.yaml", + ) + assert actual == f"{project_dir}/systems/provided/rob_system/config.yaml" + + def test_pickled_backtest(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private/backtests/fut_strategy_v1_8/20260101_210632_backtest.pck", + ) + assert ( + actual + == f"{project_dir}/private/backtests/fut_strategy_v1_8/20260101_210632_backtest.pck" + ) + + def test_ib_instrument_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.config.ib_config_futures.csv", + ) + assert actual == f"{project_dir}/sysbrokers/IB/config/ib_config_futures.csv" + + def test_ib_fx_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.config.ib_config_spot_FX.csv", + ) + assert actual == f"{project_dir}/sysbrokers/IB/config/ib_config_spot_FX.csv" + + def test_ib_trading_hours_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.ib_config_trading_hours.yaml", + ) + assert actual == f"{project_dir}/sysbrokers/IB/ib_config_trading_hours.yaml" + + def test_email_store_filename(self): + actual = resolve_path_and_filename_for_package( + "/home/rob/logs/email_store.log", + ) + assert actual == "/home/rob/logs/email_store.log" + @pytest.mark.skipif(sys.platform in ["linux", "darwin"], reason="Only runs on windows") class TestFileUtilsWindoze: @@ -122,7 +202,6 @@ def test_resolve_path_absolute_trailing(self): actual = get_resolved_pathname("C:\\home\\rob\\") assert actual == "C:\\home\\rob" - @pytest.mark.xfail(reason="Cannot work with new implementation") def test_resolve_path_absolute_dotted(self): actual = get_resolved_pathname(".home.rob") assert actual == "\\home\\rob" @@ -132,9 +211,10 @@ def test_resolve_path_relative(self, project_dir): assert actual == f"{project_dir}\\syscore\\tests" def test_resolve_path_non_existent(self, project_dir): - with pytest.raises(ModuleNotFoundError): - get_resolved_pathname("syscore.testz") + actual = get_resolved_pathname("syscore.testz") + assert actual == f"{project_dir}\\syscore\\testz" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_dotted_dir_name(self, tmp_path): directory = tmp_path / "dir.name.with.dots" directory.mkdir() @@ -143,6 +223,7 @@ def test_resolve_dotted_dir_name(self, tmp_path): resolved_path = get_resolved_pathname(str(file)) assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_dotted_file_name(self, tmp_path): directory = tmp_path / "dir_name" directory.mkdir() @@ -159,7 +240,6 @@ def test_resolve_package_combined(self): actual = resolve_path_and_filename_for_package("C:\\home\\rob\\file.csv") assert actual == "C:\\home\\rob\\file.csv" - @pytest.mark.xfail(reason="Cannot work with new implementation") def test_resolve_package_combined_dotted(self): actual = resolve_path_and_filename_for_package(".home.rob.file.csv") assert actual == "\\home\\rob\\file.csv" @@ -174,12 +254,14 @@ def test_resolve_package_module_combined(self, project_dir): ) assert actual == f"{project_dir}\\syscore\\tests\\pricetestdata.csv" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_package_module_combined_dotted_filename(self, project_dir): actual = resolve_path_and_filename_for_package( "syscore.tests.price.test.data.csv" ) assert actual == f"{project_dir}\\syscore\\tests\\price.test.data.csv" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( self, tmp_path ): @@ -192,6 +274,7 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( ) assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt" + @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( self, tmp_path ): @@ -203,3 +286,88 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( f"{tmp_path}\\dir_name\\dotted.filename.txt" ) assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt" + + # No Separate filename + + def test_csv_data(self, project_dir): + actual = resolve_path_and_filename_for_package( + "data.futures.csvconfig.instrumentconfig.csv" + ) + assert ( + actual == f"{project_dir}\\data\\futures\\csvconfig\\instrumentconfig.csv" + ) + + def test_logging(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syslogging.logging_prod.yaml", + ) + assert actual == f"{project_dir}\\syslogging\\logging_prod.yaml" + + def test_config_defaults(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysdata.config.defaults.yaml", + ) + assert actual == f"{project_dir}\\sysdata\\config\\defaults.yaml" + + def test_control_config_defaults(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscontrol.control_config.yaml", + ) + assert actual == f"{project_dir}\\syscontrol\\control_config.yaml" + + def test_private_config_dots(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private.private_config.yaml", + ) + assert actual == f"{project_dir}\\private\\private_config.yaml" + + def test_private_config_slash(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private/private_config.yaml", + ) + assert actual == f"{project_dir}\\private\\private_config.yaml" + + def test_strategy_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "systems.provided.rob_system.config.yaml", + ) + assert actual == f"{project_dir}\\systems\\provided\\rob_system\\config.yaml" + + def test_pickled_backtest(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private/backtests/fut_strategy_v1_8/20260101_210632_backtest.pck", + ) + assert ( + actual + == f"{project_dir}\\private\\backtests\\fut_strategy_v1_8\\20260101_210632_backtest.pck" + ) + + def test_ib_instrument_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.config.ib_config_futures.csv", + ) + assert actual == f"{project_dir}\\sysbrokers\\IB\\config\\ib_config_futures.csv" + + def test_ib_fx_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.config.ib_config_spot_FX.csv", + ) + assert actual == f"{project_dir}\\sysbrokers\\IB\\config\\ib_config_spot_FX.csv" + + def test_ib_trading_hours_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.ib_config_trading_hours.yaml", + ) + assert actual == f"{project_dir}\\sysbrokers\\IB\\ib_config_trading_hours.yaml" + + def test_email_store_filename(self): + actual = resolve_path_and_filename_for_package( + "C:\\home\\rob\\logs\\email_store.log", + ) + assert actual == "C:\\home\\rob\\logs\\email_store.log" + + def test_convert_email_store_filename(self): + actual = resolve_path_and_filename_for_package( + "/home/rob/logs/email_store.log", + ) + assert actual == "\\home\\rob\\logs\\email_store.log" From 899778a3dec3e84fb5986a0b7827daf4122bcb8f Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Fri, 2 Jan 2026 13:16:48 +0000 Subject: [PATCH 14/28] revert --- sysdata/config/private_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sysdata/config/private_config.py b/sysdata/config/private_config.py index 51357bd5ced..652e2febe96 100644 --- a/sysdata/config/private_config.py +++ b/sysdata/config/private_config.py @@ -1,4 +1,4 @@ -from syscore.fileutils import resolve_path_and_filename_for_package, does_filename_exist +from syscore.fileutils import resolve_path_and_filename_for_package from syscore.constants import arg_not_supplied from sysdata.config.private_directory import get_private_config_dir @@ -7,7 +7,7 @@ PRIVATE_CONFIG_FILE = "private_config.yaml" -def get_private_config_as_dict() -> dict: +def get_private_config_as_dict(filename: str = arg_not_supplied) -> dict: dir = get_private_config_dir() try: private_file = resolve_path_and_filename_for_package(dir, PRIVATE_CONFIG_FILE) From 22d53d036afdf3eae0bea43ea0b809a8e7db9a09 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Fri, 2 Jan 2026 16:59:02 +0000 Subject: [PATCH 15/28] pass full path for prod logging config --- docs/production.md | 2 +- syslogging/logger.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/production.md b/docs/production.md index 38c1f03aad9..d42b0e45b5a 100644 --- a/docs/production.md +++ b/docs/production.md @@ -720,7 +720,7 @@ In production, the requirements are more complex than in sim. As well as the con Configure the default production setup with: ``` -PYSYS_LOGGING_CONFIG=syslogging.logging_prod.yaml +PYSYS_LOGGING_CONFIG=/home/user/config/logging_prod.yaml ``` At the client side, (pysystemtrade) there are three handlers: socket, console, and email. There is a server (separate process) for the socket handler. More details on each below diff --git a/syslogging/logger.py b/syslogging/logger.py index f046513073f..6f349af3267 100644 --- a/syslogging/logger.py +++ b/syslogging/logger.py @@ -98,10 +98,9 @@ def _configure_sim(): def _configure_prod(logging_config_file): print(f"Attempting to configure prod logging from {logging_config_file}") - config_path = resolve_path_and_filename_for_package(logging_config_file) - if Path(config_path).exists(): + if Path(logging_config_file).exists(): try: - config = parse_config(path=config_path) + config = parse_config(path=logging_config_file) host, port = _get_log_server_config(config) try: _check_log_server(host, port) @@ -116,7 +115,9 @@ def _configure_prod(logging_config_file): print(f"ERROR: Problem configuring prod logging, reverting to sim: {exc}") _configure_sim() else: - print(f"ERROR: prod logging config '{config_path}' not found, reverting to sim") + print( + f"ERROR: prod logging config '{logging_config_file}' not found, reverting to sim" + ) _configure_sim() From e72e19b05a5cdf3e1fc344bae479737b2091b2d2 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Fri, 2 Jan 2026 16:59:26 +0000 Subject: [PATCH 16/28] deprecation warning for not passing filename --- syscore/fileutils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/syscore/fileutils.py b/syscore/fileutils.py index d5bf79c658b..04beb50c263 100755 --- a/syscore/fileutils.py +++ b/syscore/fileutils.py @@ -5,6 +5,7 @@ import os from pathlib import Path from typing import List, Tuple +import warnings from syscore.dateutils import SECONDS_PER_DAY @@ -136,6 +137,11 @@ def resolve_path_and_filename_for_package( path_and_filename_as_list = transform_path_into_list(path_and_filename) if separate_filename is None: + warnings.warn( + "Passing no 'separate_filename' is deprecated, " + "and will cause an error in future", + DeprecationWarning, + ) ( path_as_list, separate_filename, From 1665c28ed8e35e64eac4c41d326d763127aa5640 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Fri, 2 Jan 2026 16:59:59 +0000 Subject: [PATCH 17/28] passing config defaults filename separately --- sysdata/config/defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sysdata/config/defaults.py b/sysdata/config/defaults.py index a1268a0495c..c6fb8da035d 100644 --- a/sysdata/config/defaults.py +++ b/sysdata/config/defaults.py @@ -10,7 +10,7 @@ from syscore.constants import arg_not_supplied import yaml -DEFAULT_FILENAME = "sysdata.config.defaults.yaml" +DEFAULT_FILENAME = "defaults.yaml" def get_system_defaults_dict(filename: str = arg_not_supplied) -> dict: @@ -20,7 +20,7 @@ def get_system_defaults_dict(filename: str = arg_not_supplied) -> dict: """ if filename is arg_not_supplied: filename = DEFAULT_FILENAME - default_file = resolve_path_and_filename_for_package(filename) + default_file = resolve_path_and_filename_for_package("sysdata.config", filename) with open(default_file) as file_to_parse: default_dict = yaml.load(file_to_parse, Loader=yaml.FullLoader) From 221f8f28dc6ab0d7e0f0791348387e1b5a825ea9 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Mon, 5 Jan 2026 21:48:03 +0000 Subject: [PATCH 18/28] simplify config --- sysbrokers/IB/ib_trading_hours.py | 26 ++++++++++----- syscore/fileutils.py | 11 +++---- sysdata/config/configdata.py | 20 +++--------- sysdata/config/control_config.py | 15 ++++++--- sysdata/config/private_config.py | 30 +++++++++++++---- sysdata/config/private_directory.py | 25 --------------- .../private_control_config.yaml | 2 +- sysdata/tests/test_config.py | 32 +++++++++++++++++-- syslogging/logger.py | 1 - systems/basesystem.py | 2 +- systems/tests/test_position_sizing.py | 2 +- 11 files changed, 93 insertions(+), 73 deletions(-) delete mode 100644 sysdata/config/private_directory.py diff --git a/sysbrokers/IB/ib_trading_hours.py b/sysbrokers/IB/ib_trading_hours.py index 256ea40c9c0..9a054986233 100644 --- a/sysbrokers/IB/ib_trading_hours.py +++ b/sysbrokers/IB/ib_trading_hours.py @@ -1,26 +1,36 @@ import datetime from ib_insync import ContractDetails as ibContractDetails -from sysdata.config.private_directory import get_full_path_for_private_config +from sysdata.config.private_config import ( + get_private_config_dir, +) from sysobjects.production.trading_hours.trading_hours import ( tradingHours, listOfTradingHours, ) -from syscore.fileutils import does_filename_exist +from syscore.fileutils import ( + does_filename_exist, + does_path_exist, + resolve_path_and_filename_for_package, +) from sysdata.config.production_config import get_production_config from sysdata.production.trading_hours import read_trading_hours IB_CONFIG_TRADING_HOURS_FILE = "sysbrokers.IB.ib_config_trading_hours.yaml" -PRIVATE_CONFIG_TRADING_HOURS_FILE = get_full_path_for_private_config( - "private_config_trading_hours.yaml" -) +PRIVATE_CONFIG_TRADING_HOURS_FILE = "private_config_trading_hours.yaml" def get_saved_trading_hours(): - if does_filename_exist(PRIVATE_CONFIG_TRADING_HOURS_FILE): - return read_trading_hours(PRIVATE_CONFIG_TRADING_HOURS_FILE) + private_path = resolve_path_and_filename_for_package( + get_private_config_dir(), PRIVATE_CONFIG_TRADING_HOURS_FILE + ) + if does_path_exist(private_path): + return read_trading_hours(private_path) else: - return read_trading_hours(IB_CONFIG_TRADING_HOURS_FILE) + default_path = resolve_path_and_filename_for_package( + IB_CONFIG_TRADING_HOURS_FILE + ) + return read_trading_hours(default_path) def get_trading_hours_from_contract_details( diff --git a/syscore/fileutils.py b/syscore/fileutils.py index 04beb50c263..8e92baae999 100755 --- a/syscore/fileutils.py +++ b/syscore/fileutils.py @@ -5,7 +5,6 @@ import os from pathlib import Path from typing import List, Tuple -import warnings from syscore.dateutils import SECONDS_PER_DAY @@ -137,11 +136,6 @@ def resolve_path_and_filename_for_package( path_and_filename_as_list = transform_path_into_list(path_and_filename) if separate_filename is None: - warnings.warn( - "Passing no 'separate_filename' is deprecated, " - "and will cause an error in future", - DeprecationWarning, - ) ( path_as_list, separate_filename, @@ -387,6 +381,7 @@ def full_filename_for_file_in_home_dir(filename: str) -> str: def does_filename_exist(filename: str) -> bool: + # TODO resolved_filename = resolve_path_and_filename_for_package(filename) file_exists = does_resolved_filename_exist(resolved_filename) return file_exists @@ -395,3 +390,7 @@ def does_filename_exist(filename: str) -> bool: def does_resolved_filename_exist(resolved_filename: str) -> bool: file_exists = os.path.isfile(resolved_filename) return file_exists + + +def does_path_exist(filename: str) -> bool: + return Path(filename).exists() diff --git a/sysdata/config/configdata.py b/sysdata/config/configdata.py index 70c9be923b9..a07004b5d66 100644 --- a/sysdata/config/configdata.py +++ b/sysdata/config/configdata.py @@ -12,8 +12,6 @@ """ -from pathlib import Path - import yaml from syscore.exceptions import missingData @@ -22,12 +20,8 @@ from sysdata.config.defaults import get_system_defaults_dict from sysdata.config.private_config import ( get_private_config_as_dict, - PRIVATE_CONFIG_FILE, -) -from sysdata.config.private_directory import ( - get_full_path_for_private_config, - PRIVATE_CONFIG_DIR_ENV_VAR, ) + from syslogging.logger import * from sysdata.config.fill_config_dict_with_defaults import fill_config_dict_with_defaults @@ -57,7 +51,7 @@ def __init__( multiple elements, latter elements will overwrite earlier ones) - :type config_object: str or dict + :type config_object: str or dict or list :returns: new Config object @@ -179,11 +173,10 @@ def _create_config_from_dict(self, config_object): self.add_elements(attr_names) - def system_init(self, base_system): + def system_init(self): """ This is run when added to a base system - :param base_system :return: nothing """ @@ -286,12 +279,7 @@ def default_config(cls): if hasattr(cls, "evaluated"): return cls.evaluated - if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR): - config = Config( - private_filename=get_full_path_for_private_config(PRIVATE_CONFIG_FILE) - ) - else: - config = Config() + config = Config(get_private_config_as_dict()) config.fill_with_defaults() cls.evaluated = config diff --git a/sysdata/config/control_config.py b/sysdata/config/control_config.py index 72d1f6c150f..ae76c9171b6 100644 --- a/sysdata/config/control_config.py +++ b/sysdata/config/control_config.py @@ -1,5 +1,6 @@ from sysdata.config.configdata import Config -from sysdata.config.private_directory import get_full_path_for_private_config +from syscore.fileutils import resolve_path_and_filename_for_package +from sysdata.config.private_config import get_private_config_dir from yaml.parser import ParserError PRIVATE_CONTROL_CONFIG_FILE = "private_control_config.yaml" @@ -7,12 +8,18 @@ def get_control_config() -> Config: - private_control_path = get_full_path_for_private_config(PRIVATE_CONTROL_CONFIG_FILE) + dir = get_private_config_dir() + private_control_path = resolve_path_and_filename_for_package( + dir, PRIVATE_CONTROL_CONFIG_FILE + ) + default_control_path = resolve_path_and_filename_for_package( + DEFAULT_CONTROL_CONFIG_FILE + ) try: control_config = Config( private_filename=private_control_path, - default_filename=DEFAULT_CONTROL_CONFIG_FILE, + default_filename=default_control_path, ) control_config.fill_with_defaults() @@ -21,7 +28,7 @@ def get_control_config() -> Config: except FileNotFoundError: raise Exception( "Need to have either %s or %s or both present:" - % (str(DEFAULT_CONTROL_CONFIG_FILE), str(private_control_path)) + % (private_control_path, default_control_path) ) except BaseException as be: raise Exception("Problem reading control config: %s" % str(be)) diff --git a/sysdata/config/private_config.py b/sysdata/config/private_config.py index 652e2febe96..d576e545857 100644 --- a/sysdata/config/private_config.py +++ b/sysdata/config/private_config.py @@ -1,22 +1,38 @@ +import os +import yaml +from pathlib import Path + from syscore.fileutils import resolve_path_and_filename_for_package from syscore.constants import arg_not_supplied -from sysdata.config.private_directory import get_private_config_dir -import yaml +DEFAULT_PRIVATE_DIR = "private" PRIVATE_CONFIG_FILE = "private_config.yaml" +PRIVATE_CONFIG_DIR_ENV_VAR = "PYSYS_PRIVATE_CONFIG_DIR" def get_private_config_as_dict(filename: str = arg_not_supplied) -> dict: - dir = get_private_config_dir() + private_dir = get_private_config_dir() + if filename is arg_not_supplied: + filename = PRIVATE_CONFIG_FILE try: - private_file = resolve_path_and_filename_for_package(dir, PRIVATE_CONFIG_FILE) - with open(private_file) as file_to_parse: + private_path = resolve_path_and_filename_for_package(private_dir, filename) + with open(private_path) as file_to_parse: private_dict = yaml.load(file_to_parse, Loader=yaml.FullLoader) return private_dict - except (FileNotFoundError, ModuleNotFoundError): + except Exception: print( - f"Private configuration '{dir}.{PRIVATE_CONFIG_FILE}' does not exist; no problem if running in sim mode" + f"Private configuration '{private_path}' is missing or " + f"misconfigured; no problem if running in sim mode" ) return {} + + +def get_private_config_dir(): + if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR): + private_config_dir = Path(os.environ[PRIVATE_CONFIG_DIR_ENV_VAR]) + else: + private_config_dir = Path(DEFAULT_PRIVATE_DIR) + + return str(private_config_dir) diff --git a/sysdata/config/private_directory.py b/sysdata/config/private_directory.py deleted file mode 100644 index 0b7b6ec400b..00000000000 --- a/sysdata/config/private_directory.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -from pathlib import Path - -DEFAULT_PRIVATE_DIR = "private" -PRIVATE_CONFIG_DIR_ENV_VAR = "PYSYS_PRIVATE_CONFIG_DIR" - - -def get_full_path_for_private_config(filename: str): - ## FIXME: should use os path join instead of '/'? - - if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR): - private_config_path = f"{os.environ[PRIVATE_CONFIG_DIR_ENV_VAR]}/{filename}" - else: - private_config_path = f"{DEFAULT_PRIVATE_DIR}/{filename}" - - return private_config_path - - -def get_private_config_dir(): - if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR): - private_config_dir = Path(os.environ[PRIVATE_CONFIG_DIR_ENV_VAR]) - else: - private_config_dir = Path(DEFAULT_PRIVATE_DIR) - - return str(private_config_dir) diff --git a/sysdata/tests/custom_private_config/private_control_config.yaml b/sysdata/tests/custom_private_config/private_control_config.yaml index 836a43e9c42..234a7e95a6f 100644 --- a/sysdata/tests/custom_private_config/private_control_config.yaml +++ b/sysdata/tests/custom_private_config/private_control_config.yaml @@ -1,2 +1,2 @@ process_configuration_start_time: - run_stack_handler: '01:00' \ No newline at end of file + run_stack_handler: '01:00' diff --git a/sysdata/tests/test_config.py b/sysdata/tests/test_config.py index 20353713927..728efd09a56 100644 --- a/sysdata/tests/test_config.py +++ b/sysdata/tests/test_config.py @@ -1,10 +1,34 @@ from sysdata.config.configdata import Config from sysdata.config.control_config import get_control_config -from sysdata.config.private_directory import PRIVATE_CONFIG_DIR_ENV_VAR +from sysdata.config.private_config import PRIVATE_CONFIG_DIR_ENV_VAR class TestConfig: - def test_default(self): + def test_init_dict(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + config = Config(dict(parameters=dict(p1=3, p2=4.6), another_thing="foo")) + assert config.as_dict()["parameters"]["p2"] == 4.6 + assert config.as_dict()["another_thing"] == "foo" + + def test_init_str(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + config = Config("systems.provided.example.exampleconfig.yaml") + assert config.as_dict()["forecast_cap"] == 21.0 + + def test_init_list(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + config = Config( + [ + "systems.provided.example.exampleconfig.yaml", + dict(parameters=dict(p1=3, p2=4.6), another_thing="foo"), + ] + ) + assert config.as_dict()["forecast_cap"] == 21.0 + assert config.as_dict()["parameters"]["p2"] == 4.6 + assert config.as_dict()["another_thing"] == "foo" + + def test_default(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) Config.reset() config = Config.default_config() assert config.get_element("ib_idoffset") == 100 @@ -25,7 +49,9 @@ def test_bad_custom_dir(self, monkeypatch): config = Config.default_config() assert config.get_element("ib_idoffset") == 100 - def test_default_control(self): + def test_default_control(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + Config.reset() config = get_control_config() assert ( config.as_dict()["process_configuration_start_time"]["run_stack_handler"] diff --git a/syslogging/logger.py b/syslogging/logger.py index 6f349af3267..399ff0e3089 100644 --- a/syslogging/logger.py +++ b/syslogging/logger.py @@ -6,7 +6,6 @@ import syslogging from syslogging.adapter import * from syslogging.pyyaml_env import parse_config -from syscore.fileutils import resolve_path_and_filename_for_package CONFIG_ENV_VAR = "PYSYS_LOGGING_CONFIG" LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s %(message)s" diff --git a/systems/basesystem.py b/systems/basesystem.py index 4d4d4487291..b66679cfdd0 100644 --- a/systems/basesystem.py +++ b/systems/basesystem.py @@ -77,7 +77,7 @@ def __init__( self._config = config self._log = log - self.config.system_init(self) + self.config.system_init() self.data.system_init(self) self._setup_stages(stage_list) self._cache = systemCache(self) diff --git a/systems/tests/test_position_sizing.py b/systems/tests/test_position_sizing.py index f3e4fec0f70..c52c3f9b24d 100644 --- a/systems/tests/test_position_sizing.py +++ b/systems/tests/test_position_sizing.py @@ -1,6 +1,6 @@ import unittest from _pytest.monkeypatch import MonkeyPatch -from sysdata.config.private_directory import PRIVATE_CONFIG_DIR_ENV_VAR +from sysdata.config.private_config import PRIVATE_CONFIG_DIR_ENV_VAR from systems.tests.testdata import get_test_object_futures_with_comb_forecasts from systems.basesystem import System from systems.positionsizing import PositionSizing From ec5b09170668c093b53c700cdb1debd610e07c61 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Tue, 6 Jan 2026 16:36:08 +0000 Subject: [PATCH 19/28] better type hints --- syscore/fileutils.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/syscore/fileutils.py b/syscore/fileutils.py index 8e92baae999..1e415a4e3b1 100755 --- a/syscore/fileutils.py +++ b/syscore/fileutils.py @@ -4,7 +4,7 @@ from importlib import import_module import os from pathlib import Path -from typing import List, Tuple +from typing import List, Tuple, TextIO from syscore.dateutils import SECONDS_PER_DAY @@ -26,7 +26,7 @@ def rename_files_with_extension_in_pathname_as_archive_files( pathname: str, extension: str = ".txt", archive_extension: str = ".arch" -): +) -> None: """ Find all the files with a particular extension in a directory, and rename them eg thing.txt will become thing_yyyymmdd.txt where yyyymmdd is todays date @@ -47,7 +47,7 @@ def rename_files_with_extension_in_pathname_as_archive_files( def rename_file_as_archive( full_filename: str, old_extension: str = ".txt", archive_extension: str = ".arch" -): +) -> None: """ Rename a file with archive suffix and extension eg thing.txt will become thing_yyyymmdd.arch where yyyymmdd is todays date @@ -62,8 +62,8 @@ def rename_file_as_archive( def delete_old_files_with_extension_in_pathname( - pathname: str, days_old=30, extension=".arch" -): + pathname: str, days_old: int = 30, extension: str = ".arch" +) -> None: """ Find all the files with a particular extension in a directory, and delete them if older than x days @@ -77,7 +77,7 @@ def delete_old_files_with_extension_in_pathname( delete_file_if_too_old(filename, days_old=days_old) -def delete_file_if_too_old(full_filename_with_ext: str, days_old: int = 30): +def delete_file_if_too_old(full_filename_with_ext: str, days_old: int = 30) -> None: file_age = get_file_or_folder_age_in_days(full_filename_with_ext) if file_age > days_old: print("Deleting %s" % full_filename_with_ext) @@ -103,7 +103,7 @@ def get_file_or_folder_age_in_days(full_filename_with_ext: str) -> float: def resolve_path_and_filename_for_package( - path_and_filename: str, separate_filename=None + path_and_filename: str, separate_filename: str | None = None ) -> str: """ A way of resolving relative and absolute filenames, and dealing with awkward OS specific things @@ -230,8 +230,8 @@ def add_reserved_characters_to_pathname(pathname: str) -> str: def extract_filename_from_combined_path_and_filename_list( - path_and_filename_as_list: list, -) -> Tuple[list, str]: + path_and_filename_as_list: list[str], +) -> Tuple[list[str], str]: """ >>> extract_filename_from_combined_path_and_filename_list(['home', 'rob','file', 'csv']) (['home', 'rob'], 'file.csv') @@ -308,12 +308,14 @@ def get_absolute_linux_pathname_from_list(path_as_list: List[str]) -> str: '/home/rob' """ pathname = os.path.join(*path_as_list) - pathname = os.path.sep + pathname + pathname = os.path.sep + str(pathname) return pathname -def get_absolute_windows_pathname_from_list(path_as_list: list) -> str: +def get_absolute_windows_pathname_from_list( + path_as_list: list[str], +) -> str: """ Test will fail on linux >>> get_absolute_windows_pathname_from_list(['C:','home','rob']) @@ -327,7 +329,7 @@ def get_absolute_windows_pathname_from_list(path_as_list: list) -> str: pathname = os.path.join(*path_as_list) - return pathname + return str(pathname) """ @@ -337,7 +339,9 @@ def get_absolute_windows_pathname_from_list(path_as_list: list) -> str: """ -def write_list_of_lists_as_html_table_in_file(file, list_of_lists: list): +def write_list_of_lists_as_html_table_in_file( + file: TextIO, list_of_lists: list[str] +) -> None: file.write("") for sublist in list_of_lists: file.write("
") @@ -347,7 +351,9 @@ def write_list_of_lists_as_html_table_in_file(file, list_of_lists: list): file.write("
") -def files_with_extension_in_pathname(pathname: str, extension=".csv") -> List[str]: +def files_with_extension_in_pathname( + pathname: str, extension: str = ".csv" +) -> List[str]: """ Find all the files with a particular extension in a directory @@ -360,7 +366,7 @@ def files_with_extension_in_pathname(pathname: str, extension=".csv") -> List[st def files_with_extension_in_resolved_pathname( - resolved_pathname: str, extension=".csv" + resolved_pathname: str, extension: str = ".csv" ) -> List[str]: """ Find all the files with a particular extension in a directory From f4b9f24a57effb547970f52964c22738c03875aa Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Tue, 13 Jan 2026 15:39:33 +0000 Subject: [PATCH 20/28] new file resolving implementation --- sysbrokers/IB/ib_trading_hours.py | 5 +- syscore/__init__.py | 7 ++ syscore/fileutils.py | 177 +++++++----------------------- syscore/tests/test_fileutils.py | 30 +++-- 4 files changed, 74 insertions(+), 145 deletions(-) diff --git a/sysbrokers/IB/ib_trading_hours.py b/sysbrokers/IB/ib_trading_hours.py index 9a054986233..77c5b40b3f9 100644 --- a/sysbrokers/IB/ib_trading_hours.py +++ b/sysbrokers/IB/ib_trading_hours.py @@ -9,8 +9,7 @@ listOfTradingHours, ) from syscore.fileutils import ( - does_filename_exist, - does_path_exist, + does_resolved_filename_exist, resolve_path_and_filename_for_package, ) from sysdata.config.production_config import get_production_config @@ -24,7 +23,7 @@ def get_saved_trading_hours(): private_path = resolve_path_and_filename_for_package( get_private_config_dir(), PRIVATE_CONFIG_TRADING_HOURS_FILE ) - if does_path_exist(private_path): + if does_resolved_filename_exist(private_path): return read_trading_hours(private_path) else: default_path = resolve_path_and_filename_for_package( diff --git a/syscore/__init__.py b/syscore/__init__.py index e69de29bb2d..61008830ee9 100755 --- a/syscore/__init__.py +++ b/syscore/__init__.py @@ -0,0 +1,7 @@ +from importlib.util import find_spec +from pathlib import Path + +module_spec = find_spec(__name__) +path = Path(module_spec.origin) + +PYSYS_PROJECT_DIR = path.parent.parent diff --git a/syscore/fileutils.py b/syscore/fileutils.py index 1e415a4e3b1..ad42f0dfaf7 100755 --- a/syscore/fileutils.py +++ b/syscore/fileutils.py @@ -1,21 +1,13 @@ import glob import datetime import time -from importlib import import_module import os -from pathlib import Path +from pathlib import Path, PurePath from typing import List, Tuple, TextIO +from syscore import PYSYS_PROJECT_DIR from syscore.dateutils import SECONDS_PER_DAY -# DO NOT DELETE: all these are unused: but are required to get the filename padding to work - - -""" - - FILES IN DIRECTORIES - -""" """ @@ -134,22 +126,21 @@ def resolve_path_and_filename_for_package( """ - path_and_filename_as_list = transform_path_into_list(path_and_filename) + path_and_filename_as_list = _transform_path_into_list(path_and_filename) if separate_filename is None: ( path_as_list, separate_filename, - ) = extract_filename_from_combined_path_and_filename_list( + ) = _extract_filename_from_combined_path_and_filename_list( path_and_filename_as_list ) else: path_as_list = path_and_filename_as_list - resolved_pathname = get_pathname_from_list(path_as_list) + absolute_path = _make_absolute(path_as_list) + result = absolute_path / separate_filename - resolved_path_and_filename = os.path.join(resolved_pathname, separate_filename) - - return resolved_path_and_filename + return str(result) def get_resolved_pathname(pathname: str) -> str: @@ -168,60 +159,71 @@ def get_resolved_pathname(pathname: str) -> str: """ - if isinstance(pathname, Path): + if isinstance(pathname, Path) and pathname.exists(): # special case when already a Path - pathname = str(pathname.absolute()) - - if "@" in pathname or "::" in pathname: - # This is an ssh address for rsync - don't change - return pathname + return str(pathname.absolute()) + else: + pathname = str(pathname) + if "@" in pathname or "::" in pathname: + # This is an ssh address for rsync - don't change + return pathname - # Turn /,\ into . so system independent - path_as_list = transform_path_into_list(pathname) - resolved_pathname = get_pathname_from_list(path_as_list) + path_as_list = _transform_path_into_list(pathname) + result = _make_absolute(path_as_list) - return resolved_pathname + return str(result) ## something unlikely to occur naturally in a pathname RESERVED_CHARACTERS = "&!*" -def transform_path_into_list(pathname: str) -> List[str]: +def _make_absolute(path_as_list: list[str]) -> PurePath: + path_obj = PurePath(*path_as_list) + if not path_obj.is_absolute(): + path_obj = PYSYS_PROJECT_DIR / path_obj + + return path_obj + + +def _transform_path_into_list(pathname: str) -> List[str]: """ - >>> path_as_list("/home/rob/test.csv") + >>> _transform_path_into_list("/home/rob/test.csv") ['', 'home', 'rob', 'test', 'csv'] - >>> path_as_list("/home/rob/") + >>> _transform_path_into_list("/home/rob/") ['', 'home', 'rob'] - >>> path_as_list(".home.rob") + >>> _transform_path_into_list(".home.rob") ['', 'home', 'rob'] - >>> path_as_list('C:\\home\\rob\\'') + >>> _transform_path_into_list('C:\\home\\rob\\'') ['C:', 'home', 'rob'] - >>> path_as_list('C:\\home\\rob\\test.csv') + >>> _transform_path_into_list('C:\\home\\rob\\test.csv') ['C:', 'home', 'rob', 'test', 'csv'] - >>> path_as_list("syscore.tests.fileutils.csv") + >>> _transform_path_into_list("syscore.tests.fileutils.csv") ['syscore', 'tests', 'fileutils', 'csv'] - >>> path_as_list("syscore.tests") + >>> _transform_path_into_list("syscore.tests") ['syscore', 'tests'] """ - pathname_replace = add_reserved_characters_to_pathname(pathname) + pathname_replace = _add_reserved_characters_to_pathname(pathname) path_as_list = pathname_replace.rsplit(RESERVED_CHARACTERS) if path_as_list[-1] == "": path_as_list.pop() + if path_as_list[0] == "": + path_as_list[0] = f"{os.sep}{path_as_list[0]}" + return path_as_list -def add_reserved_characters_to_pathname(pathname: str) -> str: +def _add_reserved_characters_to_pathname(pathname: str) -> str: pathname_replace = pathname.replace(".", RESERVED_CHARACTERS) pathname_replace = pathname_replace.replace("/", RESERVED_CHARACTERS) pathname_replace = pathname_replace.replace("\\", RESERVED_CHARACTERS) @@ -229,11 +231,11 @@ def add_reserved_characters_to_pathname(pathname: str) -> str: return pathname_replace -def extract_filename_from_combined_path_and_filename_list( +def _extract_filename_from_combined_path_and_filename_list( path_and_filename_as_list: list[str], ) -> Tuple[list[str], str]: """ - >>> extract_filename_from_combined_path_and_filename_list(['home', 'rob','file', 'csv']) + >>> _extract_filename_from_combined_path_and_filename_list(['home', 'rob','file', 'csv']) (['home', 'rob'], 'file.csv') """ ## need -2 because want extension @@ -245,93 +247,6 @@ def extract_filename_from_combined_path_and_filename_list( return path_and_filename_as_list, separate_filename -def get_pathname_from_list(path_as_list: List[str]) -> str: - """ - >>> get_pathname_from_list(['C:', 'home', 'rob']) - 'C:\\home\\rob' - >>> get_pathname_from_list(['','home','rob']) - '/home/rob' - >>> get_pathname_from_list(['syscore','tests']) - '/home/rob/pysystemtrade/syscore/tests' - """ - if path_as_list[0] == "": - # path_type_absolute - resolved_pathname = get_absolute_linux_pathname_from_list(path_as_list[1:]) - elif is_windoze_path_list(path_as_list): - # windoze - resolved_pathname = get_absolute_windows_pathname_from_list(path_as_list) - else: - # relative - resolved_pathname = get_relative_pathname_from_list(path_as_list) - - return resolved_pathname - - -def is_windoze_path_list(path_as_list: List[str]) -> bool: - """ - >>> is_windoze_path_list(['C:']) - True - >>> is_windoze_path_list(['wibble']) - False - """ - return path_as_list[0].endswith(":") - - -def get_relative_pathname_from_list(path_as_list: List[str]) -> str: - """ - - >>> get_relative_pathname_from_list(['syscore','tests']) - '/home/rob/pysystemtrade/syscore/tests' - """ - package_name = path_as_list[0] - paths_or_files = path_as_list[1:] - - if len(paths_or_files) == 0: - directory_name_of_package = os.path.dirname( - import_module(package_name).__file__ - ) - return directory_name_of_package - - last_item_in_list = path_as_list.pop() - pathname = os.path.join( - get_relative_pathname_from_list(path_as_list), last_item_in_list - ) - - return pathname - - -def get_absolute_linux_pathname_from_list(path_as_list: List[str]) -> str: - """ - Returns the absolute pathname from a list - - >>> get_absolute_linux_pathname_from_list(['home', 'rob']) - '/home/rob' - """ - pathname = os.path.join(*path_as_list) - pathname = os.path.sep + str(pathname) - - return pathname - - -def get_absolute_windows_pathname_from_list( - path_as_list: list[str], -) -> str: - """ - Test will fail on linux - >>> get_absolute_windows_pathname_from_list(['C:','home','rob']) - 'C:\\home\\rob' - """ - drive_part_of_path = path_as_list[0] - if drive_part_of_path.endswith(":"): - ## add back backslash - drive_part_of_path = drive_part_of_path.replace(":", ":\\") - path_as_list[0] = drive_part_of_path - - pathname = os.path.join(*path_as_list) - - return str(pathname) - - """ HTML @@ -381,22 +296,14 @@ def files_with_extension_in_resolved_pathname( def full_filename_for_file_in_home_dir(filename: str) -> str: - pathname = os.path.expanduser("~") - - return os.path.join(pathname, filename) + return str(Path.home() / filename) def does_filename_exist(filename: str) -> bool: - # TODO resolved_filename = resolve_path_and_filename_for_package(filename) file_exists = does_resolved_filename_exist(resolved_filename) return file_exists def does_resolved_filename_exist(resolved_filename: str) -> bool: - file_exists = os.path.isfile(resolved_filename) - return file_exists - - -def does_path_exist(filename: str) -> bool: - return Path(filename).exists() + return Path(resolved_filename).exists() diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py index 8cd250e2b37..657359e8b7f 100644 --- a/syscore/tests/test_fileutils.py +++ b/syscore/tests/test_fileutils.py @@ -35,22 +35,20 @@ def test_resolve_path_non_existent(self, project_dir): actual = get_resolved_pathname("syscore.testz") assert actual == f"{project_dir}/syscore/testz" - @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_dotted_dir_name(self, tmp_path): directory = tmp_path / "dir.name.with.dots" directory.mkdir() file = directory / "hello.txt" file.write_text("content", encoding="utf-8") - resolved_path = get_resolved_pathname(str(file)) + resolved_path = get_resolved_pathname(file) assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" - @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_dotted_file_name(self, tmp_path): directory = tmp_path / "dir_name" directory.mkdir() file = directory / "dotted.filename.txt" file.write_text("content", encoding="utf-8") - resolved_path = get_resolved_pathname(str(file)) + resolved_path = get_resolved_pathname(file) assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" def test_resolve_package_separate(self): @@ -81,14 +79,20 @@ def test_resolve_package_module_combined(self, project_dir): ) assert actual == f"{project_dir}/syscore/tests/pricetestdata.csv" - @pytest.mark.xfail(reason="Cannot work with existing implementation") + @pytest.mark.xfail(reason="Cannot work with existing or new implementation") def test_resolve_package_module_combined_dotted_filename(self, project_dir): actual = resolve_path_and_filename_for_package( "syscore.tests.price.test.data.csv" ) assert actual == f"{project_dir}/syscore/tests/price.test.data.csv" - @pytest.mark.xfail(reason="Cannot work with existing implementation") + def test_resolve_package_module_combined_dotted_filename(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests", "price.test.data.csv" + ) + assert actual == f"{project_dir}/syscore/tests/price.test.data.csv" + + @pytest.mark.xfail(reason="Cannot work with existing or new implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( self, tmp_path ): @@ -101,7 +105,7 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( ) assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" - @pytest.mark.xfail(reason="Cannot work with existing implementation") + @pytest.mark.xfail(reason="Cannot work with existing or new implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( self, tmp_path ): @@ -114,6 +118,18 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( ) assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + def test_resolve_resolve_path_and_filename_for_package_with_separate_dotted_file_name( + self, tmp_path + ): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}/dir_name/", "dotted.filename.txt" + ) + assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + # No Separate filename def test_csv_data(self, project_dir): From 96542be84f028b89d44eb8622460db978a89891a Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Mon, 19 Jan 2026 12:52:55 +0000 Subject: [PATCH 21/28] tidy --- docs/production.md | 2 +- sysdata/config/defaults.py | 4 ++-- syslogging/logger.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/production.md b/docs/production.md index d42b0e45b5a..38c1f03aad9 100644 --- a/docs/production.md +++ b/docs/production.md @@ -720,7 +720,7 @@ In production, the requirements are more complex than in sim. As well as the con Configure the default production setup with: ``` -PYSYS_LOGGING_CONFIG=/home/user/config/logging_prod.yaml +PYSYS_LOGGING_CONFIG=syslogging.logging_prod.yaml ``` At the client side, (pysystemtrade) there are three handlers: socket, console, and email. There is a server (separate process) for the socket handler. More details on each below diff --git a/sysdata/config/defaults.py b/sysdata/config/defaults.py index c6fb8da035d..a1268a0495c 100644 --- a/sysdata/config/defaults.py +++ b/sysdata/config/defaults.py @@ -10,7 +10,7 @@ from syscore.constants import arg_not_supplied import yaml -DEFAULT_FILENAME = "defaults.yaml" +DEFAULT_FILENAME = "sysdata.config.defaults.yaml" def get_system_defaults_dict(filename: str = arg_not_supplied) -> dict: @@ -20,7 +20,7 @@ def get_system_defaults_dict(filename: str = arg_not_supplied) -> dict: """ if filename is arg_not_supplied: filename = DEFAULT_FILENAME - default_file = resolve_path_and_filename_for_package("sysdata.config", filename) + default_file = resolve_path_and_filename_for_package(filename) with open(default_file) as file_to_parse: default_dict = yaml.load(file_to_parse, Loader=yaml.FullLoader) diff --git a/syslogging/logger.py b/syslogging/logger.py index 399ff0e3089..f046513073f 100644 --- a/syslogging/logger.py +++ b/syslogging/logger.py @@ -6,6 +6,7 @@ import syslogging from syslogging.adapter import * from syslogging.pyyaml_env import parse_config +from syscore.fileutils import resolve_path_and_filename_for_package CONFIG_ENV_VAR = "PYSYS_LOGGING_CONFIG" LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s %(message)s" @@ -97,9 +98,10 @@ def _configure_sim(): def _configure_prod(logging_config_file): print(f"Attempting to configure prod logging from {logging_config_file}") - if Path(logging_config_file).exists(): + config_path = resolve_path_and_filename_for_package(logging_config_file) + if Path(config_path).exists(): try: - config = parse_config(path=logging_config_file) + config = parse_config(path=config_path) host, port = _get_log_server_config(config) try: _check_log_server(host, port) @@ -114,9 +116,7 @@ def _configure_prod(logging_config_file): print(f"ERROR: Problem configuring prod logging, reverting to sim: {exc}") _configure_sim() else: - print( - f"ERROR: prod logging config '{logging_config_file}' not found, reverting to sim" - ) + print(f"ERROR: prod logging config '{config_path}' not found, reverting to sim") _configure_sim() From ac8220ce194ae008a3fd8a941937ae89ebcc105e Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Mon, 19 Jan 2026 16:04:01 +0000 Subject: [PATCH 22/28] docs updated --- docs/backtesting.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index a4e60859cd4..b2b872a948e 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -3123,7 +3123,6 @@ resolve_path_and_filename_for_package("\\home/rob.file.csv") ``` - These functions are used internally whenever a file name is passed in, so feel free to use any of these file formats when specifying eg a configuration filename. ``` ### Absolute: Windows (note use of double backslash in str) @@ -3136,6 +3135,31 @@ These functions are used internally whenever a file name is passed in, so feel f "syscore.tests.pricedata.csv" ``` +Obviously, using 'dots' as a separator brings limitations, like supporting directories or files with 'dots' in their names. This won't work: +``` +>>> resolve_path_and_filename_for_package("syscore/tests/price.test.data.csv") +'/home/user/pysystemtrade/syscore/tests/price/test/data.csv' +``` + +do this instead: +``` +>>> resolve_path_and_filename_for_package("syscore/tests", "price.test.data.csv") +'/home/user/pysystemtrade/syscore/tests/price.test.data.csv' +``` + +This will also not work: +``` +>>> get_resolved_pathname("data/dir.with.dots") +'/home/user/pysystemtrade/data/dir/with/dots' +``` + +do this instead: +``` +>>> from pathlib import Path +>>> get_resolved_pathname(Path("data/dir.with.dots")) +'/home/user/pysystemtrade/data/dir.with.dots' +``` + ## Logging ### Basic logging From fc6f33fdd6c5113503dc568deaf267a7169b148e Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Mon, 19 Jan 2026 16:04:12 +0000 Subject: [PATCH 23/28] tests updated --- syscore/tests/test_fileutils.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py index 657359e8b7f..6921c124e45 100644 --- a/syscore/tests/test_fileutils.py +++ b/syscore/tests/test_fileutils.py @@ -79,7 +79,7 @@ def test_resolve_package_module_combined(self, project_dir): ) assert actual == f"{project_dir}/syscore/tests/pricetestdata.csv" - @pytest.mark.xfail(reason="Cannot work with existing or new implementation") + @pytest.mark.xfail(reason="Cannot work with old or new implementation") def test_resolve_package_module_combined_dotted_filename(self, project_dir): actual = resolve_path_and_filename_for_package( "syscore.tests.price.test.data.csv" @@ -92,7 +92,7 @@ def test_resolve_package_module_combined_dotted_filename(self, project_dir): ) assert actual == f"{project_dir}/syscore/tests/price.test.data.csv" - @pytest.mark.xfail(reason="Cannot work with existing or new implementation") + @pytest.mark.xfail(reason="Cannot work with old or new implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( self, tmp_path ): @@ -105,7 +105,7 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( ) assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" - @pytest.mark.xfail(reason="Cannot work with existing or new implementation") + @pytest.mark.xfail(reason="Cannot work with old or new implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( self, tmp_path ): @@ -230,7 +230,6 @@ def test_resolve_path_non_existent(self, project_dir): actual = get_resolved_pathname("syscore.testz") assert actual == f"{project_dir}\\syscore\\testz" - @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_dotted_dir_name(self, tmp_path): directory = tmp_path / "dir.name.with.dots" directory.mkdir() @@ -239,7 +238,6 @@ def test_resolve_dotted_dir_name(self, tmp_path): resolved_path = get_resolved_pathname(str(file)) assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt" - @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_dotted_file_name(self, tmp_path): directory = tmp_path / "dir_name" directory.mkdir() @@ -270,14 +268,13 @@ def test_resolve_package_module_combined(self, project_dir): ) assert actual == f"{project_dir}\\syscore\\tests\\pricetestdata.csv" - @pytest.mark.xfail(reason="Cannot work with existing implementation") def test_resolve_package_module_combined_dotted_filename(self, project_dir): actual = resolve_path_and_filename_for_package( "syscore.tests.price.test.data.csv" ) assert actual == f"{project_dir}\\syscore\\tests\\price.test.data.csv" - @pytest.mark.xfail(reason="Cannot work with existing implementation") + @pytest.mark.xfail(reason="Cannot work with old or new implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( self, tmp_path ): @@ -290,7 +287,7 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( ) assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt" - @pytest.mark.xfail(reason="Cannot work with existing implementation") + @pytest.mark.xfail(reason="Cannot work with old or new implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( self, tmp_path ): @@ -303,6 +300,18 @@ def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( ) assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt" + def test_resolve_resolve_path_and_filename_for_package_with_separate_dotted_file_name( + self, tmp_path + ): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}\\dir_name\\", "dotted.filename.txt" + ) + assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt" + # No Separate filename def test_csv_data(self, project_dir): From 3fab4dce76e43bef9fd15c8c904489ca2402a4a2 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Mon, 19 Jan 2026 17:20:53 +0000 Subject: [PATCH 24/28] fix absolute --- syscore/fileutils.py | 10 +++++++--- syscore/tests/test_fileutils.py | 25 ++++++++++++++++--------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/syscore/fileutils.py b/syscore/fileutils.py index ad42f0dfaf7..23ad87be24b 100755 --- a/syscore/fileutils.py +++ b/syscore/fileutils.py @@ -2,6 +2,7 @@ import datetime import time import os +import re from pathlib import Path, PurePath from typing import List, Tuple, TextIO @@ -179,6 +180,12 @@ def get_resolved_pathname(pathname: str) -> str: def _make_absolute(path_as_list: list[str]) -> PurePath: + # handle absolute unix + if path_as_list[0] == "": + path_as_list[0] = os.sep + # handle absolute windows + if re.match("[A-Za-z]:$", path_as_list[0]): + path_as_list[0] = f"{path_as_list[0]}{os.sep}" path_obj = PurePath(*path_as_list) if not path_obj.is_absolute(): path_obj = PYSYS_PROJECT_DIR / path_obj @@ -217,9 +224,6 @@ def _transform_path_into_list(pathname: str) -> List[str]: if path_as_list[-1] == "": path_as_list.pop() - if path_as_list[0] == "": - path_as_list[0] = f"{os.sep}{path_as_list[0]}" - return path_as_list diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py index 6921c124e45..a8c7c389b9e 100644 --- a/syscore/tests/test_fileutils.py +++ b/syscore/tests/test_fileutils.py @@ -86,7 +86,7 @@ def test_resolve_package_module_combined_dotted_filename(self, project_dir): ) assert actual == f"{project_dir}/syscore/tests/price.test.data.csv" - def test_resolve_package_module_combined_dotted_filename(self, project_dir): + def test_resolve_package_module_combined_dotted_filename_sep(self, project_dir): actual = resolve_path_and_filename_for_package( "syscore.tests", "price.test.data.csv" ) @@ -214,10 +214,15 @@ def test_resolve_path_absolute(self): actual = get_resolved_pathname("C:\\home\\rob\\") assert actual == "C:\\home\\rob" + def test_resolve_path_absolute_lower_case(self): + actual = get_resolved_pathname("c:\\home\\rob\\") + assert actual == "c:\\home\\rob" + def test_resolve_path_absolute_trailing(self): actual = get_resolved_pathname("C:\\home\\rob\\") assert actual == "C:\\home\\rob" + @pytest.mark.xfail(reason="A Windows absolute path needs a root AND a drive") def test_resolve_path_absolute_dotted(self): actual = get_resolved_pathname(".home.rob") assert actual == "\\home\\rob" @@ -235,7 +240,7 @@ def test_resolve_dotted_dir_name(self, tmp_path): directory.mkdir() file = directory / "hello.txt" file.write_text("content", encoding="utf-8") - resolved_path = get_resolved_pathname(str(file)) + resolved_path = get_resolved_pathname(file) assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt" def test_resolve_dotted_file_name(self, tmp_path): @@ -243,7 +248,7 @@ def test_resolve_dotted_file_name(self, tmp_path): directory.mkdir() file = directory / "dotted.filename.txt" file.write_text("content", encoding="utf-8") - resolved_path = get_resolved_pathname(str(file)) + resolved_path = get_resolved_pathname(file) assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt" def test_resolve_package_separate(self): @@ -254,6 +259,7 @@ def test_resolve_package_combined(self): actual = resolve_path_and_filename_for_package("C:\\home\\rob\\file.csv") assert actual == "C:\\home\\rob\\file.csv" + @pytest.mark.xfail(reason="A Windows absolute path needs a root AND a drive") def test_resolve_package_combined_dotted(self): actual = resolve_path_and_filename_for_package(".home.rob.file.csv") assert actual == "\\home\\rob\\file.csv" @@ -268,12 +274,19 @@ def test_resolve_package_module_combined(self, project_dir): ) assert actual == f"{project_dir}\\syscore\\tests\\pricetestdata.csv" + @pytest.mark.xfail(reason="Cannot work with old or new implementation") def test_resolve_package_module_combined_dotted_filename(self, project_dir): actual = resolve_path_and_filename_for_package( "syscore.tests.price.test.data.csv" ) assert actual == f"{project_dir}\\syscore\\tests\\price.test.data.csv" + def test_resolve_package_module_combined_dotted_filename_sep(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests", "price.test.data.csv" + ) + assert actual == f"{project_dir}\\syscore\\tests\\price.test.data.csv" + @pytest.mark.xfail(reason="Cannot work with old or new implementation") def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( self, tmp_path @@ -390,9 +403,3 @@ def test_email_store_filename(self): "C:\\home\\rob\\logs\\email_store.log", ) assert actual == "C:\\home\\rob\\logs\\email_store.log" - - def test_convert_email_store_filename(self): - actual = resolve_path_and_filename_for_package( - "/home/rob/logs/email_store.log", - ) - assert actual == "\\home\\rob\\logs\\email_store.log" From 2933e1bad1651d061daf338691a456650c73279a Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Wed, 25 Feb 2026 13:56:15 +0000 Subject: [PATCH 25/28] add trading hours tests --- .../private_config_trading_hours.yaml | 3 +++ sysdata/tests/test_config.py | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 sysdata/tests/custom_private_config/private_config_trading_hours.yaml diff --git a/sysdata/tests/custom_private_config/private_config_trading_hours.yaml b/sysdata/tests/custom_private_config/private_config_trading_hours.yaml new file mode 100644 index 00000000000..cd381be1e36 --- /dev/null +++ b/sysdata/tests/custom_private_config/private_config_trading_hours.yaml @@ -0,0 +1,3 @@ +MET: + - '09:00' + - '14:00' diff --git a/sysdata/tests/test_config.py b/sysdata/tests/test_config.py index 728efd09a56..87c1bf01a31 100644 --- a/sysdata/tests/test_config.py +++ b/sysdata/tests/test_config.py @@ -1,6 +1,8 @@ +import datetime from sysdata.config.configdata import Config from sysdata.config.control_config import get_control_config from sysdata.config.private_config import PRIVATE_CONFIG_DIR_ENV_VAR +from sysbrokers.IB.ib_trading_hours import get_saved_trading_hours class TestConfig: @@ -77,3 +79,21 @@ def test_control_bad_custom_dir(self, monkeypatch): config.as_dict()["process_configuration_start_time"]["run_stack_handler"] == "00:01" ) + + def test_trading_hours_default(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + config = get_saved_trading_hours() + assert config["MET"]["Monday"][0].closing_time == datetime.time(15) + + def test_trading_hours_custom(self, monkeypatch): + monkeypatch.setenv( + PRIVATE_CONFIG_DIR_ENV_VAR, "sysdata.tests.custom_private_config" + ) + config = get_saved_trading_hours() + assert config["MET"]["Monday"][0].opening_time == datetime.time(9) + assert config["MET"]["Monday"][0].closing_time == datetime.time(14) + + def test_trading_hours_bad_custom_dir(self, monkeypatch): + monkeypatch.setenv(PRIVATE_CONFIG_DIR_ENV_VAR, "sysdata.tests") + config = get_saved_trading_hours() + assert config["MET"]["Monday"][0].closing_time == datetime.time(15) From 3742a3fbea9edcd16571d122aabb1d7711b15e4d Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Wed, 25 Feb 2026 16:27:23 +0000 Subject: [PATCH 26/28] ensure all config files are present in normal install --- pyproject.toml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9b5b5969659..133ba7d477c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,8 @@ [build-system] -requires = ["setuptools >= 61.0"] +requires = [ + "setuptools >= 61.0", + "setuptools-scm >= 8.0" +] build-backend = "setuptools.build_meta" [project] @@ -47,8 +50,11 @@ find = {} [tool.setuptools.package-data] "data" = ["*.csv"] +"data.futures" = ["*.csv"] "private" = ["*.yaml"] -"sysbrokers" = ["*.csv", "*.yaml"] +"sysbrokers" = ["*.csv"] +"sysbrokers.IB" = ["*.yaml"] +"sysbrokers.IB.config" = ["*.csv"] "syscontrol" = ["*.yaml"] "sysdata" = ["*.csv"] "sysdata.config" = ["*.yaml"] From 72021949eb1648706f2a8f2e95560bf83e1cecd5 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Wed, 25 Feb 2026 18:01:38 +0000 Subject: [PATCH 27/28] ensure __main__ function for all sysproduction scripts --- sysproduction/interactive_update_capital_manual.py | 3 +++ sysproduction/update_strategy_orders.py | 3 +++ sysproduction/update_system_backtests.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/sysproduction/interactive_update_capital_manual.py b/sysproduction/interactive_update_capital_manual.py index ecd08053ca8..5f53be7d74b 100644 --- a/sysproduction/interactive_update_capital_manual.py +++ b/sysproduction/interactive_update_capital_manual.py @@ -266,3 +266,6 @@ def delete_all_capital(data: dataBlob): ) else: print("OK you decided not to do it") + +if __name__ == "__main__": + interactive_update_capital_manual() diff --git a/sysproduction/update_strategy_orders.py b/sysproduction/update_strategy_orders.py index 5a53e03b1b6..f997132a811 100644 --- a/sysproduction/update_strategy_orders.py +++ b/sysproduction/update_strategy_orders.py @@ -28,3 +28,6 @@ def update_strategy_orders(): data, strategy_name, process_name, name_of_main_generator_method ) strategy_order_generator.run_strategy_method() + +if __name__ == "__main__": + update_strategy_orders() diff --git a/sysproduction/update_system_backtests.py b/sysproduction/update_system_backtests.py index 32b646e7f18..35ed6cd324e 100644 --- a/sysproduction/update_system_backtests.py +++ b/sysproduction/update_system_backtests.py @@ -27,3 +27,6 @@ def update_system_backtests(): data, strategy_name, process_name, backtest_function ) system_backtest_runner.run_strategy_method() + +if __name__ == "__main__": + update_system_backtests() From d0dff892b85e6b1c21295df9e489a183d54340a3 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Wed, 25 Feb 2026 18:04:42 +0000 Subject: [PATCH 28/28] black --- sysproduction/interactive_update_capital_manual.py | 1 + sysproduction/update_strategy_orders.py | 1 + sysproduction/update_system_backtests.py | 1 + 3 files changed, 3 insertions(+) diff --git a/sysproduction/interactive_update_capital_manual.py b/sysproduction/interactive_update_capital_manual.py index 5f53be7d74b..50553414188 100644 --- a/sysproduction/interactive_update_capital_manual.py +++ b/sysproduction/interactive_update_capital_manual.py @@ -267,5 +267,6 @@ def delete_all_capital(data: dataBlob): else: print("OK you decided not to do it") + if __name__ == "__main__": interactive_update_capital_manual() diff --git a/sysproduction/update_strategy_orders.py b/sysproduction/update_strategy_orders.py index f997132a811..302793fd626 100644 --- a/sysproduction/update_strategy_orders.py +++ b/sysproduction/update_strategy_orders.py @@ -29,5 +29,6 @@ def update_strategy_orders(): ) strategy_order_generator.run_strategy_method() + if __name__ == "__main__": update_strategy_orders() diff --git a/sysproduction/update_system_backtests.py b/sysproduction/update_system_backtests.py index 35ed6cd324e..0bd5dfe112c 100644 --- a/sysproduction/update_system_backtests.py +++ b/sysproduction/update_system_backtests.py @@ -28,5 +28,6 @@ def update_system_backtests(): ) system_backtest_runner.run_strategy_method() + if __name__ == "__main__": update_system_backtests()