diff --git a/.github/workflows/os-test.yml b/.github/workflows/os-test.yml index c3253a50285..761c0dd1e58 100644 --- a/.github/workflows/os-test.yml +++ b/.github/workflows/os-test.yml @@ -25,4 +25,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - python -m pip install . + python -m pip install '.[dev]' + + - name: Test with pytest + run: | + pytest syscore/tests/test_fileutils.py diff --git a/docs/backtesting.md b/docs/backtesting.md index d489c91cfa1..a70ce4a5635 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -3123,7 +3123,6 @@ resolve_path_and_filename_for_package("\\home/rob.file.csv") ``` - These functions are used internally whenever a file name is passed in, so feel free to use any of these file formats when specifying eg a configuration filename. ``` ### Absolute: Windows (note use of double backslash in str) @@ -3136,6 +3135,31 @@ These functions are used internally whenever a file name is passed in, so feel f "syscore.tests.pricedata.csv" ``` +Obviously, using 'dots' as a separator brings limitations, like supporting directories or files with 'dots' in their names. This won't work: +``` +>>> resolve_path_and_filename_for_package("syscore/tests/price.test.data.csv") +'/home/user/pysystemtrade/syscore/tests/price/test/data.csv' +``` + +do this instead: +``` +>>> resolve_path_and_filename_for_package("syscore/tests", "price.test.data.csv") +'/home/user/pysystemtrade/syscore/tests/price.test.data.csv' +``` + +This will also not work: +``` +>>> get_resolved_pathname("data/dir.with.dots") +'/home/user/pysystemtrade/data/dir/with/dots' +``` + +do this instead: +``` +>>> from pathlib import Path +>>> get_resolved_pathname(Path("data/dir.with.dots")) +'/home/user/pysystemtrade/data/dir.with.dots' +``` + ## Logging ### Basic logging diff --git a/pyproject.toml b/pyproject.toml index 9b5b5969659..133ba7d477c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,8 @@ [build-system] -requires = ["setuptools >= 61.0"] +requires = [ + "setuptools >= 61.0", + "setuptools-scm >= 8.0" +] build-backend = "setuptools.build_meta" [project] @@ -47,8 +50,11 @@ find = {} [tool.setuptools.package-data] "data" = ["*.csv"] +"data.futures" = ["*.csv"] "private" = ["*.yaml"] -"sysbrokers" = ["*.csv", "*.yaml"] +"sysbrokers" = ["*.csv"] +"sysbrokers.IB" = ["*.yaml"] +"sysbrokers.IB.config" = ["*.csv"] "syscontrol" = ["*.yaml"] "sysdata" = ["*.csv"] "sysdata.config" = ["*.yaml"] diff --git a/sysbrokers/IB/ib_trading_hours.py b/sysbrokers/IB/ib_trading_hours.py index 256ea40c9c0..77c5b40b3f9 100644 --- a/sysbrokers/IB/ib_trading_hours.py +++ b/sysbrokers/IB/ib_trading_hours.py @@ -1,26 +1,35 @@ import datetime from ib_insync import ContractDetails as ibContractDetails -from sysdata.config.private_directory import get_full_path_for_private_config +from sysdata.config.private_config import ( + get_private_config_dir, +) from sysobjects.production.trading_hours.trading_hours import ( tradingHours, listOfTradingHours, ) -from syscore.fileutils import does_filename_exist +from syscore.fileutils import ( + does_resolved_filename_exist, + resolve_path_and_filename_for_package, +) from sysdata.config.production_config import get_production_config from sysdata.production.trading_hours import read_trading_hours IB_CONFIG_TRADING_HOURS_FILE = "sysbrokers.IB.ib_config_trading_hours.yaml" -PRIVATE_CONFIG_TRADING_HOURS_FILE = get_full_path_for_private_config( - "private_config_trading_hours.yaml" -) +PRIVATE_CONFIG_TRADING_HOURS_FILE = "private_config_trading_hours.yaml" def get_saved_trading_hours(): - if does_filename_exist(PRIVATE_CONFIG_TRADING_HOURS_FILE): - return read_trading_hours(PRIVATE_CONFIG_TRADING_HOURS_FILE) + private_path = resolve_path_and_filename_for_package( + get_private_config_dir(), PRIVATE_CONFIG_TRADING_HOURS_FILE + ) + if does_resolved_filename_exist(private_path): + return read_trading_hours(private_path) else: - return read_trading_hours(IB_CONFIG_TRADING_HOURS_FILE) + default_path = resolve_path_and_filename_for_package( + IB_CONFIG_TRADING_HOURS_FILE + ) + return read_trading_hours(default_path) def get_trading_hours_from_contract_details( diff --git a/syscore/__init__.py b/syscore/__init__.py index e69de29bb2d..61008830ee9 100755 --- a/syscore/__init__.py +++ b/syscore/__init__.py @@ -0,0 +1,7 @@ +from importlib.util import find_spec +from pathlib import Path + +module_spec = find_spec(__name__) +path = Path(module_spec.origin) + +PYSYS_PROJECT_DIR = path.parent.parent diff --git a/syscore/fileutils.py b/syscore/fileutils.py index d5bf79c658b..23ad87be24b 100755 --- a/syscore/fileutils.py +++ b/syscore/fileutils.py @@ -1,21 +1,14 @@ import glob import datetime import time -from importlib import import_module import os -from pathlib import Path -from typing import List, Tuple +import re +from pathlib import Path, PurePath +from typing import List, Tuple, TextIO +from syscore import PYSYS_PROJECT_DIR from syscore.dateutils import SECONDS_PER_DAY -# DO NOT DELETE: all these are unused: but are required to get the filename padding to work - - -""" - - FILES IN DIRECTORIES - -""" """ @@ -26,7 +19,7 @@ def rename_files_with_extension_in_pathname_as_archive_files( pathname: str, extension: str = ".txt", archive_extension: str = ".arch" -): +) -> None: """ Find all the files with a particular extension in a directory, and rename them eg thing.txt will become thing_yyyymmdd.txt where yyyymmdd is todays date @@ -47,7 +40,7 @@ def rename_files_with_extension_in_pathname_as_archive_files( def rename_file_as_archive( full_filename: str, old_extension: str = ".txt", archive_extension: str = ".arch" -): +) -> None: """ Rename a file with archive suffix and extension eg thing.txt will become thing_yyyymmdd.arch where yyyymmdd is todays date @@ -62,8 +55,8 @@ def rename_file_as_archive( def delete_old_files_with_extension_in_pathname( - pathname: str, days_old=30, extension=".arch" -): + pathname: str, days_old: int = 30, extension: str = ".arch" +) -> None: """ Find all the files with a particular extension in a directory, and delete them if older than x days @@ -77,7 +70,7 @@ def delete_old_files_with_extension_in_pathname( delete_file_if_too_old(filename, days_old=days_old) -def delete_file_if_too_old(full_filename_with_ext: str, days_old: int = 30): +def delete_file_if_too_old(full_filename_with_ext: str, days_old: int = 30) -> None: file_age = get_file_or_folder_age_in_days(full_filename_with_ext) if file_age > days_old: print("Deleting %s" % full_filename_with_ext) @@ -103,7 +96,7 @@ def get_file_or_folder_age_in_days(full_filename_with_ext: str) -> float: def resolve_path_and_filename_for_package( - path_and_filename: str, separate_filename=None + path_and_filename: str, separate_filename: str | None = None ) -> str: """ A way of resolving relative and absolute filenames, and dealing with awkward OS specific things @@ -134,22 +127,21 @@ def resolve_path_and_filename_for_package( """ - path_and_filename_as_list = transform_path_into_list(path_and_filename) + path_and_filename_as_list = _transform_path_into_list(path_and_filename) if separate_filename is None: ( path_as_list, separate_filename, - ) = extract_filename_from_combined_path_and_filename_list( + ) = _extract_filename_from_combined_path_and_filename_list( path_and_filename_as_list ) else: path_as_list = path_and_filename_as_list - resolved_pathname = get_pathname_from_list(path_as_list) - - resolved_path_and_filename = os.path.join(resolved_pathname, separate_filename) + absolute_path = _make_absolute(path_as_list) + result = absolute_path / separate_filename - return resolved_path_and_filename + return str(result) def get_resolved_pathname(pathname: str) -> str: @@ -168,51 +160,65 @@ def get_resolved_pathname(pathname: str) -> str: """ - if isinstance(pathname, Path): + if isinstance(pathname, Path) and pathname.exists(): # special case when already a Path - pathname = str(pathname.absolute()) - - if "@" in pathname or "::" in pathname: - # This is an ssh address for rsync - don't change - return pathname + return str(pathname.absolute()) + else: + pathname = str(pathname) + if "@" in pathname or "::" in pathname: + # This is an ssh address for rsync - don't change + return pathname - # Turn /,\ into . so system independent - path_as_list = transform_path_into_list(pathname) - resolved_pathname = get_pathname_from_list(path_as_list) + path_as_list = _transform_path_into_list(pathname) + result = _make_absolute(path_as_list) - return resolved_pathname + return str(result) ## something unlikely to occur naturally in a pathname RESERVED_CHARACTERS = "&!*" -def transform_path_into_list(pathname: str) -> List[str]: +def _make_absolute(path_as_list: list[str]) -> PurePath: + # handle absolute unix + if path_as_list[0] == "": + path_as_list[0] = os.sep + # handle absolute windows + if re.match("[A-Za-z]:$", path_as_list[0]): + path_as_list[0] = f"{path_as_list[0]}{os.sep}" + path_obj = PurePath(*path_as_list) + if not path_obj.is_absolute(): + path_obj = PYSYS_PROJECT_DIR / path_obj + + return path_obj + + +def _transform_path_into_list(pathname: str) -> List[str]: """ - >>> path_as_list("/home/rob/test.csv") + >>> _transform_path_into_list("/home/rob/test.csv") ['', 'home', 'rob', 'test', 'csv'] - >>> path_as_list("/home/rob/") + >>> _transform_path_into_list("/home/rob/") ['', 'home', 'rob'] - >>> path_as_list(".home.rob") + >>> _transform_path_into_list(".home.rob") ['', 'home', 'rob'] - >>> path_as_list('C:\\home\\rob\\'') + >>> _transform_path_into_list('C:\\home\\rob\\'') ['C:', 'home', 'rob'] - >>> path_as_list('C:\\home\\rob\\test.csv') + >>> _transform_path_into_list('C:\\home\\rob\\test.csv') ['C:', 'home', 'rob', 'test', 'csv'] - >>> path_as_list("syscore.tests.fileutils.csv") + >>> _transform_path_into_list("syscore.tests.fileutils.csv") ['syscore', 'tests', 'fileutils', 'csv'] - >>> path_as_list("syscore.tests") + >>> _transform_path_into_list("syscore.tests") ['syscore', 'tests'] """ - pathname_replace = add_reserved_characters_to_pathname(pathname) + pathname_replace = _add_reserved_characters_to_pathname(pathname) path_as_list = pathname_replace.rsplit(RESERVED_CHARACTERS) if path_as_list[-1] == "": @@ -221,7 +227,7 @@ def transform_path_into_list(pathname: str) -> List[str]: return path_as_list -def add_reserved_characters_to_pathname(pathname: str) -> str: +def _add_reserved_characters_to_pathname(pathname: str) -> str: pathname_replace = pathname.replace(".", RESERVED_CHARACTERS) pathname_replace = pathname_replace.replace("/", RESERVED_CHARACTERS) pathname_replace = pathname_replace.replace("\\", RESERVED_CHARACTERS) @@ -229,11 +235,11 @@ def add_reserved_characters_to_pathname(pathname: str) -> str: return pathname_replace -def extract_filename_from_combined_path_and_filename_list( - path_and_filename_as_list: list, -) -> Tuple[list, str]: +def _extract_filename_from_combined_path_and_filename_list( + path_and_filename_as_list: list[str], +) -> Tuple[list[str], str]: """ - >>> extract_filename_from_combined_path_and_filename_list(['home', 'rob','file', 'csv']) + >>> _extract_filename_from_combined_path_and_filename_list(['home', 'rob','file', 'csv']) (['home', 'rob'], 'file.csv') """ ## need -2 because want extension @@ -245,91 +251,6 @@ def extract_filename_from_combined_path_and_filename_list( return path_and_filename_as_list, separate_filename -def get_pathname_from_list(path_as_list: List[str]) -> str: - """ - >>> get_pathname_from_list(['C:', 'home', 'rob']) - 'C:\\home\\rob' - >>> get_pathname_from_list(['','home','rob']) - '/home/rob' - >>> get_pathname_from_list(['syscore','tests']) - '/home/rob/pysystemtrade/syscore/tests' - """ - if path_as_list[0] == "": - # path_type_absolute - resolved_pathname = get_absolute_linux_pathname_from_list(path_as_list[1:]) - elif is_windoze_path_list(path_as_list): - # windoze - resolved_pathname = get_absolute_windows_pathname_from_list(path_as_list) - else: - # relative - resolved_pathname = get_relative_pathname_from_list(path_as_list) - - return resolved_pathname - - -def is_windoze_path_list(path_as_list: List[str]) -> bool: - """ - >>> is_windoze_path_list(['C:']) - True - >>> is_windoze_path_list(['wibble']) - False - """ - return path_as_list[0].endswith(":") - - -def get_relative_pathname_from_list(path_as_list: List[str]) -> str: - """ - - >>> get_relative_pathname_from_list(['syscore','tests']) - '/home/rob/pysystemtrade/syscore/tests' - """ - package_name = path_as_list[0] - paths_or_files = path_as_list[1:] - - if len(paths_or_files) == 0: - directory_name_of_package = os.path.dirname( - import_module(package_name).__file__ - ) - return directory_name_of_package - - last_item_in_list = path_as_list.pop() - pathname = os.path.join( - get_relative_pathname_from_list(path_as_list), last_item_in_list - ) - - return pathname - - -def get_absolute_linux_pathname_from_list(path_as_list: List[str]) -> str: - """ - Returns the absolute pathname from a list - - >>> get_absolute_linux_pathname_from_list(['home', 'rob']) - '/home/rob' - """ - pathname = os.path.join(*path_as_list) - pathname = os.path.sep + pathname - - return pathname - - -def get_absolute_windows_pathname_from_list(path_as_list: list) -> str: - """ - Test will fail on linux - >>> get_absolute_windows_pathname_from_list(['C:','home','rob']) - 'C:\\home\\rob' - """ - drive_part_of_path = path_as_list[0] - if drive_part_of_path.endswith(":"): - ## add back backslash - drive_part_of_path = drive_part_of_path.replace(":", ":\\") - path_as_list[0] = drive_part_of_path - - pathname = os.path.join(*path_as_list) - - return pathname - - """ HTML @@ -337,7 +258,9 @@ def get_absolute_windows_pathname_from_list(path_as_list: list) -> str: """ -def write_list_of_lists_as_html_table_in_file(file, list_of_lists: list): +def write_list_of_lists_as_html_table_in_file( + file: TextIO, list_of_lists: list[str] +) -> None: file.write("") for sublist in list_of_lists: file.write("
") @@ -347,7 +270,9 @@ def write_list_of_lists_as_html_table_in_file(file, list_of_lists: list): file.write("
") -def files_with_extension_in_pathname(pathname: str, extension=".csv") -> List[str]: +def files_with_extension_in_pathname( + pathname: str, extension: str = ".csv" +) -> List[str]: """ Find all the files with a particular extension in a directory @@ -360,7 +285,7 @@ def files_with_extension_in_pathname(pathname: str, extension=".csv") -> List[st def files_with_extension_in_resolved_pathname( - resolved_pathname: str, extension=".csv" + resolved_pathname: str, extension: str = ".csv" ) -> List[str]: """ Find all the files with a particular extension in a directory @@ -375,9 +300,7 @@ def files_with_extension_in_resolved_pathname( def full_filename_for_file_in_home_dir(filename: str) -> str: - pathname = os.path.expanduser("~") - - return os.path.join(pathname, filename) + return str(Path.home() / filename) def does_filename_exist(filename: str) -> bool: @@ -387,5 +310,4 @@ def does_filename_exist(filename: str) -> bool: def does_resolved_filename_exist(resolved_filename: str) -> bool: - file_exists = os.path.isfile(resolved_filename) - return file_exists + return Path(resolved_filename).exists() diff --git a/syscore/tests/price.test.data.csv b/syscore/tests/price.test.data.csv new file mode 100644 index 00000000000..b4c9c312a5b --- /dev/null +++ b/syscore/tests/price.test.data.csv @@ -0,0 +1,529 @@ +DATETIME,ADJ +2013-04-15,139.0825 +2013-04-16, +2013-04-17, +2013-04-18, +2013-04-19, +2013-04-22, +2013-04-23, +2013-04-24, +2013-04-25, +2013-04-26, +2013-04-29, +2013-04-30, +2013-05-01, +2013-05-02, +2013-05-03,138.7825 +2013-05-06,137.0525 +2013-05-07, +2013-05-08, +2013-05-09, +2013-05-10, +2013-05-13, +2013-05-14, +2013-05-15, +2013-05-16, +2013-05-17, +2013-05-20, +2013-05-21, +2013-05-22, +2013-05-23, +2013-05-24, +2013-05-27, +2013-05-28,134.4125 +2013-05-29, +2013-05-30, +2013-05-31, +2013-06-03,134.9825 +2013-06-04,134.8025 +2013-06-05,135.1225 +2013-06-06,135.2825 +2013-06-07,134.9225 +2013-06-10,134.1625 +2013-06-11,133.9425 +2013-06-12,134.1025 +2013-06-13,134.4325 +2013-06-14,135.1625 +2013-06-17,135.0225 +2013-06-18,134.7025 +2013-06-19,133.8925 +2013-06-20,133.5725 +2013-06-21,132.5025 +2013-06-24,132.1325 +2013-06-25,131.8025 +2013-06-26,132.4325 +2013-06-27,132.9925 +2013-06-28,132.9025 +2013-07-01,132.9625 +2013-07-02,133.5125 +2013-07-03,133.5825 +2013-07-04,133.7125 +2013-07-05,132.9225 +2013-07-08,133.3625 +2013-07-09,134.0725 +2013-07-10,134.0025 +2013-07-11,134.1825 +2013-07-12,135.1125 +2013-07-15,134.7825 +2013-07-16,135.1225 +2013-07-17,135.2825 +2013-07-18,135.4425 +2013-07-19,135.4825 +2013-07-22,135.5325 +2013-07-23,135.1025 +2013-07-24,133.6825 +2013-07-25,133.6925 +2013-07-26,133.8425 +2013-07-29,133.7025 +2013-07-30,133.7125 +2013-07-31,134.2825 +2013-08-01,133.6025 +2013-08-02,134.1125 +2013-08-05,133.5625 +2013-08-06,133.3825 +2013-08-07,133.5225 +2013-08-08,133.6625 +2013-08-09,133.7125 +2013-08-12,133.3725 +2013-08-13,132.1325 +2013-08-14,132.1725 +2013-08-15,131.5325 +2013-08-16,131.0825 +2013-08-19,131.2825 +2013-08-20,131.9625 +2013-08-21,131.3125 +2013-08-22,131.0525 +2013-08-23,130.8825 +2013-08-26,131.3525 +2013-08-27,132.2625 +2013-08-28,131.5825 +2013-08-29,132.0625 +2013-08-30,131.7525 +2013-09-02,131.3225 +2013-09-03,131.1525 +2013-09-04,130.8325 +2013-09-05,129.7325 +2013-09-06,130.8125 +2013-09-09,130.7525 +2013-09-10,129.9325 +2013-09-11,130.6125 +2013-09-12,130.8725 +2013-09-13,131.2225 +2013-09-16,131.4425 +2013-09-17,131.3425 +2013-09-18,132.3225 +2013-09-19,131.7725 +2013-09-20,131.7925 +2013-09-23,132.0725 +2013-09-24,133.0025 +2013-09-25,133.2425 +2013-09-26,133.1025 +2013-09-27,133.6425 +2013-09-30,133.7825 +2013-10-01,133.5425 +2013-10-02,133.6025 +2013-10-03,133.5125 +2013-10-04,133.2425 +2013-10-07,133.5925 +2013-10-08,133.5025 +2013-10-09,133.5225 +2013-10-10,132.9325 +2013-10-11,133.0725 +2013-10-14,133.0325 +2013-10-15,132.5225 +2013-10-16,132.1925 +2013-10-17,132.9625 +2013-10-18,133.3325 +2013-10-21,133.1925 +2013-10-22,133.8225 +2013-10-23,134.1425 +2013-10-24,134.1525 +2013-10-25,134.3425 +2013-10-28,134.4325 +2013-10-29,134.5525 +2013-10-30,134.7325 +2013-10-31,135.2825 +2013-11-01,135.1325 +2013-11-04,135.1725 +2013-11-05,134.4425 +2013-11-06,134.4325 +2013-11-07,135.1125 +2013-11-08,134.3025 +2013-11-11,134.2925 +2013-11-12,133.9825 +2013-11-13,134.6125 +2013-11-14,134.9825 +2013-11-15,134.9125 +2013-11-18,135.1325 +2013-11-19,134.6925 +2013-11-20,134.7225 +2013-11-21,134.2625 +2013-11-22,134.2325 +2013-11-25,134.6025 +2013-11-26,134.9825 +2013-11-27,134.9125 +2013-11-28,134.9325 +2013-11-29,134.9925 +2013-12-02,134.4025 +2013-12-03,134.5625 +2013-12-04,133.6625 +2013-12-05,133.2425 +2013-12-06,133.5575 +2013-12-09,133.5375 +2013-12-10,133.6175 +2013-12-11,133.7475 +2013-12-12,133.5475 +2013-12-13,133.6975 +2013-12-16,133.6975 +2013-12-17,133.7375 +2013-12-18,133.5875 +2013-12-19,133.2475 +2013-12-20,133.3675 +2013-12-23,133.1975 +2013-12-24, +2013-12-25, +2013-12-26, +2013-12-27,132.4075 +2013-12-30,132.6175 +2013-12-31, +2014-01-01, +2014-01-02,132.5675 +2014-01-03,132.5375 +2014-01-06,132.9875 +2014-01-07,133.2075 +2014-01-08,132.9875 +2014-01-09,133.0075 +2014-01-10,133.6975 +2014-01-13,134.0975 +2014-01-14,134.1275 +2014-01-15,133.9875 +2014-01-16,134.6075 +2014-01-17,134.8475 +2014-01-20,135.0175 +2014-01-21,135.0975 +2014-01-22,134.9575 +2014-01-23,135.4875 +2014-01-24,136.1075 +2014-01-27,135.9975 +2014-01-28,135.9075 +2014-01-29,136.3475 +2014-01-30,136.6775 +2014-01-31,137.3275 +2014-02-03,137.4675 +2014-02-04,137.4275 +2014-02-05,137.5175 +2014-02-06,136.7775 +2014-02-07,137.2775 +2014-02-10,137.0975 +2014-02-11,137.0875 +2014-02-12,136.6175 +2014-02-13,137.2275 +2014-02-14,137.0475 +2014-02-17,137.0475 +2014-02-18,137.1675 +2014-02-19,137.3575 +2014-02-20,136.8975 +2014-02-21,137.3175 +2014-02-24,137.1275 +2014-02-25,137.4175 +2014-02-26,137.9075 +2014-02-27,138.5175 +2014-02-28,137.8275 +2014-03-03,138.5875 +2014-03-04,138.2 +2014-03-05,138.14 +2014-03-06,137.55 +2014-03-07,137.615 +2014-03-10,137.78 +2014-03-11,137.77 +2014-03-12,138.33 +2014-03-13,139.09 +2014-03-14,138.8 +2014-03-17,138.55 +2014-03-18,138.6275 +2014-03-19,137.86 +2014-03-20,137.76 +2014-03-21,137.93 +2014-03-24, +2014-03-25,138.57 +2014-03-26,138.9 +2014-03-27,139.09 +2014-03-28,139.01 +2014-03-31,138.84 +2014-04-01,138.65 +2014-04-02,138.18 +2014-04-03,138.3 +2014-04-04, +2014-04-07,139.01 +2014-04-08,138.8525 +2014-04-09, +2014-04-10,139.31 +2014-04-11,139.63 +2014-04-14,139.27 +2014-04-15,139.88 +2014-04-16,139.77 +2014-04-17,139.08 +2014-04-18, +2014-04-21, +2014-04-22,139.23 +2014-04-23,139.24 +2014-04-24,139.31 +2014-04-25,139.74 +2014-04-28,139.64 +2014-04-29,139.63 +2014-04-30,140.03 +2014-05-01, +2014-05-02,140.18 +2014-05-05,140.02 +2014-05-06,140.03 +2014-05-07,139.9 +2014-05-08,140.25 +2014-05-09,140.16 +2014-05-12,140.07 +2014-05-13,140.64 +2014-05-14,141.24 +2014-05-15,141.99 +2014-05-16,141.65 +2014-05-19,141.49 +2014-05-20,141.59 +2014-05-21,141.28 +2014-05-22,141.4 +2014-05-23,141.39 +2014-05-26,141.38 +2014-05-27,141.755 +2014-05-28,142.39 +2014-05-29,142.06 +2014-05-30,142.17 +2014-06-02,142.02 +2014-06-03,141.32 +2014-06-04,141.215 +2014-06-05,141.66 +2014-06-06,142.18 +2014-06-09,141.95 +2014-06-10,141.5 +2014-06-11,141.52 +2014-06-12,141.795 +2014-06-13,141.97 +2014-06-16,142.22 +2014-06-17,142.19 +2014-06-18,142.18 +2014-06-19,142.48 +2014-06-20,142.34 +2014-06-23,142.54 +2014-06-24,142.78 +2014-06-25,143.3 +2014-06-26,143.59 +2014-06-27,143.38 +2014-06-30,143.6 +2014-07-01,143.51 +2014-07-02,143.07 +2014-07-03,142.895 +2014-07-04,143.4 +2014-07-07,143.45 +2014-07-08,143.96 +2014-07-09,144.05 +2014-07-10,144.11 +2014-07-11,144.04 +2014-07-14,144.11 +2014-07-15,144.11 +2014-07-16,144.35 +2014-07-17,144.96 +2014-07-18,144.73 +2014-07-21,144.79 +2014-07-22,144.68 +2014-07-23,144.81 +2014-07-24,144.44 +2014-07-25,144.91 +2014-07-28,144.79 +2014-07-29,145.22 +2014-07-30,144.46 +2014-07-31,144.675 +2014-08-01, +2014-08-04, +2014-08-05, +2014-08-06,145.43 +2014-08-07,145.555 +2014-08-08,146.015 +2014-08-11,145.88 +2014-08-12,145.81 +2014-08-13,146.34 +2014-08-14,146.41 +2014-08-15,146.87 +2014-08-18,146.38 +2014-08-19,146.62 +2014-08-20,146.75 +2014-08-21,146.76 +2014-08-22,146.96 +2014-08-25,147.39 +2014-08-26,147.45 +2014-08-27,147.86 +2014-08-28,148.2 +2014-08-29,148.02 +2014-09-01,148.035 +2014-09-02,147.51 +2014-09-03,147.42 +2014-09-04,147.26 +2014-09-05,147.54 +2014-09-08,147.22 +2014-09-09,146.87 +2014-09-10,146.82 +2014-09-11,146.85 +2014-09-12,146.31 +2014-09-15,146.71 +2014-09-16,146.64 +2014-09-17,146.74 +2014-09-18,146.49 +2014-09-19,147.14 +2014-09-22,147.42 +2014-09-23,147.5 +2014-09-24,147.38 +2014-09-25,147.9 +2014-09-26,147.9 +2014-09-29,148.02 +2014-09-30,148.2 +2014-10-01,148.87 +2014-10-02,148.51 +2014-10-03,148.44 +2014-10-06,148.61 +2014-10-07,148.81 +2014-10-08,148.72 +2014-10-09,148.69 +2014-10-10,149.01 +2014-10-13,148.94 +2014-10-14,149.59 +2014-10-15,150.06 +2014-10-16,150.21 +2014-10-17,149.17 +2014-10-20,149.31 +2014-10-21,148.95 +2014-10-22,149.19 +2014-10-23,148.67 +2014-10-24,148.83 +2014-10-27,149.03 +2014-10-28,148.98 +2014-10-29,148.75 +2014-10-30,149.37 +2014-10-31,149.55 +2014-11-03,149.41 +2014-11-04,149.75 +2014-11-05,149.69 +2014-11-06,149.61 +2014-11-07,149.87 +2014-11-10,149.59 +2014-11-11,149.78 +2014-11-12,149.82 +2014-11-13,150.2 +2014-11-14,150.4 +2014-11-17,150.11 +2014-11-18,150.08 +2014-11-19,149.56 +2014-11-20,150.12 +2014-11-21,150.51 +2014-11-24,150.52 +2014-11-25,150.86 +2014-11-26,151.03 +2014-11-27,151.42 +2014-11-28,151.43 +2014-12-01,150.68 +2014-12-02,150.76 +2014-12-03,150.84 +2014-12-04,150.77 +2014-12-05,150.61 +2014-12-08,151.45 +2014-12-09,151.66 +2014-12-10,151.95 +2014-12-11,151.92 +2014-12-12,152.47 +2014-12-15,152.35 +2014-12-16,152.9 +2014-12-17,152.8 +2014-12-18,152.64 +2014-12-19,152.96 +2014-12-22,152.93 +2014-12-23,152.78 +2014-12-24, +2014-12-25, +2014-12-26, +2014-12-29,153.54 +2014-12-30,153.62 +2014-12-31, +2015-01-01, +2015-01-02,154.29 +2015-01-05,154.09 +2015-01-06,154.72 +2015-01-07,154.48 +2015-01-08,154.12 +2015-01-09,154.22 +2015-01-12,154.58 +2015-01-13,154.61 +2015-01-14,155.01 +2015-01-15,155.43 +2015-01-16,155.28 +2015-01-19,155.54 +2015-01-20,155.4 +2015-01-21,154.48 +2015-01-22,155.48 +2015-01-23,156.62 +2015-01-26,156.04 +2015-01-27,156 +2015-01-28,156.98 +2015-01-29,156.33 +2015-01-30,157.26 +2015-02-02,157.16 +2015-02-03,156.49 +2015-02-04,156.53 +2015-02-05,156.39 +2015-02-06,156.22 +2015-02-09,156.49 +2015-02-10,156.29 +2015-02-11,156.54 +2015-02-12,156.94 +2015-02-13,156.62 +2015-02-16,157.09 +2015-02-17,156.24 +2015-02-18,156.48 +2015-02-19,156.29 +2015-02-20,156.13 +2015-02-23,156.72 +2015-02-24,156.72 +2015-02-25,157.35 +2015-02-26,157.49 +2015-02-27,157.355 +2015-03-02,156.82 +2015-03-03,156.54 +2015-03-04,156.51 +2015-03-05,157.01 +2015-03-06,156.25 +2015-03-09,157.34 +2015-03-10,158.47 +2015-03-11,158.56 +2015-03-12,158 +2015-03-13,157.89 +2015-03-16,157.71 +2015-03-17,157.59 +2015-03-18,159.07 +2015-03-19,158.77 +2015-03-20,158.87 +2015-03-23,158.39 +2015-03-24,158.34 +2015-03-25,158.16 +2015-03-26,158.37 +2015-03-27,158.52 +2015-03-30,158.47 +2015-03-31,158.79 +2015-04-01,158.9 +2015-04-02,158.71 +2015-04-03, +2015-04-06, +2015-04-07,158.89 +2015-04-08,159.12 +2015-04-09,159.01 +2015-04-10,159.16 +2015-04-13,159.31 +2015-04-14,159.43 +2015-04-15,159.98 +2015-04-16,160.31 +2015-04-17,160.45 +2015-04-20,160.31 +2015-04-21,159.9 +2015-04-22,159.225 diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py new file mode 100644 index 00000000000..a8c7c389b9e --- /dev/null +++ b/syscore/tests/test_fileutils.py @@ -0,0 +1,405 @@ +import pytest +import sys +from pathlib import Path +from syscore.fileutils import ( + resolve_path_and_filename_for_package, + get_resolved_pathname, +) + + +@pytest.fixture() +def project_dir(request): + module_path = Path(request.module.__file__) + return str(module_path.parent.parent.parent.absolute()) + + +@pytest.mark.skipif(sys.platform.startswith("win"), reason="Only runs on unix") +class TestFileUtilsUnix: + def test_resolve_path_absolute(self): + actual = get_resolved_pathname("/home/rob") + assert actual == "/home/rob" + + def test_resolve_path_absolute_trailing(self): + actual = get_resolved_pathname("/home/rob/") + assert actual == "/home/rob" + + def test_resolve_path_absolute_dotted(self): + actual = get_resolved_pathname(".home.rob") + assert actual == "/home/rob" + + def test_resolve_path_relative(self, project_dir): + actual = get_resolved_pathname("syscore.tests") + assert actual == f"{project_dir}/syscore/tests" + + def test_resolve_path_non_existent(self, project_dir): + actual = get_resolved_pathname("syscore.testz") + assert actual == f"{project_dir}/syscore/testz" + + def test_resolve_dotted_dir_name(self, tmp_path): + directory = tmp_path / "dir.name.with.dots" + directory.mkdir() + file = directory / "hello.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(file) + assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" + + def test_resolve_dotted_file_name(self, tmp_path): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(file) + assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + + def test_resolve_package_separate(self): + actual = resolve_path_and_filename_for_package("/home/rob/", "file.csv") + assert actual == "/home/rob/file.csv" + + def test_resolve_package_combined(self): + actual = resolve_path_and_filename_for_package("/home/rob/file.csv") + assert actual == "/home/rob/file.csv" + + def test_resolve_package_combined_dotted(self): + actual = resolve_path_and_filename_for_package(".home.rob.file.csv") + assert actual == "/home/rob/file.csv" + + def test_resolve_package_module_separate(self, project_dir): + actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv") + assert actual == f"{project_dir}/syscore/tests/file.csv" + + def test_resolve_package_module_instr_data_module(self, project_dir): + actual = resolve_path_and_filename_for_package( + "data.futures.csvconfig", "instrumentconfig.csv" + ) + assert actual == f"{project_dir}/data/futures/csvconfig/instrumentconfig.csv" + + def test_resolve_package_module_combined(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests.pricetestdata.csv" + ) + assert actual == f"{project_dir}/syscore/tests/pricetestdata.csv" + + @pytest.mark.xfail(reason="Cannot work with old or new implementation") + def test_resolve_package_module_combined_dotted_filename(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests.price.test.data.csv" + ) + assert actual == f"{project_dir}/syscore/tests/price.test.data.csv" + + def test_resolve_package_module_combined_dotted_filename_sep(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests", "price.test.data.csv" + ) + assert actual == f"{project_dir}/syscore/tests/price.test.data.csv" + + @pytest.mark.xfail(reason="Cannot work with old or new implementation") + def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( + self, tmp_path + ): + directory = tmp_path / "dir.name.with.dots" + directory.mkdir() + file = directory / "hello.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}/dir.name.with.dots", "hello.txt" + ) + assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt" + + @pytest.mark.xfail(reason="Cannot work with old or new implementation") + def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( + self, tmp_path + ): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}/dir_name/dotted.filename.txt" + ) + assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + + def test_resolve_resolve_path_and_filename_for_package_with_separate_dotted_file_name( + self, tmp_path + ): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}/dir_name/", "dotted.filename.txt" + ) + assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt" + + # No Separate filename + + def test_csv_data(self, project_dir): + actual = resolve_path_and_filename_for_package( + "data.futures.csvconfig.instrumentconfig.csv" + ) + assert actual == f"{project_dir}/data/futures/csvconfig/instrumentconfig.csv" + + def test_logging(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syslogging.logging_prod.yaml", + ) + assert actual == f"{project_dir}/syslogging/logging_prod.yaml" + + def test_config_defaults(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysdata.config.defaults.yaml", + ) + assert actual == f"{project_dir}/sysdata/config/defaults.yaml" + + def test_control_config_defaults(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscontrol.control_config.yaml", + ) + assert actual == f"{project_dir}/syscontrol/control_config.yaml" + + def test_private_config_dots(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private.private_config.yaml", + ) + assert actual == f"{project_dir}/private/private_config.yaml" + + def test_private_config_slash(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private/private_config.yaml", + ) + assert actual == f"{project_dir}/private/private_config.yaml" + + def test_strategy_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "systems.provided.rob_system.config.yaml", + ) + assert actual == f"{project_dir}/systems/provided/rob_system/config.yaml" + + def test_pickled_backtest(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private/backtests/fut_strategy_v1_8/20260101_210632_backtest.pck", + ) + assert ( + actual + == f"{project_dir}/private/backtests/fut_strategy_v1_8/20260101_210632_backtest.pck" + ) + + def test_ib_instrument_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.config.ib_config_futures.csv", + ) + assert actual == f"{project_dir}/sysbrokers/IB/config/ib_config_futures.csv" + + def test_ib_fx_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.config.ib_config_spot_FX.csv", + ) + assert actual == f"{project_dir}/sysbrokers/IB/config/ib_config_spot_FX.csv" + + def test_ib_trading_hours_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.ib_config_trading_hours.yaml", + ) + assert actual == f"{project_dir}/sysbrokers/IB/ib_config_trading_hours.yaml" + + def test_email_store_filename(self): + actual = resolve_path_and_filename_for_package( + "/home/rob/logs/email_store.log", + ) + assert actual == "/home/rob/logs/email_store.log" + + +@pytest.mark.skipif(sys.platform in ["linux", "darwin"], reason="Only runs on windows") +class TestFileUtilsWindoze: + def test_resolve_path_absolute(self): + actual = get_resolved_pathname("C:\\home\\rob\\") + assert actual == "C:\\home\\rob" + + def test_resolve_path_absolute_lower_case(self): + actual = get_resolved_pathname("c:\\home\\rob\\") + assert actual == "c:\\home\\rob" + + def test_resolve_path_absolute_trailing(self): + actual = get_resolved_pathname("C:\\home\\rob\\") + assert actual == "C:\\home\\rob" + + @pytest.mark.xfail(reason="A Windows absolute path needs a root AND a drive") + def test_resolve_path_absolute_dotted(self): + actual = get_resolved_pathname(".home.rob") + assert actual == "\\home\\rob" + + def test_resolve_path_relative(self, project_dir): + actual = get_resolved_pathname("syscore.tests") + assert actual == f"{project_dir}\\syscore\\tests" + + def test_resolve_path_non_existent(self, project_dir): + actual = get_resolved_pathname("syscore.testz") + assert actual == f"{project_dir}\\syscore\\testz" + + def test_resolve_dotted_dir_name(self, tmp_path): + directory = tmp_path / "dir.name.with.dots" + directory.mkdir() + file = directory / "hello.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(file) + assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt" + + def test_resolve_dotted_file_name(self, tmp_path): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = get_resolved_pathname(file) + assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt" + + def test_resolve_package_separate(self): + actual = resolve_path_and_filename_for_package("C:\\home\\rob\\", "file.csv") + assert actual == "C:\\home\\rob\\file.csv" + + def test_resolve_package_combined(self): + actual = resolve_path_and_filename_for_package("C:\\home\\rob\\file.csv") + assert actual == "C:\\home\\rob\\file.csv" + + @pytest.mark.xfail(reason="A Windows absolute path needs a root AND a drive") + def test_resolve_package_combined_dotted(self): + actual = resolve_path_and_filename_for_package(".home.rob.file.csv") + assert actual == "\\home\\rob\\file.csv" + + def test_resolve_package_module_separate(self, project_dir): + actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv") + assert actual == f"{project_dir}\\syscore\\tests\\file.csv" + + def test_resolve_package_module_combined(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests.pricetestdata.csv" + ) + assert actual == f"{project_dir}\\syscore\\tests\\pricetestdata.csv" + + @pytest.mark.xfail(reason="Cannot work with old or new implementation") + def test_resolve_package_module_combined_dotted_filename(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests.price.test.data.csv" + ) + assert actual == f"{project_dir}\\syscore\\tests\\price.test.data.csv" + + def test_resolve_package_module_combined_dotted_filename_sep(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscore.tests", "price.test.data.csv" + ) + assert actual == f"{project_dir}\\syscore\\tests\\price.test.data.csv" + + @pytest.mark.xfail(reason="Cannot work with old or new implementation") + def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name( + self, tmp_path + ): + directory = tmp_path / "dir.name.with.dots" + directory.mkdir() + file = directory / "hello.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}\\dir.name.with.dots", "hello.txt" + ) + assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt" + + @pytest.mark.xfail(reason="Cannot work with old or new implementation") + def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name( + self, tmp_path + ): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}\\dir_name\\dotted.filename.txt" + ) + assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt" + + def test_resolve_resolve_path_and_filename_for_package_with_separate_dotted_file_name( + self, tmp_path + ): + directory = tmp_path / "dir_name" + directory.mkdir() + file = directory / "dotted.filename.txt" + file.write_text("content", encoding="utf-8") + resolved_path = resolve_path_and_filename_for_package( + f"{tmp_path}\\dir_name\\", "dotted.filename.txt" + ) + assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt" + + # No Separate filename + + def test_csv_data(self, project_dir): + actual = resolve_path_and_filename_for_package( + "data.futures.csvconfig.instrumentconfig.csv" + ) + assert ( + actual == f"{project_dir}\\data\\futures\\csvconfig\\instrumentconfig.csv" + ) + + def test_logging(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syslogging.logging_prod.yaml", + ) + assert actual == f"{project_dir}\\syslogging\\logging_prod.yaml" + + def test_config_defaults(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysdata.config.defaults.yaml", + ) + assert actual == f"{project_dir}\\sysdata\\config\\defaults.yaml" + + def test_control_config_defaults(self, project_dir): + actual = resolve_path_and_filename_for_package( + "syscontrol.control_config.yaml", + ) + assert actual == f"{project_dir}\\syscontrol\\control_config.yaml" + + def test_private_config_dots(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private.private_config.yaml", + ) + assert actual == f"{project_dir}\\private\\private_config.yaml" + + def test_private_config_slash(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private/private_config.yaml", + ) + assert actual == f"{project_dir}\\private\\private_config.yaml" + + def test_strategy_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "systems.provided.rob_system.config.yaml", + ) + assert actual == f"{project_dir}\\systems\\provided\\rob_system\\config.yaml" + + def test_pickled_backtest(self, project_dir): + actual = resolve_path_and_filename_for_package( + "private/backtests/fut_strategy_v1_8/20260101_210632_backtest.pck", + ) + assert ( + actual + == f"{project_dir}\\private\\backtests\\fut_strategy_v1_8\\20260101_210632_backtest.pck" + ) + + def test_ib_instrument_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.config.ib_config_futures.csv", + ) + assert actual == f"{project_dir}\\sysbrokers\\IB\\config\\ib_config_futures.csv" + + def test_ib_fx_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.config.ib_config_spot_FX.csv", + ) + assert actual == f"{project_dir}\\sysbrokers\\IB\\config\\ib_config_spot_FX.csv" + + def test_ib_trading_hours_config(self, project_dir): + actual = resolve_path_and_filename_for_package( + "sysbrokers.IB.ib_config_trading_hours.yaml", + ) + assert actual == f"{project_dir}\\sysbrokers\\IB\\ib_config_trading_hours.yaml" + + def test_email_store_filename(self): + actual = resolve_path_and_filename_for_package( + "C:\\home\\rob\\logs\\email_store.log", + ) + assert actual == "C:\\home\\rob\\logs\\email_store.log" diff --git a/sysdata/config/configdata.py b/sysdata/config/configdata.py index e3c7b6718d9..a07004b5d66 100644 --- a/sysdata/config/configdata.py +++ b/sysdata/config/configdata.py @@ -12,9 +12,6 @@ """ -from pathlib import Path -import os - import yaml from syscore.exceptions import missingData @@ -23,12 +20,8 @@ from sysdata.config.defaults import get_system_defaults_dict from sysdata.config.private_config import ( get_private_config_as_dict, - PRIVATE_CONFIG_FILE, -) -from sysdata.config.private_directory import ( - get_full_path_for_private_config, - PRIVATE_CONFIG_DIR_ENV_VAR, ) + from syslogging.logger import * from sysdata.config.fill_config_dict_with_defaults import fill_config_dict_with_defaults @@ -58,7 +51,7 @@ def __init__( multiple elements, latter elements will overwrite earlier ones) - :type config_object: str or dict + :type config_object: str or dict or list :returns: new Config object @@ -180,11 +173,10 @@ def _create_config_from_dict(self, config_object): self.add_elements(attr_names) - def system_init(self, base_system): + def system_init(self): """ This is run when added to a base system - :param base_system :return: nothing """ @@ -287,12 +279,7 @@ def default_config(cls): if hasattr(cls, "evaluated"): return cls.evaluated - if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR): - config = Config( - private_filename=get_full_path_for_private_config(PRIVATE_CONFIG_FILE) - ) - else: - config = Config() + config = Config(get_private_config_as_dict()) config.fill_with_defaults() cls.evaluated = config diff --git a/sysdata/config/control_config.py b/sysdata/config/control_config.py index 72d1f6c150f..ae76c9171b6 100644 --- a/sysdata/config/control_config.py +++ b/sysdata/config/control_config.py @@ -1,5 +1,6 @@ from sysdata.config.configdata import Config -from sysdata.config.private_directory import get_full_path_for_private_config +from syscore.fileutils import resolve_path_and_filename_for_package +from sysdata.config.private_config import get_private_config_dir from yaml.parser import ParserError PRIVATE_CONTROL_CONFIG_FILE = "private_control_config.yaml" @@ -7,12 +8,18 @@ def get_control_config() -> Config: - private_control_path = get_full_path_for_private_config(PRIVATE_CONTROL_CONFIG_FILE) + dir = get_private_config_dir() + private_control_path = resolve_path_and_filename_for_package( + dir, PRIVATE_CONTROL_CONFIG_FILE + ) + default_control_path = resolve_path_and_filename_for_package( + DEFAULT_CONTROL_CONFIG_FILE + ) try: control_config = Config( private_filename=private_control_path, - default_filename=DEFAULT_CONTROL_CONFIG_FILE, + default_filename=default_control_path, ) control_config.fill_with_defaults() @@ -21,7 +28,7 @@ def get_control_config() -> Config: except FileNotFoundError: raise Exception( "Need to have either %s or %s or both present:" - % (str(DEFAULT_CONTROL_CONFIG_FILE), str(private_control_path)) + % (private_control_path, default_control_path) ) except BaseException as be: raise Exception("Problem reading control config: %s" % str(be)) diff --git a/sysdata/config/private_config.py b/sysdata/config/private_config.py index 791e592f128..d576e545857 100644 --- a/sysdata/config/private_config.py +++ b/sysdata/config/private_config.py @@ -1,25 +1,38 @@ -from syscore.fileutils import resolve_path_and_filename_for_package, does_filename_exist +import os +import yaml +from pathlib import Path + +from syscore.fileutils import resolve_path_and_filename_for_package from syscore.constants import arg_not_supplied -from sysdata.config.private_directory import get_full_path_for_private_config -import yaml +DEFAULT_PRIVATE_DIR = "private" PRIVATE_CONFIG_FILE = "private_config.yaml" +PRIVATE_CONFIG_DIR_ENV_VAR = "PYSYS_PRIVATE_CONFIG_DIR" def get_private_config_as_dict(filename: str = arg_not_supplied) -> dict: + private_dir = get_private_config_dir() if filename is arg_not_supplied: - filename = get_full_path_for_private_config(PRIVATE_CONFIG_FILE) - if not does_filename_exist(filename): + filename = PRIVATE_CONFIG_FILE + try: + private_path = resolve_path_and_filename_for_package(private_dir, filename) + with open(private_path) as file_to_parse: + private_dict = yaml.load(file_to_parse, Loader=yaml.FullLoader) + return private_dict + + except Exception: print( - "Private configuration %s does not exist; no problem if running in sim mode" - % filename + f"Private configuration '{private_path}' is missing or " + f"misconfigured; no problem if running in sim mode" ) - return {} - private_file = resolve_path_and_filename_for_package(filename) - with open(private_file) as file_to_parse: - private_dict = yaml.load(file_to_parse, Loader=yaml.FullLoader) - return private_dict +def get_private_config_dir(): + if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR): + private_config_dir = Path(os.environ[PRIVATE_CONFIG_DIR_ENV_VAR]) + else: + private_config_dir = Path(DEFAULT_PRIVATE_DIR) + + return str(private_config_dir) diff --git a/sysdata/config/private_directory.py b/sysdata/config/private_directory.py deleted file mode 100644 index a844f341d45..00000000000 --- a/sysdata/config/private_directory.py +++ /dev/null @@ -1,15 +0,0 @@ -import os - -DEFAULT_PRIVATE_DIR = "private" -PRIVATE_CONFIG_DIR_ENV_VAR = "PYSYS_PRIVATE_CONFIG_DIR" - - -def get_full_path_for_private_config(filename: str): - ## FIXME: should use os path join instead of '/'? - - if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR): - private_config_path = f"{os.environ[PRIVATE_CONFIG_DIR_ENV_VAR]}/{filename}" - else: - private_config_path = f"{DEFAULT_PRIVATE_DIR}/{filename}" - - return private_config_path diff --git a/sysdata/parquet/parquet_access.py b/sysdata/parquet/parquet_access.py index 990d7d34f9f..48b7123061e 100644 --- a/sysdata/parquet/parquet_access.py +++ b/sysdata/parquet/parquet_access.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path import pandas as pd from syscore.exceptions import missingFile from syscore.fileutils import ( @@ -19,13 +19,13 @@ def get_all_identifiers_with_data_type(self, data_type: str): path = self._get_pathname_given_data_type(data_type) return files_with_extension_in_pathname(path, extension=EXTENSION) - def does_idenitifier_with_data_type_exist( + def does_identifier_with_data_type_exist( self, data_type: str, identifier: str ) -> bool: filename = self._get_filename_given_data_type_and_identifier( data_type=data_type, identifier=identifier ) - return os.path.isfile(filename) + return Path(filename).exists() def delete_data_given_data_type_and_identifier( self, data_type: str, identifier: str @@ -34,7 +34,7 @@ def delete_data_given_data_type_and_identifier( data_type=data_type, identifier=identifier ) try: - os.remove(filename) + Path(filename).unlink() except FileNotFoundError: raise missingFile(f"File '{filename}' does not exist") @@ -66,7 +66,6 @@ def _get_filename_given_data_type_and_identifier( def _get_pathname_given_data_type(self, data_type: str): root = self.parquet_store - path = os.path.join(root, data_type) - Path(path).mkdir(parents=True, exist_ok=True) - - return path + path = Path(root, data_type) + path.mkdir(parents=True, exist_ok=True) + return str(path) diff --git a/sysdata/parquet/parquet_futures_per_contract_prices.py b/sysdata/parquet/parquet_futures_per_contract_prices.py index 6fa5566e768..01c595f5d01 100644 --- a/sysdata/parquet/parquet_futures_per_contract_prices.py +++ b/sysdata/parquet/parquet_futures_per_contract_prices.py @@ -148,7 +148,7 @@ def has_price_data_for_contract_at_frequency( self, contract_object: futuresContract, frequency: Frequency ) -> bool: ident = from_contract_and_freq_to_key(contract_object, frequency=frequency) - return self.parquet.does_idenitifier_with_data_type_exist( + return self.parquet.does_identifier_with_data_type_exist( data_type=CONTRACT_COLLECTION, identifier=ident ) diff --git a/sysdata/tests/custom_private_config/private_config_trading_hours.yaml b/sysdata/tests/custom_private_config/private_config_trading_hours.yaml new file mode 100644 index 00000000000..cd381be1e36 --- /dev/null +++ b/sysdata/tests/custom_private_config/private_config_trading_hours.yaml @@ -0,0 +1,3 @@ +MET: + - '09:00' + - '14:00' diff --git a/sysdata/tests/custom_private_config/private_control_config.yaml b/sysdata/tests/custom_private_config/private_control_config.yaml index 836a43e9c42..234a7e95a6f 100644 --- a/sysdata/tests/custom_private_config/private_control_config.yaml +++ b/sysdata/tests/custom_private_config/private_control_config.yaml @@ -1,2 +1,2 @@ process_configuration_start_time: - run_stack_handler: '01:00' \ No newline at end of file + run_stack_handler: '01:00' diff --git a/sysdata/tests/test_config.py b/sysdata/tests/test_config.py index 20353713927..87c1bf01a31 100644 --- a/sysdata/tests/test_config.py +++ b/sysdata/tests/test_config.py @@ -1,10 +1,36 @@ +import datetime from sysdata.config.configdata import Config from sysdata.config.control_config import get_control_config -from sysdata.config.private_directory import PRIVATE_CONFIG_DIR_ENV_VAR +from sysdata.config.private_config import PRIVATE_CONFIG_DIR_ENV_VAR +from sysbrokers.IB.ib_trading_hours import get_saved_trading_hours class TestConfig: - def test_default(self): + def test_init_dict(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + config = Config(dict(parameters=dict(p1=3, p2=4.6), another_thing="foo")) + assert config.as_dict()["parameters"]["p2"] == 4.6 + assert config.as_dict()["another_thing"] == "foo" + + def test_init_str(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + config = Config("systems.provided.example.exampleconfig.yaml") + assert config.as_dict()["forecast_cap"] == 21.0 + + def test_init_list(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + config = Config( + [ + "systems.provided.example.exampleconfig.yaml", + dict(parameters=dict(p1=3, p2=4.6), another_thing="foo"), + ] + ) + assert config.as_dict()["forecast_cap"] == 21.0 + assert config.as_dict()["parameters"]["p2"] == 4.6 + assert config.as_dict()["another_thing"] == "foo" + + def test_default(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) Config.reset() config = Config.default_config() assert config.get_element("ib_idoffset") == 100 @@ -25,7 +51,9 @@ def test_bad_custom_dir(self, monkeypatch): config = Config.default_config() assert config.get_element("ib_idoffset") == 100 - def test_default_control(self): + def test_default_control(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + Config.reset() config = get_control_config() assert ( config.as_dict()["process_configuration_start_time"]["run_stack_handler"] @@ -51,3 +79,21 @@ def test_control_bad_custom_dir(self, monkeypatch): config.as_dict()["process_configuration_start_time"]["run_stack_handler"] == "00:01" ) + + def test_trading_hours_default(self, monkeypatch): + monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False) + config = get_saved_trading_hours() + assert config["MET"]["Monday"][0].closing_time == datetime.time(15) + + def test_trading_hours_custom(self, monkeypatch): + monkeypatch.setenv( + PRIVATE_CONFIG_DIR_ENV_VAR, "sysdata.tests.custom_private_config" + ) + config = get_saved_trading_hours() + assert config["MET"]["Monday"][0].opening_time == datetime.time(9) + assert config["MET"]["Monday"][0].closing_time == datetime.time(14) + + def test_trading_hours_bad_custom_dir(self, monkeypatch): + monkeypatch.setenv(PRIVATE_CONFIG_DIR_ENV_VAR, "sysdata.tests") + config = get_saved_trading_hours() + assert config["MET"]["Monday"][0].closing_time == datetime.time(15) diff --git a/sysinit/futures/barchart_futures_contract_prices.py b/sysinit/futures/barchart_futures_contract_prices.py index 4147f7f2f6a..62c4a4cc0e3 100644 --- a/sysinit/futures/barchart_futures_contract_prices.py +++ b/sysinit/futures/barchart_futures_contract_prices.py @@ -1,5 +1,5 @@ from sysdata.csv.csv_futures_contract_prices import ConfigCsvFuturesPrices -import os +from pathlib import Path from syscore.fileutils import ( get_resolved_pathname, files_with_extension_in_resolved_pathname, @@ -30,11 +30,11 @@ def strip_file_names(pathname): datecode = str(year) + "{0:02d}".format(month) new_file_name = "%s_%s00.csv" % (instrument, datecode) - new_full_name = os.path.join(resolved_pathname, new_file_name) - old_full_name = os.path.join(resolved_pathname, filename + ".csv") + new_full_name = Path(resolved_pathname, new_file_name) + old_full_name = Path(resolved_pathname, filename + ".csv") print("Rename %s to\n %s" % (old_full_name, new_full_name)) - os.rename(old_full_name, new_full_name) + old_full_name.rename(new_full_name) return None diff --git a/syslogging/logger.py b/syslogging/logger.py index f51c4f74eb6..df6d9321bd0 100644 --- a/syslogging/logger.py +++ b/syslogging/logger.py @@ -1,3 +1,4 @@ +from pathlib import Path import os import sys import socket @@ -99,7 +100,7 @@ def _configure_sim(): def _configure_prod(logging_config_file): print(f"Attempting to configure prod logging from {logging_config_file}") config_path = resolve_path_and_filename_for_package(logging_config_file) - if os.path.exists(config_path): + if Path(config_path).exists(): try: config = parse_config(path=config_path) host, port = _get_log_server_config(config) diff --git a/sysproduction/backup_db_to_csv.py b/sysproduction/backup_db_to_csv.py index 0ff4e8e694f..7b2e233d1e8 100644 --- a/sysproduction/backup_db_to_csv.py +++ b/sysproduction/backup_db_to_csv.py @@ -1,3 +1,4 @@ +from pathlib import Path import os import pandas as pd @@ -98,10 +99,9 @@ def get_data_and_create_csv_directories(logname): ) for class_name, path in class_paths.items(): - dir_name = os.path.join(csv_dump_dir, path) - class_paths[class_name] = dir_name - if not os.path.exists(dir_name): - os.makedirs(dir_name) + dir_name = Path(csv_dump_dir, path) + class_paths[class_name] = str(dir_name) + Path(dir_name).mkdir(exist_ok=True) data = dataBlob(csv_data_paths=class_paths, log_name=logname) diff --git a/sysproduction/data/backtest.py b/sysproduction/data/backtest.py index 105d440d292..ab3361f5241 100644 --- a/sysproduction/data/backtest.py +++ b/sysproduction/data/backtest.py @@ -1,5 +1,5 @@ from copy import copy -import os +from pathlib import Path from shutil import copyfile from syscore.dateutils import create_datetime_marker_string @@ -218,10 +218,7 @@ def store_backtest_state(data, system, strategy_name="default_strategy"): def ensure_backtest_directory_exists(strategy_name): full_directory = get_backtest_directory_for_strategy(strategy_name) - try: - os.makedirs(full_directory) - except FileExistsError: - pass + Path(full_directory).mkdir(exist_ok=True) def rchop(s, suffix): @@ -262,9 +259,9 @@ def get_backtest_config_filename(strategy_name, datetime_marker): def get_backtest_filename_prefix(strategy_name, datetime_marker): # eg '/home/rob/data/backtests/medium_speed_TF_carry/20200622_102913' full_directory = get_backtest_directory_for_strategy(strategy_name) - full_filename_prefix = os.path.join(full_directory, datetime_marker) + full_filename_prefix = Path(full_directory, datetime_marker) - return full_filename_prefix + return str(full_filename_prefix) def get_backtest_directory_for_strategy(strategy_name): @@ -272,9 +269,9 @@ def get_backtest_directory_for_strategy(strategy_name): directory_store_backtests = get_directory_store_backtests() directory_store_backtests = get_resolved_pathname(directory_store_backtests) - full_directory = os.path.join(directory_store_backtests, strategy_name) + full_directory = Path(directory_store_backtests, strategy_name) - return full_directory + return str(full_directory) def get_directory_store_backtests(): diff --git a/sysproduction/interactive_update_capital_manual.py b/sysproduction/interactive_update_capital_manual.py index ecd08053ca8..50553414188 100644 --- a/sysproduction/interactive_update_capital_manual.py +++ b/sysproduction/interactive_update_capital_manual.py @@ -266,3 +266,7 @@ def delete_all_capital(data: dataBlob): ) else: print("OK you decided not to do it") + + +if __name__ == "__main__": + interactive_update_capital_manual() diff --git a/sysproduction/reporting/reporting_functions.py b/sysproduction/reporting/reporting_functions.py index 82607834ff8..4410e78e915 100644 --- a/sysproduction/reporting/reporting_functions.py +++ b/sysproduction/reporting/reporting_functions.py @@ -3,6 +3,7 @@ from PyPDF2 import PdfMerger import datetime import pandas as pd +from pathlib import Path import os import shutil import matplotlib.pyplot as plt @@ -274,14 +275,14 @@ def output_file_report( data.log.debug("Written report to %s" % full_filename) -def resolve_report_filename(report_config, data: dataBlob): +def resolve_report_filename(report_config, data: dataBlob) -> str: filename_with_spaces = report_config.title filename = filename_with_spaces.replace(" ", "_") use_directory = get_directory_for_reporting(data) use_directory_resolved = get_resolved_pathname(use_directory) - full_filename = os.path.join(use_directory_resolved, filename) + full_filename = Path(use_directory_resolved, filename) - return full_filename + return str(full_filename) def get_directory_for_reporting(data): @@ -337,6 +338,6 @@ def _generate_temp_pdf_filename( TEMPFILE_PATTERN, str(datetime_to_long(datetime.datetime.now())), ) - full_filename = os.path.join(use_directory_resolved, filename) + full_filename = Path(use_directory_resolved, filename) - return full_filename + return str(full_filename) diff --git a/sysproduction/update_strategy_orders.py b/sysproduction/update_strategy_orders.py index 5a53e03b1b6..302793fd626 100644 --- a/sysproduction/update_strategy_orders.py +++ b/sysproduction/update_strategy_orders.py @@ -28,3 +28,7 @@ def update_strategy_orders(): data, strategy_name, process_name, name_of_main_generator_method ) strategy_order_generator.run_strategy_method() + + +if __name__ == "__main__": + update_strategy_orders() diff --git a/sysproduction/update_system_backtests.py b/sysproduction/update_system_backtests.py index 32b646e7f18..0bd5dfe112c 100644 --- a/sysproduction/update_system_backtests.py +++ b/sysproduction/update_system_backtests.py @@ -27,3 +27,7 @@ def update_system_backtests(): data, strategy_name, process_name, backtest_function ) system_backtest_runner.run_strategy_method() + + +if __name__ == "__main__": + update_system_backtests() diff --git a/systems/basesystem.py b/systems/basesystem.py index 4d4d4487291..b66679cfdd0 100644 --- a/systems/basesystem.py +++ b/systems/basesystem.py @@ -77,7 +77,7 @@ def __init__( self._config = config self._log = log - self.config.system_init(self) + self.config.system_init() self.data.system_init(self) self._setup_stages(stage_list) self._cache = systemCache(self) diff --git a/systems/tests/test_position_sizing.py b/systems/tests/test_position_sizing.py index f3e4fec0f70..c52c3f9b24d 100644 --- a/systems/tests/test_position_sizing.py +++ b/systems/tests/test_position_sizing.py @@ -1,6 +1,6 @@ import unittest from _pytest.monkeypatch import MonkeyPatch -from sysdata.config.private_directory import PRIVATE_CONFIG_DIR_ENV_VAR +from sysdata.config.private_config import PRIVATE_CONFIG_DIR_ENV_VAR from systems.tests.testdata import get_test_object_futures_with_comb_forecasts from systems.basesystem import System from systems.positionsizing import PositionSizing