diff --git a/.github/workflows/os-test.yml b/.github/workflows/os-test.yml
index c3253a50285..761c0dd1e58 100644
--- a/.github/workflows/os-test.yml
+++ b/.github/workflows/os-test.yml
@@ -25,4 +25,8 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools
- python -m pip install .
+ python -m pip install '.[dev]'
+
+ - name: Test with pytest
+ run: |
+ pytest syscore/tests/test_fileutils.py
diff --git a/docs/backtesting.md b/docs/backtesting.md
index d489c91cfa1..a70ce4a5635 100644
--- a/docs/backtesting.md
+++ b/docs/backtesting.md
@@ -3123,7 +3123,6 @@ resolve_path_and_filename_for_package("\\home/rob.file.csv")
```
-
These functions are used internally whenever a file name is passed in, so feel free to use any of these file formats when specifying eg a configuration filename.
```
### Absolute: Windows (note use of double backslash in str)
@@ -3136,6 +3135,31 @@ These functions are used internally whenever a file name is passed in, so feel f
"syscore.tests.pricedata.csv"
```
+Obviously, using 'dots' as a separator brings limitations, like supporting directories or files with 'dots' in their names. This won't work:
+```
+>>> resolve_path_and_filename_for_package("syscore/tests/price.test.data.csv")
+'/home/user/pysystemtrade/syscore/tests/price/test/data.csv'
+```
+
+do this instead:
+```
+>>> resolve_path_and_filename_for_package("syscore/tests", "price.test.data.csv")
+'/home/user/pysystemtrade/syscore/tests/price.test.data.csv'
+```
+
+This will also not work:
+```
+>>> get_resolved_pathname("data/dir.with.dots")
+'/home/user/pysystemtrade/data/dir/with/dots'
+```
+
+do this instead:
+```
+>>> from pathlib import Path
+>>> get_resolved_pathname(Path("data/dir.with.dots"))
+'/home/user/pysystemtrade/data/dir.with.dots'
+```
+
## Logging
### Basic logging
diff --git a/pyproject.toml b/pyproject.toml
index 9b5b5969659..133ba7d477c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,5 +1,8 @@
[build-system]
-requires = ["setuptools >= 61.0"]
+requires = [
+ "setuptools >= 61.0",
+ "setuptools-scm >= 8.0"
+]
build-backend = "setuptools.build_meta"
[project]
@@ -47,8 +50,11 @@ find = {}
[tool.setuptools.package-data]
"data" = ["*.csv"]
+"data.futures" = ["*.csv"]
"private" = ["*.yaml"]
-"sysbrokers" = ["*.csv", "*.yaml"]
+"sysbrokers" = ["*.csv"]
+"sysbrokers.IB" = ["*.yaml"]
+"sysbrokers.IB.config" = ["*.csv"]
"syscontrol" = ["*.yaml"]
"sysdata" = ["*.csv"]
"sysdata.config" = ["*.yaml"]
diff --git a/sysbrokers/IB/ib_trading_hours.py b/sysbrokers/IB/ib_trading_hours.py
index 256ea40c9c0..77c5b40b3f9 100644
--- a/sysbrokers/IB/ib_trading_hours.py
+++ b/sysbrokers/IB/ib_trading_hours.py
@@ -1,26 +1,35 @@
import datetime
from ib_insync import ContractDetails as ibContractDetails
-from sysdata.config.private_directory import get_full_path_for_private_config
+from sysdata.config.private_config import (
+ get_private_config_dir,
+)
from sysobjects.production.trading_hours.trading_hours import (
tradingHours,
listOfTradingHours,
)
-from syscore.fileutils import does_filename_exist
+from syscore.fileutils import (
+ does_resolved_filename_exist,
+ resolve_path_and_filename_for_package,
+)
from sysdata.config.production_config import get_production_config
from sysdata.production.trading_hours import read_trading_hours
IB_CONFIG_TRADING_HOURS_FILE = "sysbrokers.IB.ib_config_trading_hours.yaml"
-PRIVATE_CONFIG_TRADING_HOURS_FILE = get_full_path_for_private_config(
- "private_config_trading_hours.yaml"
-)
+PRIVATE_CONFIG_TRADING_HOURS_FILE = "private_config_trading_hours.yaml"
def get_saved_trading_hours():
- if does_filename_exist(PRIVATE_CONFIG_TRADING_HOURS_FILE):
- return read_trading_hours(PRIVATE_CONFIG_TRADING_HOURS_FILE)
+ private_path = resolve_path_and_filename_for_package(
+ get_private_config_dir(), PRIVATE_CONFIG_TRADING_HOURS_FILE
+ )
+ if does_resolved_filename_exist(private_path):
+ return read_trading_hours(private_path)
else:
- return read_trading_hours(IB_CONFIG_TRADING_HOURS_FILE)
+ default_path = resolve_path_and_filename_for_package(
+ IB_CONFIG_TRADING_HOURS_FILE
+ )
+ return read_trading_hours(default_path)
def get_trading_hours_from_contract_details(
diff --git a/syscore/__init__.py b/syscore/__init__.py
index e69de29bb2d..61008830ee9 100755
--- a/syscore/__init__.py
+++ b/syscore/__init__.py
@@ -0,0 +1,7 @@
+from importlib.util import find_spec
+from pathlib import Path
+
+module_spec = find_spec(__name__)
+path = Path(module_spec.origin)
+
+PYSYS_PROJECT_DIR = path.parent.parent
diff --git a/syscore/fileutils.py b/syscore/fileutils.py
index d5bf79c658b..23ad87be24b 100755
--- a/syscore/fileutils.py
+++ b/syscore/fileutils.py
@@ -1,21 +1,14 @@
import glob
import datetime
import time
-from importlib import import_module
import os
-from pathlib import Path
-from typing import List, Tuple
+import re
+from pathlib import Path, PurePath
+from typing import List, Tuple, TextIO
+from syscore import PYSYS_PROJECT_DIR
from syscore.dateutils import SECONDS_PER_DAY
-# DO NOT DELETE: all these are unused: but are required to get the filename padding to work
-
-
-"""
-
- FILES IN DIRECTORIES
-
-"""
"""
@@ -26,7 +19,7 @@
def rename_files_with_extension_in_pathname_as_archive_files(
pathname: str, extension: str = ".txt", archive_extension: str = ".arch"
-):
+) -> None:
"""
Find all the files with a particular extension in a directory, and rename them
eg thing.txt will become thing_yyyymmdd.txt where yyyymmdd is todays date
@@ -47,7 +40,7 @@ def rename_files_with_extension_in_pathname_as_archive_files(
def rename_file_as_archive(
full_filename: str, old_extension: str = ".txt", archive_extension: str = ".arch"
-):
+) -> None:
"""
Rename a file with archive suffix and extension
eg thing.txt will become thing_yyyymmdd.arch where yyyymmdd is todays date
@@ -62,8 +55,8 @@ def rename_file_as_archive(
def delete_old_files_with_extension_in_pathname(
- pathname: str, days_old=30, extension=".arch"
-):
+ pathname: str, days_old: int = 30, extension: str = ".arch"
+) -> None:
"""
Find all the files with a particular extension in a directory, and delete them
if older than x days
@@ -77,7 +70,7 @@ def delete_old_files_with_extension_in_pathname(
delete_file_if_too_old(filename, days_old=days_old)
-def delete_file_if_too_old(full_filename_with_ext: str, days_old: int = 30):
+def delete_file_if_too_old(full_filename_with_ext: str, days_old: int = 30) -> None:
file_age = get_file_or_folder_age_in_days(full_filename_with_ext)
if file_age > days_old:
print("Deleting %s" % full_filename_with_ext)
@@ -103,7 +96,7 @@ def get_file_or_folder_age_in_days(full_filename_with_ext: str) -> float:
def resolve_path_and_filename_for_package(
- path_and_filename: str, separate_filename=None
+ path_and_filename: str, separate_filename: str | None = None
) -> str:
"""
A way of resolving relative and absolute filenames, and dealing with awkward OS specific things
@@ -134,22 +127,21 @@ def resolve_path_and_filename_for_package(
"""
- path_and_filename_as_list = transform_path_into_list(path_and_filename)
+ path_and_filename_as_list = _transform_path_into_list(path_and_filename)
if separate_filename is None:
(
path_as_list,
separate_filename,
- ) = extract_filename_from_combined_path_and_filename_list(
+ ) = _extract_filename_from_combined_path_and_filename_list(
path_and_filename_as_list
)
else:
path_as_list = path_and_filename_as_list
- resolved_pathname = get_pathname_from_list(path_as_list)
-
- resolved_path_and_filename = os.path.join(resolved_pathname, separate_filename)
+ absolute_path = _make_absolute(path_as_list)
+ result = absolute_path / separate_filename
- return resolved_path_and_filename
+ return str(result)
def get_resolved_pathname(pathname: str) -> str:
@@ -168,51 +160,65 @@ def get_resolved_pathname(pathname: str) -> str:
"""
- if isinstance(pathname, Path):
+ if isinstance(pathname, Path) and pathname.exists():
# special case when already a Path
- pathname = str(pathname.absolute())
-
- if "@" in pathname or "::" in pathname:
- # This is an ssh address for rsync - don't change
- return pathname
+ return str(pathname.absolute())
+ else:
+ pathname = str(pathname)
+ if "@" in pathname or "::" in pathname:
+ # This is an ssh address for rsync - don't change
+ return pathname
- # Turn /,\ into . so system independent
- path_as_list = transform_path_into_list(pathname)
- resolved_pathname = get_pathname_from_list(path_as_list)
+ path_as_list = _transform_path_into_list(pathname)
+ result = _make_absolute(path_as_list)
- return resolved_pathname
+ return str(result)
## something unlikely to occur naturally in a pathname
RESERVED_CHARACTERS = "&!*"
-def transform_path_into_list(pathname: str) -> List[str]:
+def _make_absolute(path_as_list: list[str]) -> PurePath:
+ # handle absolute unix
+ if path_as_list[0] == "":
+ path_as_list[0] = os.sep
+ # handle absolute windows
+ if re.match("[A-Za-z]:$", path_as_list[0]):
+ path_as_list[0] = f"{path_as_list[0]}{os.sep}"
+ path_obj = PurePath(*path_as_list)
+ if not path_obj.is_absolute():
+ path_obj = PYSYS_PROJECT_DIR / path_obj
+
+ return path_obj
+
+
+def _transform_path_into_list(pathname: str) -> List[str]:
"""
- >>> path_as_list("/home/rob/test.csv")
+ >>> _transform_path_into_list("/home/rob/test.csv")
['', 'home', 'rob', 'test', 'csv']
- >>> path_as_list("/home/rob/")
+ >>> _transform_path_into_list("/home/rob/")
['', 'home', 'rob']
- >>> path_as_list(".home.rob")
+ >>> _transform_path_into_list(".home.rob")
['', 'home', 'rob']
- >>> path_as_list('C:\\home\\rob\\'')
+ >>> _transform_path_into_list('C:\\home\\rob\\'')
['C:', 'home', 'rob']
- >>> path_as_list('C:\\home\\rob\\test.csv')
+ >>> _transform_path_into_list('C:\\home\\rob\\test.csv')
['C:', 'home', 'rob', 'test', 'csv']
- >>> path_as_list("syscore.tests.fileutils.csv")
+ >>> _transform_path_into_list("syscore.tests.fileutils.csv")
['syscore', 'tests', 'fileutils', 'csv']
- >>> path_as_list("syscore.tests")
+ >>> _transform_path_into_list("syscore.tests")
['syscore', 'tests']
"""
- pathname_replace = add_reserved_characters_to_pathname(pathname)
+ pathname_replace = _add_reserved_characters_to_pathname(pathname)
path_as_list = pathname_replace.rsplit(RESERVED_CHARACTERS)
if path_as_list[-1] == "":
@@ -221,7 +227,7 @@ def transform_path_into_list(pathname: str) -> List[str]:
return path_as_list
-def add_reserved_characters_to_pathname(pathname: str) -> str:
+def _add_reserved_characters_to_pathname(pathname: str) -> str:
pathname_replace = pathname.replace(".", RESERVED_CHARACTERS)
pathname_replace = pathname_replace.replace("/", RESERVED_CHARACTERS)
pathname_replace = pathname_replace.replace("\\", RESERVED_CHARACTERS)
@@ -229,11 +235,11 @@ def add_reserved_characters_to_pathname(pathname: str) -> str:
return pathname_replace
-def extract_filename_from_combined_path_and_filename_list(
- path_and_filename_as_list: list,
-) -> Tuple[list, str]:
+def _extract_filename_from_combined_path_and_filename_list(
+ path_and_filename_as_list: list[str],
+) -> Tuple[list[str], str]:
"""
- >>> extract_filename_from_combined_path_and_filename_list(['home', 'rob','file', 'csv'])
+ >>> _extract_filename_from_combined_path_and_filename_list(['home', 'rob','file', 'csv'])
(['home', 'rob'], 'file.csv')
"""
## need -2 because want extension
@@ -245,91 +251,6 @@ def extract_filename_from_combined_path_and_filename_list(
return path_and_filename_as_list, separate_filename
-def get_pathname_from_list(path_as_list: List[str]) -> str:
- """
- >>> get_pathname_from_list(['C:', 'home', 'rob'])
- 'C:\\home\\rob'
- >>> get_pathname_from_list(['','home','rob'])
- '/home/rob'
- >>> get_pathname_from_list(['syscore','tests'])
- '/home/rob/pysystemtrade/syscore/tests'
- """
- if path_as_list[0] == "":
- # path_type_absolute
- resolved_pathname = get_absolute_linux_pathname_from_list(path_as_list[1:])
- elif is_windoze_path_list(path_as_list):
- # windoze
- resolved_pathname = get_absolute_windows_pathname_from_list(path_as_list)
- else:
- # relative
- resolved_pathname = get_relative_pathname_from_list(path_as_list)
-
- return resolved_pathname
-
-
-def is_windoze_path_list(path_as_list: List[str]) -> bool:
- """
- >>> is_windoze_path_list(['C:'])
- True
- >>> is_windoze_path_list(['wibble'])
- False
- """
- return path_as_list[0].endswith(":")
-
-
-def get_relative_pathname_from_list(path_as_list: List[str]) -> str:
- """
-
- >>> get_relative_pathname_from_list(['syscore','tests'])
- '/home/rob/pysystemtrade/syscore/tests'
- """
- package_name = path_as_list[0]
- paths_or_files = path_as_list[1:]
-
- if len(paths_or_files) == 0:
- directory_name_of_package = os.path.dirname(
- import_module(package_name).__file__
- )
- return directory_name_of_package
-
- last_item_in_list = path_as_list.pop()
- pathname = os.path.join(
- get_relative_pathname_from_list(path_as_list), last_item_in_list
- )
-
- return pathname
-
-
-def get_absolute_linux_pathname_from_list(path_as_list: List[str]) -> str:
- """
- Returns the absolute pathname from a list
-
- >>> get_absolute_linux_pathname_from_list(['home', 'rob'])
- '/home/rob'
- """
- pathname = os.path.join(*path_as_list)
- pathname = os.path.sep + pathname
-
- return pathname
-
-
-def get_absolute_windows_pathname_from_list(path_as_list: list) -> str:
- """
- Test will fail on linux
- >>> get_absolute_windows_pathname_from_list(['C:','home','rob'])
- 'C:\\home\\rob'
- """
- drive_part_of_path = path_as_list[0]
- if drive_part_of_path.endswith(":"):
- ## add back backslash
- drive_part_of_path = drive_part_of_path.replace(":", ":\\")
- path_as_list[0] = drive_part_of_path
-
- pathname = os.path.join(*path_as_list)
-
- return pathname
-
-
"""
HTML
@@ -337,7 +258,9 @@ def get_absolute_windows_pathname_from_list(path_as_list: list) -> str:
"""
-def write_list_of_lists_as_html_table_in_file(file, list_of_lists: list):
+def write_list_of_lists_as_html_table_in_file(
+ file: TextIO, list_of_lists: list[str]
+) -> None:
file.write("
")
for sublist in list_of_lists:
file.write(" | ")
@@ -347,7 +270,9 @@ def write_list_of_lists_as_html_table_in_file(file, list_of_lists: list):
file.write(" |
")
-def files_with_extension_in_pathname(pathname: str, extension=".csv") -> List[str]:
+def files_with_extension_in_pathname(
+ pathname: str, extension: str = ".csv"
+) -> List[str]:
"""
Find all the files with a particular extension in a directory
@@ -360,7 +285,7 @@ def files_with_extension_in_pathname(pathname: str, extension=".csv") -> List[st
def files_with_extension_in_resolved_pathname(
- resolved_pathname: str, extension=".csv"
+ resolved_pathname: str, extension: str = ".csv"
) -> List[str]:
"""
Find all the files with a particular extension in a directory
@@ -375,9 +300,7 @@ def files_with_extension_in_resolved_pathname(
def full_filename_for_file_in_home_dir(filename: str) -> str:
- pathname = os.path.expanduser("~")
-
- return os.path.join(pathname, filename)
+ return str(Path.home() / filename)
def does_filename_exist(filename: str) -> bool:
@@ -387,5 +310,4 @@ def does_filename_exist(filename: str) -> bool:
def does_resolved_filename_exist(resolved_filename: str) -> bool:
- file_exists = os.path.isfile(resolved_filename)
- return file_exists
+ return Path(resolved_filename).exists()
diff --git a/syscore/tests/price.test.data.csv b/syscore/tests/price.test.data.csv
new file mode 100644
index 00000000000..b4c9c312a5b
--- /dev/null
+++ b/syscore/tests/price.test.data.csv
@@ -0,0 +1,529 @@
+DATETIME,ADJ
+2013-04-15,139.0825
+2013-04-16,
+2013-04-17,
+2013-04-18,
+2013-04-19,
+2013-04-22,
+2013-04-23,
+2013-04-24,
+2013-04-25,
+2013-04-26,
+2013-04-29,
+2013-04-30,
+2013-05-01,
+2013-05-02,
+2013-05-03,138.7825
+2013-05-06,137.0525
+2013-05-07,
+2013-05-08,
+2013-05-09,
+2013-05-10,
+2013-05-13,
+2013-05-14,
+2013-05-15,
+2013-05-16,
+2013-05-17,
+2013-05-20,
+2013-05-21,
+2013-05-22,
+2013-05-23,
+2013-05-24,
+2013-05-27,
+2013-05-28,134.4125
+2013-05-29,
+2013-05-30,
+2013-05-31,
+2013-06-03,134.9825
+2013-06-04,134.8025
+2013-06-05,135.1225
+2013-06-06,135.2825
+2013-06-07,134.9225
+2013-06-10,134.1625
+2013-06-11,133.9425
+2013-06-12,134.1025
+2013-06-13,134.4325
+2013-06-14,135.1625
+2013-06-17,135.0225
+2013-06-18,134.7025
+2013-06-19,133.8925
+2013-06-20,133.5725
+2013-06-21,132.5025
+2013-06-24,132.1325
+2013-06-25,131.8025
+2013-06-26,132.4325
+2013-06-27,132.9925
+2013-06-28,132.9025
+2013-07-01,132.9625
+2013-07-02,133.5125
+2013-07-03,133.5825
+2013-07-04,133.7125
+2013-07-05,132.9225
+2013-07-08,133.3625
+2013-07-09,134.0725
+2013-07-10,134.0025
+2013-07-11,134.1825
+2013-07-12,135.1125
+2013-07-15,134.7825
+2013-07-16,135.1225
+2013-07-17,135.2825
+2013-07-18,135.4425
+2013-07-19,135.4825
+2013-07-22,135.5325
+2013-07-23,135.1025
+2013-07-24,133.6825
+2013-07-25,133.6925
+2013-07-26,133.8425
+2013-07-29,133.7025
+2013-07-30,133.7125
+2013-07-31,134.2825
+2013-08-01,133.6025
+2013-08-02,134.1125
+2013-08-05,133.5625
+2013-08-06,133.3825
+2013-08-07,133.5225
+2013-08-08,133.6625
+2013-08-09,133.7125
+2013-08-12,133.3725
+2013-08-13,132.1325
+2013-08-14,132.1725
+2013-08-15,131.5325
+2013-08-16,131.0825
+2013-08-19,131.2825
+2013-08-20,131.9625
+2013-08-21,131.3125
+2013-08-22,131.0525
+2013-08-23,130.8825
+2013-08-26,131.3525
+2013-08-27,132.2625
+2013-08-28,131.5825
+2013-08-29,132.0625
+2013-08-30,131.7525
+2013-09-02,131.3225
+2013-09-03,131.1525
+2013-09-04,130.8325
+2013-09-05,129.7325
+2013-09-06,130.8125
+2013-09-09,130.7525
+2013-09-10,129.9325
+2013-09-11,130.6125
+2013-09-12,130.8725
+2013-09-13,131.2225
+2013-09-16,131.4425
+2013-09-17,131.3425
+2013-09-18,132.3225
+2013-09-19,131.7725
+2013-09-20,131.7925
+2013-09-23,132.0725
+2013-09-24,133.0025
+2013-09-25,133.2425
+2013-09-26,133.1025
+2013-09-27,133.6425
+2013-09-30,133.7825
+2013-10-01,133.5425
+2013-10-02,133.6025
+2013-10-03,133.5125
+2013-10-04,133.2425
+2013-10-07,133.5925
+2013-10-08,133.5025
+2013-10-09,133.5225
+2013-10-10,132.9325
+2013-10-11,133.0725
+2013-10-14,133.0325
+2013-10-15,132.5225
+2013-10-16,132.1925
+2013-10-17,132.9625
+2013-10-18,133.3325
+2013-10-21,133.1925
+2013-10-22,133.8225
+2013-10-23,134.1425
+2013-10-24,134.1525
+2013-10-25,134.3425
+2013-10-28,134.4325
+2013-10-29,134.5525
+2013-10-30,134.7325
+2013-10-31,135.2825
+2013-11-01,135.1325
+2013-11-04,135.1725
+2013-11-05,134.4425
+2013-11-06,134.4325
+2013-11-07,135.1125
+2013-11-08,134.3025
+2013-11-11,134.2925
+2013-11-12,133.9825
+2013-11-13,134.6125
+2013-11-14,134.9825
+2013-11-15,134.9125
+2013-11-18,135.1325
+2013-11-19,134.6925
+2013-11-20,134.7225
+2013-11-21,134.2625
+2013-11-22,134.2325
+2013-11-25,134.6025
+2013-11-26,134.9825
+2013-11-27,134.9125
+2013-11-28,134.9325
+2013-11-29,134.9925
+2013-12-02,134.4025
+2013-12-03,134.5625
+2013-12-04,133.6625
+2013-12-05,133.2425
+2013-12-06,133.5575
+2013-12-09,133.5375
+2013-12-10,133.6175
+2013-12-11,133.7475
+2013-12-12,133.5475
+2013-12-13,133.6975
+2013-12-16,133.6975
+2013-12-17,133.7375
+2013-12-18,133.5875
+2013-12-19,133.2475
+2013-12-20,133.3675
+2013-12-23,133.1975
+2013-12-24,
+2013-12-25,
+2013-12-26,
+2013-12-27,132.4075
+2013-12-30,132.6175
+2013-12-31,
+2014-01-01,
+2014-01-02,132.5675
+2014-01-03,132.5375
+2014-01-06,132.9875
+2014-01-07,133.2075
+2014-01-08,132.9875
+2014-01-09,133.0075
+2014-01-10,133.6975
+2014-01-13,134.0975
+2014-01-14,134.1275
+2014-01-15,133.9875
+2014-01-16,134.6075
+2014-01-17,134.8475
+2014-01-20,135.0175
+2014-01-21,135.0975
+2014-01-22,134.9575
+2014-01-23,135.4875
+2014-01-24,136.1075
+2014-01-27,135.9975
+2014-01-28,135.9075
+2014-01-29,136.3475
+2014-01-30,136.6775
+2014-01-31,137.3275
+2014-02-03,137.4675
+2014-02-04,137.4275
+2014-02-05,137.5175
+2014-02-06,136.7775
+2014-02-07,137.2775
+2014-02-10,137.0975
+2014-02-11,137.0875
+2014-02-12,136.6175
+2014-02-13,137.2275
+2014-02-14,137.0475
+2014-02-17,137.0475
+2014-02-18,137.1675
+2014-02-19,137.3575
+2014-02-20,136.8975
+2014-02-21,137.3175
+2014-02-24,137.1275
+2014-02-25,137.4175
+2014-02-26,137.9075
+2014-02-27,138.5175
+2014-02-28,137.8275
+2014-03-03,138.5875
+2014-03-04,138.2
+2014-03-05,138.14
+2014-03-06,137.55
+2014-03-07,137.615
+2014-03-10,137.78
+2014-03-11,137.77
+2014-03-12,138.33
+2014-03-13,139.09
+2014-03-14,138.8
+2014-03-17,138.55
+2014-03-18,138.6275
+2014-03-19,137.86
+2014-03-20,137.76
+2014-03-21,137.93
+2014-03-24,
+2014-03-25,138.57
+2014-03-26,138.9
+2014-03-27,139.09
+2014-03-28,139.01
+2014-03-31,138.84
+2014-04-01,138.65
+2014-04-02,138.18
+2014-04-03,138.3
+2014-04-04,
+2014-04-07,139.01
+2014-04-08,138.8525
+2014-04-09,
+2014-04-10,139.31
+2014-04-11,139.63
+2014-04-14,139.27
+2014-04-15,139.88
+2014-04-16,139.77
+2014-04-17,139.08
+2014-04-18,
+2014-04-21,
+2014-04-22,139.23
+2014-04-23,139.24
+2014-04-24,139.31
+2014-04-25,139.74
+2014-04-28,139.64
+2014-04-29,139.63
+2014-04-30,140.03
+2014-05-01,
+2014-05-02,140.18
+2014-05-05,140.02
+2014-05-06,140.03
+2014-05-07,139.9
+2014-05-08,140.25
+2014-05-09,140.16
+2014-05-12,140.07
+2014-05-13,140.64
+2014-05-14,141.24
+2014-05-15,141.99
+2014-05-16,141.65
+2014-05-19,141.49
+2014-05-20,141.59
+2014-05-21,141.28
+2014-05-22,141.4
+2014-05-23,141.39
+2014-05-26,141.38
+2014-05-27,141.755
+2014-05-28,142.39
+2014-05-29,142.06
+2014-05-30,142.17
+2014-06-02,142.02
+2014-06-03,141.32
+2014-06-04,141.215
+2014-06-05,141.66
+2014-06-06,142.18
+2014-06-09,141.95
+2014-06-10,141.5
+2014-06-11,141.52
+2014-06-12,141.795
+2014-06-13,141.97
+2014-06-16,142.22
+2014-06-17,142.19
+2014-06-18,142.18
+2014-06-19,142.48
+2014-06-20,142.34
+2014-06-23,142.54
+2014-06-24,142.78
+2014-06-25,143.3
+2014-06-26,143.59
+2014-06-27,143.38
+2014-06-30,143.6
+2014-07-01,143.51
+2014-07-02,143.07
+2014-07-03,142.895
+2014-07-04,143.4
+2014-07-07,143.45
+2014-07-08,143.96
+2014-07-09,144.05
+2014-07-10,144.11
+2014-07-11,144.04
+2014-07-14,144.11
+2014-07-15,144.11
+2014-07-16,144.35
+2014-07-17,144.96
+2014-07-18,144.73
+2014-07-21,144.79
+2014-07-22,144.68
+2014-07-23,144.81
+2014-07-24,144.44
+2014-07-25,144.91
+2014-07-28,144.79
+2014-07-29,145.22
+2014-07-30,144.46
+2014-07-31,144.675
+2014-08-01,
+2014-08-04,
+2014-08-05,
+2014-08-06,145.43
+2014-08-07,145.555
+2014-08-08,146.015
+2014-08-11,145.88
+2014-08-12,145.81
+2014-08-13,146.34
+2014-08-14,146.41
+2014-08-15,146.87
+2014-08-18,146.38
+2014-08-19,146.62
+2014-08-20,146.75
+2014-08-21,146.76
+2014-08-22,146.96
+2014-08-25,147.39
+2014-08-26,147.45
+2014-08-27,147.86
+2014-08-28,148.2
+2014-08-29,148.02
+2014-09-01,148.035
+2014-09-02,147.51
+2014-09-03,147.42
+2014-09-04,147.26
+2014-09-05,147.54
+2014-09-08,147.22
+2014-09-09,146.87
+2014-09-10,146.82
+2014-09-11,146.85
+2014-09-12,146.31
+2014-09-15,146.71
+2014-09-16,146.64
+2014-09-17,146.74
+2014-09-18,146.49
+2014-09-19,147.14
+2014-09-22,147.42
+2014-09-23,147.5
+2014-09-24,147.38
+2014-09-25,147.9
+2014-09-26,147.9
+2014-09-29,148.02
+2014-09-30,148.2
+2014-10-01,148.87
+2014-10-02,148.51
+2014-10-03,148.44
+2014-10-06,148.61
+2014-10-07,148.81
+2014-10-08,148.72
+2014-10-09,148.69
+2014-10-10,149.01
+2014-10-13,148.94
+2014-10-14,149.59
+2014-10-15,150.06
+2014-10-16,150.21
+2014-10-17,149.17
+2014-10-20,149.31
+2014-10-21,148.95
+2014-10-22,149.19
+2014-10-23,148.67
+2014-10-24,148.83
+2014-10-27,149.03
+2014-10-28,148.98
+2014-10-29,148.75
+2014-10-30,149.37
+2014-10-31,149.55
+2014-11-03,149.41
+2014-11-04,149.75
+2014-11-05,149.69
+2014-11-06,149.61
+2014-11-07,149.87
+2014-11-10,149.59
+2014-11-11,149.78
+2014-11-12,149.82
+2014-11-13,150.2
+2014-11-14,150.4
+2014-11-17,150.11
+2014-11-18,150.08
+2014-11-19,149.56
+2014-11-20,150.12
+2014-11-21,150.51
+2014-11-24,150.52
+2014-11-25,150.86
+2014-11-26,151.03
+2014-11-27,151.42
+2014-11-28,151.43
+2014-12-01,150.68
+2014-12-02,150.76
+2014-12-03,150.84
+2014-12-04,150.77
+2014-12-05,150.61
+2014-12-08,151.45
+2014-12-09,151.66
+2014-12-10,151.95
+2014-12-11,151.92
+2014-12-12,152.47
+2014-12-15,152.35
+2014-12-16,152.9
+2014-12-17,152.8
+2014-12-18,152.64
+2014-12-19,152.96
+2014-12-22,152.93
+2014-12-23,152.78
+2014-12-24,
+2014-12-25,
+2014-12-26,
+2014-12-29,153.54
+2014-12-30,153.62
+2014-12-31,
+2015-01-01,
+2015-01-02,154.29
+2015-01-05,154.09
+2015-01-06,154.72
+2015-01-07,154.48
+2015-01-08,154.12
+2015-01-09,154.22
+2015-01-12,154.58
+2015-01-13,154.61
+2015-01-14,155.01
+2015-01-15,155.43
+2015-01-16,155.28
+2015-01-19,155.54
+2015-01-20,155.4
+2015-01-21,154.48
+2015-01-22,155.48
+2015-01-23,156.62
+2015-01-26,156.04
+2015-01-27,156
+2015-01-28,156.98
+2015-01-29,156.33
+2015-01-30,157.26
+2015-02-02,157.16
+2015-02-03,156.49
+2015-02-04,156.53
+2015-02-05,156.39
+2015-02-06,156.22
+2015-02-09,156.49
+2015-02-10,156.29
+2015-02-11,156.54
+2015-02-12,156.94
+2015-02-13,156.62
+2015-02-16,157.09
+2015-02-17,156.24
+2015-02-18,156.48
+2015-02-19,156.29
+2015-02-20,156.13
+2015-02-23,156.72
+2015-02-24,156.72
+2015-02-25,157.35
+2015-02-26,157.49
+2015-02-27,157.355
+2015-03-02,156.82
+2015-03-03,156.54
+2015-03-04,156.51
+2015-03-05,157.01
+2015-03-06,156.25
+2015-03-09,157.34
+2015-03-10,158.47
+2015-03-11,158.56
+2015-03-12,158
+2015-03-13,157.89
+2015-03-16,157.71
+2015-03-17,157.59
+2015-03-18,159.07
+2015-03-19,158.77
+2015-03-20,158.87
+2015-03-23,158.39
+2015-03-24,158.34
+2015-03-25,158.16
+2015-03-26,158.37
+2015-03-27,158.52
+2015-03-30,158.47
+2015-03-31,158.79
+2015-04-01,158.9
+2015-04-02,158.71
+2015-04-03,
+2015-04-06,
+2015-04-07,158.89
+2015-04-08,159.12
+2015-04-09,159.01
+2015-04-10,159.16
+2015-04-13,159.31
+2015-04-14,159.43
+2015-04-15,159.98
+2015-04-16,160.31
+2015-04-17,160.45
+2015-04-20,160.31
+2015-04-21,159.9
+2015-04-22,159.225
diff --git a/syscore/tests/test_fileutils.py b/syscore/tests/test_fileutils.py
new file mode 100644
index 00000000000..a8c7c389b9e
--- /dev/null
+++ b/syscore/tests/test_fileutils.py
@@ -0,0 +1,405 @@
+import pytest
+import sys
+from pathlib import Path
+from syscore.fileutils import (
+ resolve_path_and_filename_for_package,
+ get_resolved_pathname,
+)
+
+
+@pytest.fixture()
+def project_dir(request):
+ module_path = Path(request.module.__file__)
+ return str(module_path.parent.parent.parent.absolute())
+
+
+@pytest.mark.skipif(sys.platform.startswith("win"), reason="Only runs on unix")
+class TestFileUtilsUnix:
+ def test_resolve_path_absolute(self):
+ actual = get_resolved_pathname("/home/rob")
+ assert actual == "/home/rob"
+
+ def test_resolve_path_absolute_trailing(self):
+ actual = get_resolved_pathname("/home/rob/")
+ assert actual == "/home/rob"
+
+ def test_resolve_path_absolute_dotted(self):
+ actual = get_resolved_pathname(".home.rob")
+ assert actual == "/home/rob"
+
+ def test_resolve_path_relative(self, project_dir):
+ actual = get_resolved_pathname("syscore.tests")
+ assert actual == f"{project_dir}/syscore/tests"
+
+ def test_resolve_path_non_existent(self, project_dir):
+ actual = get_resolved_pathname("syscore.testz")
+ assert actual == f"{project_dir}/syscore/testz"
+
+ def test_resolve_dotted_dir_name(self, tmp_path):
+ directory = tmp_path / "dir.name.with.dots"
+ directory.mkdir()
+ file = directory / "hello.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = get_resolved_pathname(file)
+ assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt"
+
+ def test_resolve_dotted_file_name(self, tmp_path):
+ directory = tmp_path / "dir_name"
+ directory.mkdir()
+ file = directory / "dotted.filename.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = get_resolved_pathname(file)
+ assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt"
+
+ def test_resolve_package_separate(self):
+ actual = resolve_path_and_filename_for_package("/home/rob/", "file.csv")
+ assert actual == "/home/rob/file.csv"
+
+ def test_resolve_package_combined(self):
+ actual = resolve_path_and_filename_for_package("/home/rob/file.csv")
+ assert actual == "/home/rob/file.csv"
+
+ def test_resolve_package_combined_dotted(self):
+ actual = resolve_path_and_filename_for_package(".home.rob.file.csv")
+ assert actual == "/home/rob/file.csv"
+
+ def test_resolve_package_module_separate(self, project_dir):
+ actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv")
+ assert actual == f"{project_dir}/syscore/tests/file.csv"
+
+ def test_resolve_package_module_instr_data_module(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "data.futures.csvconfig", "instrumentconfig.csv"
+ )
+ assert actual == f"{project_dir}/data/futures/csvconfig/instrumentconfig.csv"
+
+ def test_resolve_package_module_combined(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syscore.tests.pricetestdata.csv"
+ )
+ assert actual == f"{project_dir}/syscore/tests/pricetestdata.csv"
+
+ @pytest.mark.xfail(reason="Cannot work with old or new implementation")
+ def test_resolve_package_module_combined_dotted_filename(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syscore.tests.price.test.data.csv"
+ )
+ assert actual == f"{project_dir}/syscore/tests/price.test.data.csv"
+
+ def test_resolve_package_module_combined_dotted_filename_sep(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syscore.tests", "price.test.data.csv"
+ )
+ assert actual == f"{project_dir}/syscore/tests/price.test.data.csv"
+
+ @pytest.mark.xfail(reason="Cannot work with old or new implementation")
+ def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name(
+ self, tmp_path
+ ):
+ directory = tmp_path / "dir.name.with.dots"
+ directory.mkdir()
+ file = directory / "hello.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = resolve_path_and_filename_for_package(
+ f"{tmp_path}/dir.name.with.dots", "hello.txt"
+ )
+ assert resolved_path == f"{tmp_path}/dir.name.with.dots/hello.txt"
+
+ @pytest.mark.xfail(reason="Cannot work with old or new implementation")
+ def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name(
+ self, tmp_path
+ ):
+ directory = tmp_path / "dir_name"
+ directory.mkdir()
+ file = directory / "dotted.filename.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = resolve_path_and_filename_for_package(
+ f"{tmp_path}/dir_name/dotted.filename.txt"
+ )
+ assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt"
+
+ def test_resolve_resolve_path_and_filename_for_package_with_separate_dotted_file_name(
+ self, tmp_path
+ ):
+ directory = tmp_path / "dir_name"
+ directory.mkdir()
+ file = directory / "dotted.filename.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = resolve_path_and_filename_for_package(
+ f"{tmp_path}/dir_name/", "dotted.filename.txt"
+ )
+ assert resolved_path == f"{tmp_path}/dir_name/dotted.filename.txt"
+
+ # No Separate filename
+
+ def test_csv_data(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "data.futures.csvconfig.instrumentconfig.csv"
+ )
+ assert actual == f"{project_dir}/data/futures/csvconfig/instrumentconfig.csv"
+
+ def test_logging(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syslogging.logging_prod.yaml",
+ )
+ assert actual == f"{project_dir}/syslogging/logging_prod.yaml"
+
+ def test_config_defaults(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "sysdata.config.defaults.yaml",
+ )
+ assert actual == f"{project_dir}/sysdata/config/defaults.yaml"
+
+ def test_control_config_defaults(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syscontrol.control_config.yaml",
+ )
+ assert actual == f"{project_dir}/syscontrol/control_config.yaml"
+
+ def test_private_config_dots(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "private.private_config.yaml",
+ )
+ assert actual == f"{project_dir}/private/private_config.yaml"
+
+ def test_private_config_slash(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "private/private_config.yaml",
+ )
+ assert actual == f"{project_dir}/private/private_config.yaml"
+
+ def test_strategy_config(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "systems.provided.rob_system.config.yaml",
+ )
+ assert actual == f"{project_dir}/systems/provided/rob_system/config.yaml"
+
+ def test_pickled_backtest(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "private/backtests/fut_strategy_v1_8/20260101_210632_backtest.pck",
+ )
+ assert (
+ actual
+ == f"{project_dir}/private/backtests/fut_strategy_v1_8/20260101_210632_backtest.pck"
+ )
+
+ def test_ib_instrument_config(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "sysbrokers.IB.config.ib_config_futures.csv",
+ )
+ assert actual == f"{project_dir}/sysbrokers/IB/config/ib_config_futures.csv"
+
+ def test_ib_fx_config(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "sysbrokers.IB.config.ib_config_spot_FX.csv",
+ )
+ assert actual == f"{project_dir}/sysbrokers/IB/config/ib_config_spot_FX.csv"
+
+ def test_ib_trading_hours_config(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "sysbrokers.IB.ib_config_trading_hours.yaml",
+ )
+ assert actual == f"{project_dir}/sysbrokers/IB/ib_config_trading_hours.yaml"
+
+ def test_email_store_filename(self):
+ actual = resolve_path_and_filename_for_package(
+ "/home/rob/logs/email_store.log",
+ )
+ assert actual == "/home/rob/logs/email_store.log"
+
+
+@pytest.mark.skipif(sys.platform in ["linux", "darwin"], reason="Only runs on windows")
+class TestFileUtilsWindoze:
+ def test_resolve_path_absolute(self):
+ actual = get_resolved_pathname("C:\\home\\rob\\")
+ assert actual == "C:\\home\\rob"
+
+ def test_resolve_path_absolute_lower_case(self):
+ actual = get_resolved_pathname("c:\\home\\rob\\")
+ assert actual == "c:\\home\\rob"
+
+ def test_resolve_path_absolute_trailing(self):
+ actual = get_resolved_pathname("C:\\home\\rob\\")
+ assert actual == "C:\\home\\rob"
+
+ @pytest.mark.xfail(reason="A Windows absolute path needs a root AND a drive")
+ def test_resolve_path_absolute_dotted(self):
+ actual = get_resolved_pathname(".home.rob")
+ assert actual == "\\home\\rob"
+
+ def test_resolve_path_relative(self, project_dir):
+ actual = get_resolved_pathname("syscore.tests")
+ assert actual == f"{project_dir}\\syscore\\tests"
+
+ def test_resolve_path_non_existent(self, project_dir):
+ actual = get_resolved_pathname("syscore.testz")
+ assert actual == f"{project_dir}\\syscore\\testz"
+
+ def test_resolve_dotted_dir_name(self, tmp_path):
+ directory = tmp_path / "dir.name.with.dots"
+ directory.mkdir()
+ file = directory / "hello.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = get_resolved_pathname(file)
+ assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt"
+
+ def test_resolve_dotted_file_name(self, tmp_path):
+ directory = tmp_path / "dir_name"
+ directory.mkdir()
+ file = directory / "dotted.filename.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = get_resolved_pathname(file)
+ assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt"
+
+ def test_resolve_package_separate(self):
+ actual = resolve_path_and_filename_for_package("C:\\home\\rob\\", "file.csv")
+ assert actual == "C:\\home\\rob\\file.csv"
+
+ def test_resolve_package_combined(self):
+ actual = resolve_path_and_filename_for_package("C:\\home\\rob\\file.csv")
+ assert actual == "C:\\home\\rob\\file.csv"
+
+ @pytest.mark.xfail(reason="A Windows absolute path needs a root AND a drive")
+ def test_resolve_package_combined_dotted(self):
+ actual = resolve_path_and_filename_for_package(".home.rob.file.csv")
+ assert actual == "\\home\\rob\\file.csv"
+
+ def test_resolve_package_module_separate(self, project_dir):
+ actual = resolve_path_and_filename_for_package("syscore.tests", "file.csv")
+ assert actual == f"{project_dir}\\syscore\\tests\\file.csv"
+
+ def test_resolve_package_module_combined(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syscore.tests.pricetestdata.csv"
+ )
+ assert actual == f"{project_dir}\\syscore\\tests\\pricetestdata.csv"
+
+ @pytest.mark.xfail(reason="Cannot work with old or new implementation")
+ def test_resolve_package_module_combined_dotted_filename(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syscore.tests.price.test.data.csv"
+ )
+ assert actual == f"{project_dir}\\syscore\\tests\\price.test.data.csv"
+
+ def test_resolve_package_module_combined_dotted_filename_sep(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syscore.tests", "price.test.data.csv"
+ )
+ assert actual == f"{project_dir}\\syscore\\tests\\price.test.data.csv"
+
+ @pytest.mark.xfail(reason="Cannot work with old or new implementation")
+ def test_resolve_resolve_path_and_filename_for_package_with_dotted_dir_name(
+ self, tmp_path
+ ):
+ directory = tmp_path / "dir.name.with.dots"
+ directory.mkdir()
+ file = directory / "hello.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = resolve_path_and_filename_for_package(
+ f"{tmp_path}\\dir.name.with.dots", "hello.txt"
+ )
+ assert resolved_path == f"{tmp_path}\\dir.name.with.dots\\hello.txt"
+
+ @pytest.mark.xfail(reason="Cannot work with old or new implementation")
+ def test_resolve_resolve_path_and_filename_for_package_with_dotted_file_name(
+ self, tmp_path
+ ):
+ directory = tmp_path / "dir_name"
+ directory.mkdir()
+ file = directory / "dotted.filename.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = resolve_path_and_filename_for_package(
+ f"{tmp_path}\\dir_name\\dotted.filename.txt"
+ )
+ assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt"
+
+ def test_resolve_resolve_path_and_filename_for_package_with_separate_dotted_file_name(
+ self, tmp_path
+ ):
+ directory = tmp_path / "dir_name"
+ directory.mkdir()
+ file = directory / "dotted.filename.txt"
+ file.write_text("content", encoding="utf-8")
+ resolved_path = resolve_path_and_filename_for_package(
+ f"{tmp_path}\\dir_name\\", "dotted.filename.txt"
+ )
+ assert resolved_path == f"{tmp_path}\\dir_name\\dotted.filename.txt"
+
+ # No Separate filename
+
+ def test_csv_data(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "data.futures.csvconfig.instrumentconfig.csv"
+ )
+ assert (
+ actual == f"{project_dir}\\data\\futures\\csvconfig\\instrumentconfig.csv"
+ )
+
+ def test_logging(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syslogging.logging_prod.yaml",
+ )
+ assert actual == f"{project_dir}\\syslogging\\logging_prod.yaml"
+
+ def test_config_defaults(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "sysdata.config.defaults.yaml",
+ )
+ assert actual == f"{project_dir}\\sysdata\\config\\defaults.yaml"
+
+ def test_control_config_defaults(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "syscontrol.control_config.yaml",
+ )
+ assert actual == f"{project_dir}\\syscontrol\\control_config.yaml"
+
+ def test_private_config_dots(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "private.private_config.yaml",
+ )
+ assert actual == f"{project_dir}\\private\\private_config.yaml"
+
+ def test_private_config_slash(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "private/private_config.yaml",
+ )
+ assert actual == f"{project_dir}\\private\\private_config.yaml"
+
+ def test_strategy_config(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "systems.provided.rob_system.config.yaml",
+ )
+ assert actual == f"{project_dir}\\systems\\provided\\rob_system\\config.yaml"
+
+ def test_pickled_backtest(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "private/backtests/fut_strategy_v1_8/20260101_210632_backtest.pck",
+ )
+ assert (
+ actual
+ == f"{project_dir}\\private\\backtests\\fut_strategy_v1_8\\20260101_210632_backtest.pck"
+ )
+
+ def test_ib_instrument_config(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "sysbrokers.IB.config.ib_config_futures.csv",
+ )
+ assert actual == f"{project_dir}\\sysbrokers\\IB\\config\\ib_config_futures.csv"
+
+ def test_ib_fx_config(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "sysbrokers.IB.config.ib_config_spot_FX.csv",
+ )
+ assert actual == f"{project_dir}\\sysbrokers\\IB\\config\\ib_config_spot_FX.csv"
+
+ def test_ib_trading_hours_config(self, project_dir):
+ actual = resolve_path_and_filename_for_package(
+ "sysbrokers.IB.ib_config_trading_hours.yaml",
+ )
+ assert actual == f"{project_dir}\\sysbrokers\\IB\\ib_config_trading_hours.yaml"
+
+ def test_email_store_filename(self):
+ actual = resolve_path_and_filename_for_package(
+ "C:\\home\\rob\\logs\\email_store.log",
+ )
+ assert actual == "C:\\home\\rob\\logs\\email_store.log"
diff --git a/sysdata/config/configdata.py b/sysdata/config/configdata.py
index e3c7b6718d9..a07004b5d66 100644
--- a/sysdata/config/configdata.py
+++ b/sysdata/config/configdata.py
@@ -12,9 +12,6 @@
"""
-from pathlib import Path
-import os
-
import yaml
from syscore.exceptions import missingData
@@ -23,12 +20,8 @@
from sysdata.config.defaults import get_system_defaults_dict
from sysdata.config.private_config import (
get_private_config_as_dict,
- PRIVATE_CONFIG_FILE,
-)
-from sysdata.config.private_directory import (
- get_full_path_for_private_config,
- PRIVATE_CONFIG_DIR_ENV_VAR,
)
+
from syslogging.logger import *
from sysdata.config.fill_config_dict_with_defaults import fill_config_dict_with_defaults
@@ -58,7 +51,7 @@ def __init__(
multiple elements, latter elements will overwrite
earlier ones)
- :type config_object: str or dict
+ :type config_object: str or dict or list
:returns: new Config object
@@ -180,11 +173,10 @@ def _create_config_from_dict(self, config_object):
self.add_elements(attr_names)
- def system_init(self, base_system):
+ def system_init(self):
"""
This is run when added to a base system
- :param base_system
:return: nothing
"""
@@ -287,12 +279,7 @@ def default_config(cls):
if hasattr(cls, "evaluated"):
return cls.evaluated
- if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR):
- config = Config(
- private_filename=get_full_path_for_private_config(PRIVATE_CONFIG_FILE)
- )
- else:
- config = Config()
+ config = Config(get_private_config_as_dict())
config.fill_with_defaults()
cls.evaluated = config
diff --git a/sysdata/config/control_config.py b/sysdata/config/control_config.py
index 72d1f6c150f..ae76c9171b6 100644
--- a/sysdata/config/control_config.py
+++ b/sysdata/config/control_config.py
@@ -1,5 +1,6 @@
from sysdata.config.configdata import Config
-from sysdata.config.private_directory import get_full_path_for_private_config
+from syscore.fileutils import resolve_path_and_filename_for_package
+from sysdata.config.private_config import get_private_config_dir
from yaml.parser import ParserError
PRIVATE_CONTROL_CONFIG_FILE = "private_control_config.yaml"
@@ -7,12 +8,18 @@
def get_control_config() -> Config:
- private_control_path = get_full_path_for_private_config(PRIVATE_CONTROL_CONFIG_FILE)
+ dir = get_private_config_dir()
+ private_control_path = resolve_path_and_filename_for_package(
+ dir, PRIVATE_CONTROL_CONFIG_FILE
+ )
+ default_control_path = resolve_path_and_filename_for_package(
+ DEFAULT_CONTROL_CONFIG_FILE
+ )
try:
control_config = Config(
private_filename=private_control_path,
- default_filename=DEFAULT_CONTROL_CONFIG_FILE,
+ default_filename=default_control_path,
)
control_config.fill_with_defaults()
@@ -21,7 +28,7 @@ def get_control_config() -> Config:
except FileNotFoundError:
raise Exception(
"Need to have either %s or %s or both present:"
- % (str(DEFAULT_CONTROL_CONFIG_FILE), str(private_control_path))
+ % (private_control_path, default_control_path)
)
except BaseException as be:
raise Exception("Problem reading control config: %s" % str(be))
diff --git a/sysdata/config/private_config.py b/sysdata/config/private_config.py
index 791e592f128..d576e545857 100644
--- a/sysdata/config/private_config.py
+++ b/sysdata/config/private_config.py
@@ -1,25 +1,38 @@
-from syscore.fileutils import resolve_path_and_filename_for_package, does_filename_exist
+import os
+import yaml
+from pathlib import Path
+
+from syscore.fileutils import resolve_path_and_filename_for_package
from syscore.constants import arg_not_supplied
-from sysdata.config.private_directory import get_full_path_for_private_config
-import yaml
+DEFAULT_PRIVATE_DIR = "private"
PRIVATE_CONFIG_FILE = "private_config.yaml"
+PRIVATE_CONFIG_DIR_ENV_VAR = "PYSYS_PRIVATE_CONFIG_DIR"
def get_private_config_as_dict(filename: str = arg_not_supplied) -> dict:
+ private_dir = get_private_config_dir()
if filename is arg_not_supplied:
- filename = get_full_path_for_private_config(PRIVATE_CONFIG_FILE)
- if not does_filename_exist(filename):
+ filename = PRIVATE_CONFIG_FILE
+ try:
+ private_path = resolve_path_and_filename_for_package(private_dir, filename)
+ with open(private_path) as file_to_parse:
+ private_dict = yaml.load(file_to_parse, Loader=yaml.FullLoader)
+ return private_dict
+
+ except Exception:
print(
- "Private configuration %s does not exist; no problem if running in sim mode"
- % filename
+ f"Private configuration '{private_path}' is missing or "
+ f"misconfigured; no problem if running in sim mode"
)
-
return {}
- private_file = resolve_path_and_filename_for_package(filename)
- with open(private_file) as file_to_parse:
- private_dict = yaml.load(file_to_parse, Loader=yaml.FullLoader)
- return private_dict
+def get_private_config_dir():
+ if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR):
+ private_config_dir = Path(os.environ[PRIVATE_CONFIG_DIR_ENV_VAR])
+ else:
+ private_config_dir = Path(DEFAULT_PRIVATE_DIR)
+
+ return str(private_config_dir)
diff --git a/sysdata/config/private_directory.py b/sysdata/config/private_directory.py
deleted file mode 100644
index a844f341d45..00000000000
--- a/sysdata/config/private_directory.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import os
-
-DEFAULT_PRIVATE_DIR = "private"
-PRIVATE_CONFIG_DIR_ENV_VAR = "PYSYS_PRIVATE_CONFIG_DIR"
-
-
-def get_full_path_for_private_config(filename: str):
- ## FIXME: should use os path join instead of '/'?
-
- if os.getenv(PRIVATE_CONFIG_DIR_ENV_VAR):
- private_config_path = f"{os.environ[PRIVATE_CONFIG_DIR_ENV_VAR]}/{filename}"
- else:
- private_config_path = f"{DEFAULT_PRIVATE_DIR}/{filename}"
-
- return private_config_path
diff --git a/sysdata/parquet/parquet_access.py b/sysdata/parquet/parquet_access.py
index 990d7d34f9f..48b7123061e 100644
--- a/sysdata/parquet/parquet_access.py
+++ b/sysdata/parquet/parquet_access.py
@@ -1,4 +1,4 @@
-import os
+from pathlib import Path
import pandas as pd
from syscore.exceptions import missingFile
from syscore.fileutils import (
@@ -19,13 +19,13 @@ def get_all_identifiers_with_data_type(self, data_type: str):
path = self._get_pathname_given_data_type(data_type)
return files_with_extension_in_pathname(path, extension=EXTENSION)
- def does_idenitifier_with_data_type_exist(
+ def does_identifier_with_data_type_exist(
self, data_type: str, identifier: str
) -> bool:
filename = self._get_filename_given_data_type_and_identifier(
data_type=data_type, identifier=identifier
)
- return os.path.isfile(filename)
+ return Path(filename).exists()
def delete_data_given_data_type_and_identifier(
self, data_type: str, identifier: str
@@ -34,7 +34,7 @@ def delete_data_given_data_type_and_identifier(
data_type=data_type, identifier=identifier
)
try:
- os.remove(filename)
+ Path(filename).unlink()
except FileNotFoundError:
raise missingFile(f"File '{filename}' does not exist")
@@ -66,7 +66,6 @@ def _get_filename_given_data_type_and_identifier(
def _get_pathname_given_data_type(self, data_type: str):
root = self.parquet_store
- path = os.path.join(root, data_type)
- Path(path).mkdir(parents=True, exist_ok=True)
-
- return path
+ path = Path(root, data_type)
+ path.mkdir(parents=True, exist_ok=True)
+ return str(path)
diff --git a/sysdata/parquet/parquet_futures_per_contract_prices.py b/sysdata/parquet/parquet_futures_per_contract_prices.py
index 6fa5566e768..01c595f5d01 100644
--- a/sysdata/parquet/parquet_futures_per_contract_prices.py
+++ b/sysdata/parquet/parquet_futures_per_contract_prices.py
@@ -148,7 +148,7 @@ def has_price_data_for_contract_at_frequency(
self, contract_object: futuresContract, frequency: Frequency
) -> bool:
ident = from_contract_and_freq_to_key(contract_object, frequency=frequency)
- return self.parquet.does_idenitifier_with_data_type_exist(
+ return self.parquet.does_identifier_with_data_type_exist(
data_type=CONTRACT_COLLECTION, identifier=ident
)
diff --git a/sysdata/tests/custom_private_config/private_config_trading_hours.yaml b/sysdata/tests/custom_private_config/private_config_trading_hours.yaml
new file mode 100644
index 00000000000..cd381be1e36
--- /dev/null
+++ b/sysdata/tests/custom_private_config/private_config_trading_hours.yaml
@@ -0,0 +1,3 @@
+MET:
+ - '09:00'
+ - '14:00'
diff --git a/sysdata/tests/custom_private_config/private_control_config.yaml b/sysdata/tests/custom_private_config/private_control_config.yaml
index 836a43e9c42..234a7e95a6f 100644
--- a/sysdata/tests/custom_private_config/private_control_config.yaml
+++ b/sysdata/tests/custom_private_config/private_control_config.yaml
@@ -1,2 +1,2 @@
process_configuration_start_time:
- run_stack_handler: '01:00'
\ No newline at end of file
+ run_stack_handler: '01:00'
diff --git a/sysdata/tests/test_config.py b/sysdata/tests/test_config.py
index 20353713927..87c1bf01a31 100644
--- a/sysdata/tests/test_config.py
+++ b/sysdata/tests/test_config.py
@@ -1,10 +1,36 @@
+import datetime
from sysdata.config.configdata import Config
from sysdata.config.control_config import get_control_config
-from sysdata.config.private_directory import PRIVATE_CONFIG_DIR_ENV_VAR
+from sysdata.config.private_config import PRIVATE_CONFIG_DIR_ENV_VAR
+from sysbrokers.IB.ib_trading_hours import get_saved_trading_hours
class TestConfig:
- def test_default(self):
+ def test_init_dict(self, monkeypatch):
+ monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False)
+ config = Config(dict(parameters=dict(p1=3, p2=4.6), another_thing="foo"))
+ assert config.as_dict()["parameters"]["p2"] == 4.6
+ assert config.as_dict()["another_thing"] == "foo"
+
+ def test_init_str(self, monkeypatch):
+ monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False)
+ config = Config("systems.provided.example.exampleconfig.yaml")
+ assert config.as_dict()["forecast_cap"] == 21.0
+
+ def test_init_list(self, monkeypatch):
+ monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False)
+ config = Config(
+ [
+ "systems.provided.example.exampleconfig.yaml",
+ dict(parameters=dict(p1=3, p2=4.6), another_thing="foo"),
+ ]
+ )
+ assert config.as_dict()["forecast_cap"] == 21.0
+ assert config.as_dict()["parameters"]["p2"] == 4.6
+ assert config.as_dict()["another_thing"] == "foo"
+
+ def test_default(self, monkeypatch):
+ monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False)
Config.reset()
config = Config.default_config()
assert config.get_element("ib_idoffset") == 100
@@ -25,7 +51,9 @@ def test_bad_custom_dir(self, monkeypatch):
config = Config.default_config()
assert config.get_element("ib_idoffset") == 100
- def test_default_control(self):
+ def test_default_control(self, monkeypatch):
+ monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False)
+ Config.reset()
config = get_control_config()
assert (
config.as_dict()["process_configuration_start_time"]["run_stack_handler"]
@@ -51,3 +79,21 @@ def test_control_bad_custom_dir(self, monkeypatch):
config.as_dict()["process_configuration_start_time"]["run_stack_handler"]
== "00:01"
)
+
+ def test_trading_hours_default(self, monkeypatch):
+ monkeypatch.delenv(PRIVATE_CONFIG_DIR_ENV_VAR, raising=False)
+ config = get_saved_trading_hours()
+ assert config["MET"]["Monday"][0].closing_time == datetime.time(15)
+
+ def test_trading_hours_custom(self, monkeypatch):
+ monkeypatch.setenv(
+ PRIVATE_CONFIG_DIR_ENV_VAR, "sysdata.tests.custom_private_config"
+ )
+ config = get_saved_trading_hours()
+ assert config["MET"]["Monday"][0].opening_time == datetime.time(9)
+ assert config["MET"]["Monday"][0].closing_time == datetime.time(14)
+
+ def test_trading_hours_bad_custom_dir(self, monkeypatch):
+ monkeypatch.setenv(PRIVATE_CONFIG_DIR_ENV_VAR, "sysdata.tests")
+ config = get_saved_trading_hours()
+ assert config["MET"]["Monday"][0].closing_time == datetime.time(15)
diff --git a/sysinit/futures/barchart_futures_contract_prices.py b/sysinit/futures/barchart_futures_contract_prices.py
index 4147f7f2f6a..62c4a4cc0e3 100644
--- a/sysinit/futures/barchart_futures_contract_prices.py
+++ b/sysinit/futures/barchart_futures_contract_prices.py
@@ -1,5 +1,5 @@
from sysdata.csv.csv_futures_contract_prices import ConfigCsvFuturesPrices
-import os
+from pathlib import Path
from syscore.fileutils import (
get_resolved_pathname,
files_with_extension_in_resolved_pathname,
@@ -30,11 +30,11 @@ def strip_file_names(pathname):
datecode = str(year) + "{0:02d}".format(month)
new_file_name = "%s_%s00.csv" % (instrument, datecode)
- new_full_name = os.path.join(resolved_pathname, new_file_name)
- old_full_name = os.path.join(resolved_pathname, filename + ".csv")
+ new_full_name = Path(resolved_pathname, new_file_name)
+ old_full_name = Path(resolved_pathname, filename + ".csv")
print("Rename %s to\n %s" % (old_full_name, new_full_name))
- os.rename(old_full_name, new_full_name)
+ old_full_name.rename(new_full_name)
return None
diff --git a/syslogging/logger.py b/syslogging/logger.py
index f51c4f74eb6..df6d9321bd0 100644
--- a/syslogging/logger.py
+++ b/syslogging/logger.py
@@ -1,3 +1,4 @@
+from pathlib import Path
import os
import sys
import socket
@@ -99,7 +100,7 @@ def _configure_sim():
def _configure_prod(logging_config_file):
print(f"Attempting to configure prod logging from {logging_config_file}")
config_path = resolve_path_and_filename_for_package(logging_config_file)
- if os.path.exists(config_path):
+ if Path(config_path).exists():
try:
config = parse_config(path=config_path)
host, port = _get_log_server_config(config)
diff --git a/sysproduction/backup_db_to_csv.py b/sysproduction/backup_db_to_csv.py
index 0ff4e8e694f..7b2e233d1e8 100644
--- a/sysproduction/backup_db_to_csv.py
+++ b/sysproduction/backup_db_to_csv.py
@@ -1,3 +1,4 @@
+from pathlib import Path
import os
import pandas as pd
@@ -98,10 +99,9 @@ def get_data_and_create_csv_directories(logname):
)
for class_name, path in class_paths.items():
- dir_name = os.path.join(csv_dump_dir, path)
- class_paths[class_name] = dir_name
- if not os.path.exists(dir_name):
- os.makedirs(dir_name)
+ dir_name = Path(csv_dump_dir, path)
+ class_paths[class_name] = str(dir_name)
+ Path(dir_name).mkdir(exist_ok=True)
data = dataBlob(csv_data_paths=class_paths, log_name=logname)
diff --git a/sysproduction/data/backtest.py b/sysproduction/data/backtest.py
index 105d440d292..ab3361f5241 100644
--- a/sysproduction/data/backtest.py
+++ b/sysproduction/data/backtest.py
@@ -1,5 +1,5 @@
from copy import copy
-import os
+from pathlib import Path
from shutil import copyfile
from syscore.dateutils import create_datetime_marker_string
@@ -218,10 +218,7 @@ def store_backtest_state(data, system, strategy_name="default_strategy"):
def ensure_backtest_directory_exists(strategy_name):
full_directory = get_backtest_directory_for_strategy(strategy_name)
- try:
- os.makedirs(full_directory)
- except FileExistsError:
- pass
+ Path(full_directory).mkdir(exist_ok=True)
def rchop(s, suffix):
@@ -262,9 +259,9 @@ def get_backtest_config_filename(strategy_name, datetime_marker):
def get_backtest_filename_prefix(strategy_name, datetime_marker):
# eg '/home/rob/data/backtests/medium_speed_TF_carry/20200622_102913'
full_directory = get_backtest_directory_for_strategy(strategy_name)
- full_filename_prefix = os.path.join(full_directory, datetime_marker)
+ full_filename_prefix = Path(full_directory, datetime_marker)
- return full_filename_prefix
+ return str(full_filename_prefix)
def get_backtest_directory_for_strategy(strategy_name):
@@ -272,9 +269,9 @@ def get_backtest_directory_for_strategy(strategy_name):
directory_store_backtests = get_directory_store_backtests()
directory_store_backtests = get_resolved_pathname(directory_store_backtests)
- full_directory = os.path.join(directory_store_backtests, strategy_name)
+ full_directory = Path(directory_store_backtests, strategy_name)
- return full_directory
+ return str(full_directory)
def get_directory_store_backtests():
diff --git a/sysproduction/interactive_update_capital_manual.py b/sysproduction/interactive_update_capital_manual.py
index ecd08053ca8..50553414188 100644
--- a/sysproduction/interactive_update_capital_manual.py
+++ b/sysproduction/interactive_update_capital_manual.py
@@ -266,3 +266,7 @@ def delete_all_capital(data: dataBlob):
)
else:
print("OK you decided not to do it")
+
+
+if __name__ == "__main__":
+ interactive_update_capital_manual()
diff --git a/sysproduction/reporting/reporting_functions.py b/sysproduction/reporting/reporting_functions.py
index 82607834ff8..4410e78e915 100644
--- a/sysproduction/reporting/reporting_functions.py
+++ b/sysproduction/reporting/reporting_functions.py
@@ -3,6 +3,7 @@
from PyPDF2 import PdfMerger
import datetime
import pandas as pd
+from pathlib import Path
import os
import shutil
import matplotlib.pyplot as plt
@@ -274,14 +275,14 @@ def output_file_report(
data.log.debug("Written report to %s" % full_filename)
-def resolve_report_filename(report_config, data: dataBlob):
+def resolve_report_filename(report_config, data: dataBlob) -> str:
filename_with_spaces = report_config.title
filename = filename_with_spaces.replace(" ", "_")
use_directory = get_directory_for_reporting(data)
use_directory_resolved = get_resolved_pathname(use_directory)
- full_filename = os.path.join(use_directory_resolved, filename)
+ full_filename = Path(use_directory_resolved, filename)
- return full_filename
+ return str(full_filename)
def get_directory_for_reporting(data):
@@ -337,6 +338,6 @@ def _generate_temp_pdf_filename(
TEMPFILE_PATTERN,
str(datetime_to_long(datetime.datetime.now())),
)
- full_filename = os.path.join(use_directory_resolved, filename)
+ full_filename = Path(use_directory_resolved, filename)
- return full_filename
+ return str(full_filename)
diff --git a/sysproduction/update_strategy_orders.py b/sysproduction/update_strategy_orders.py
index 5a53e03b1b6..302793fd626 100644
--- a/sysproduction/update_strategy_orders.py
+++ b/sysproduction/update_strategy_orders.py
@@ -28,3 +28,7 @@ def update_strategy_orders():
data, strategy_name, process_name, name_of_main_generator_method
)
strategy_order_generator.run_strategy_method()
+
+
+if __name__ == "__main__":
+ update_strategy_orders()
diff --git a/sysproduction/update_system_backtests.py b/sysproduction/update_system_backtests.py
index 32b646e7f18..0bd5dfe112c 100644
--- a/sysproduction/update_system_backtests.py
+++ b/sysproduction/update_system_backtests.py
@@ -27,3 +27,7 @@ def update_system_backtests():
data, strategy_name, process_name, backtest_function
)
system_backtest_runner.run_strategy_method()
+
+
+if __name__ == "__main__":
+ update_system_backtests()
diff --git a/systems/basesystem.py b/systems/basesystem.py
index 4d4d4487291..b66679cfdd0 100644
--- a/systems/basesystem.py
+++ b/systems/basesystem.py
@@ -77,7 +77,7 @@ def __init__(
self._config = config
self._log = log
- self.config.system_init(self)
+ self.config.system_init()
self.data.system_init(self)
self._setup_stages(stage_list)
self._cache = systemCache(self)
diff --git a/systems/tests/test_position_sizing.py b/systems/tests/test_position_sizing.py
index f3e4fec0f70..c52c3f9b24d 100644
--- a/systems/tests/test_position_sizing.py
+++ b/systems/tests/test_position_sizing.py
@@ -1,6 +1,6 @@
import unittest
from _pytest.monkeypatch import MonkeyPatch
-from sysdata.config.private_directory import PRIVATE_CONFIG_DIR_ENV_VAR
+from sysdata.config.private_config import PRIVATE_CONFIG_DIR_ENV_VAR
from systems.tests.testdata import get_test_object_futures_with_comb_forecasts
from systems.basesystem import System
from systems.positionsizing import PositionSizing