Add Single File Pandas Loading

README.md

@@ -5,3 +5,14 @@ NUFT's data fetching from internal servers solution
 
 
 Only supporting Python 3.12 for now. Heavily in progress, more versions and whatnot soon.
+
+
+## Usage
+Lens provides a simple data fetching API. All queries require `startup` to be called on the instance before fetching.
+
+
+```python
+ln = Lens()
+ln.startup()
+df = ln.load_trades_for_day("avax", "2025-02-21")
+```

lens/core.py

@@ -1,26 +1,69 @@
 from pathlib import Path
 
+import pandas as pd
 from loguru import logger
 
+from lens.descriptor import DataDescriptor, load_data_descriptor
 from lens.util.hostname import check_valid_lens_hostname, get_hostname
 from lens.util.paths import check_file_structure_correct, check_metadata_exists, check_metadata_version_match
 
 
 class Lens:
     def __init__(self) -> None:
-        pass
+        self.started = False
+        self.descriptor: DataDescriptor
 
     def startup(self, data_root: Path) -> None:
+        self._startup_check_hostname()
+        self._startup_check_metadata(data_root)
+        self._startup_check_files(data_root)
+        logger.info("Lens startup succeeded.")
+        self.started = True
+        self.descriptor = load_data_descriptor(data_root)
+
+    def load_trades_for_day(self, coin: str, date: str) -> pd.DataFrame:
+        self._ensure_started()
+        if not self._check_coin_available(coin):
+            raise RuntimeError("Coin is not available in this dataset.")
+        for file in self.descriptor.data_paths[coin].trade_paths:
+            if date + ".csv" == file.name:
+                return pd.read_csv(file, sep="|")
+        raise RuntimeError("Failed to find requested date.")
+
+    def load_depth_for_day(self, coin: str, date: str) -> pd.DataFrame:
+        self._ensure_started()
+        if not self._check_coin_available(coin):
+            raise RuntimeError("Coin is not available in this dataset.")
+        for file in self.descriptor.data_paths[coin].depth_paths:
+            if date + ".csv" == file.name:
+                return pd.read_csv(file, sep="|")
+        raise RuntimeError("Failed to find requested date.")
+
+    def _check_coin_available(self, coin: str) -> bool:
+        return coin in self.descriptor.data_paths.keys()
+
+    def _ensure_started(self) -> None:
+        if not self.started:
+            raise RuntimeError("This operation requires lens to be started before calling.")
+
+    def _startup_check_hostname(self) -> None:
         if not check_valid_lens_hostname(get_hostname()):
             raise RuntimeError("Cannot start lens on invalid host.")
         logger.info("Host check succeeded.")
+
+    def _startup_check_metadata(self, data_root: Path) -> None:
         if not check_metadata_exists(data_root):
             raise RuntimeError("Failed to find metadata.json file in the data directory.")
         logger.info("Metadata found.")
         if not check_metadata_version_match(data_root):
             raise RuntimeError("Metadata version does not match current version!")
         logger.info("Metadata versions match.")
+
+    def _startup_check_files(self, data_root: Path) -> None:
         if not check_file_structure_correct(data_root):
             raise RuntimeError("File structure does not match expected directory structure.")
         logger.info("File structure is correct.")
-        logger.info("Lens startup succeeded.")
+
+    def _force_startup_unsafe(self, data_root: Path) -> None:
+        self.started = True
+        self.descriptor = load_data_descriptor(data_root)

lens/test/common.py

@@ -0,0 +1,3 @@
+from pathlib import Path
+
+TEST_DATA_PATH = Path(__file__).parent / "testing_data"

lens/test/test_core.py

@@ -0,0 +1,55 @@
+import pytest
+
+from lens.core import Lens
+from lens.test.common import TEST_DATA_PATH
+
+
+def test_core_startup_metadata() -> None:
+    ln = Lens()
+    ln._startup_check_metadata(TEST_DATA_PATH)
+
+
+def test_core_startup_file_structure() -> None:
+    ln = Lens()
+    ln._startup_check_files(TEST_DATA_PATH)
+
+
+def test_core_startup_trade_load() -> None:
+    ln = Lens()
+    ln._force_startup_unsafe(TEST_DATA_PATH)
+    df = ln.load_trades_for_day("avax", "2025-02-21")
+    assert df.shape[1] == 5
+
+
+def test_core_startup_trade_load_invalid_symbol() -> None:
+    ln = Lens()
+    ln._force_startup_unsafe(TEST_DATA_PATH)
+    with pytest.raises(RuntimeError):
+        ln.load_trades_for_day("avaxnop", "2025-02-21")
+
+
+def test_core_startup_trade_load_nonexistent_date() -> None:
+    ln = Lens()
+    ln._force_startup_unsafe(TEST_DATA_PATH)
+    with pytest.raises(RuntimeError):
+        ln.load_trades_for_day("avax", "2025-02-28")
+
+
+def test_core_startup_depth() -> None:
+    ln = Lens()
+    ln._force_startup_unsafe(TEST_DATA_PATH)
+    ln.load_depth_for_day("avax", "2025-02-21")
+
+
+def test_core_startup_depth_load_invalid_symbol() -> None:
+    ln = Lens()
+    ln._force_startup_unsafe(TEST_DATA_PATH)
+    with pytest.raises(RuntimeError):
+        ln.load_trades_for_day("avaxnop", "2025-02-21")
+
+
+def test_core_startup_depth_load_nonexistent_date() -> None:
+    ln = Lens()
+    ln._force_startup_unsafe(TEST_DATA_PATH)
+    with pytest.raises(RuntimeError):
+        ln.load_trades_for_day("avax", "2025-02-28")

lens/test/testing_data/avax/depth/2025-02-21.csv

@@ -0,0 +1,4 @@
+pair|time|bid_0_price|bid_0_size|bid_1_price|bid_1_size|bid_2_price|bid_2_size|bid_3_price|bid_3_size|bid_4_price|bid_4_size|bid_5_price|bid_5_size|bid_6_price|bid_6_size|bid_7_price|bid_7_size|bid_8_price|bid_8_size|bid_9_price|bid_9_size|bid_10_price|bid_10_size|bid_11_price|bid_11_size|bid_12_price|bid_12_size|bid_13_price|bid_13_size|bid_14_price|bid_14_size|bid_15_price|bid_15_size|bid_16_price|bid_16_size|bid_17_price|bid_17_size|bid_18_price|bid_18_size|bid_19_price|bid_19_size|bid_20_price|bid_20_size|bid_21_price|bid_21_size|bid_22_price|bid_22_size|bid_23_price|bid_23_size|bid_24_price|bid_24_size|ask_0_price|ask_0_size|ask_1_price|ask_1_size|ask_2_price|ask_2_size|ask_3_price|ask_3_size|ask_4_price|ask_4_size|ask_5_price|ask_5_size|ask_6_price|ask_6_size|ask_7_price|ask_7_size|ask_8_price|ask_8_size|ask_9_price|ask_9_size|ask_10_price|ask_10_size|ask_11_price|ask_11_size|ask_12_price|ask_12_size|ask_13_price|ask_13_size|ask_14_price|ask_14_size|ask_15_price|ask_15_size|ask_16_price|ask_16_size|ask_17_price|ask_17_size|ask_18_price|ask_18_size|ask_19_price|ask_19_size|ask_20_price|ask_20_size|ask_21_price|ask_21_size|ask_22_price|ask_22_size|ask_23_price|ask_23_size|ask_24_price|ask_24_size
+AVAXUSDT|1740096000044000|25.141|14933.7540|25.14|77732.8800|25.139|6636.6960|25.138|5630.9120|25.137|3041.5770|25.136|5152.8800|25.135|5956.9950|25.134|6936.9840|25.133|7565.0330|25.132|11309.4000|25.131|4850.2830|25.13|6659.4500|25.129|6332.5080|25.128|15277.8240|25.127|16634.0740|25.126|7990.0680|25.125|5628.0000|25.124|3718.3520|25.123|3291.1130|25.122|6983.9160|25.121|7209.7270|25.12|7008.4800|25.119|7460.3430|25.118|5551.0780|25.117|3039.1570|25.142|1131.3900|25.143|427.4310|25.144|477.7360|25.145|2313.3400|25.146|678.9420|25.147|678.9690|25.148|2967.4640|25.149|4728.0120|25.15|6564.1500|25.151|7972.8670|25.152|5432.8320|25.153|5558.8130|25.154|5483.5720|25.155|9533.7450|25.156|10490.0520|25.157|10238.8990|25.158|5585.0760|25.159|6214.2730|25.16|14492.1600|25.161|3346.4130|25.162|12304.2180|25.163|3321.5160|25.164|11197.9800|25.165|3271.4500|25.166|8355.1120
+AVAXUSDT|1740096004445000|25.141|10307.8100|25.14|76274.7600|25.139|5631.1360|25.138|3418.7680|25.137|3041.5770|25.136|5152.8800|25.135|5956.9950|25.134|6936.9840|25.133|7565.0330|25.132|11309.4000|25.131|4850.2830|25.13|6609.1900|25.129|6332.5080|25.128|15277.8240|25.127|16634.0740|25.126|7990.0680|25.125|5628.0000|25.124|3718.3520|25.123|3291.1130|25.122|6983.9160|25.121|7209.7270|25.12|7008.4800|25.119|7460.3430|25.118|5551.0780|25.117|3039.1570|25.142|2413.6320|25.143|427.4310|25.144|477.7360|25.145|2313.3400|25.146|704.0880|25.147|4752.7830|25.148|2967.4640|25.149|4728.0120|25.15|7695.9000|25.151|7972.8670|25.152|5432.8320|25.153|6087.0260|25.154|7017.9660|25.155|9533.7450|25.156|10490.0520|25.157|10238.8990|25.158|5585.0760|25.159|6214.2730|25.16|14492.1600|25.161|3346.4130|25.162|12304.2180|25.163|3321.5160|25.164|11197.9800|25.165|3271.4500|25.166|8355.1120
+AVAXUSDT|1740096004497000|25.141|6938.9160|25.14|76048.5000|25.139|5580.8580|25.138|3167.3880|25.137|2966.1660|25.136|4826.1120|25.135|5956.9950|25.134|6936.9840|25.133|7565.0330|25.132|10228.7240|25.131|5930.9160|25.13|6609.1900|25.129|6332.5080|25.128|15277.8240|25.127|16634.0740|25.126|7990.0680|25.125|5628.0000|25.124|3718.3520|25.123|3291.1130|25.122|5702.6940|25.121|7209.7270|25.12|7008.4800|25.119|7460.3430|25.118|5551.0780|25.117|3039.1570|25.142|1156.5320|25.143|25.1430|25.144|477.7360|25.145|2313.3400|25.146|704.0880|25.147|678.9690|25.148|2615.3920|25.149|4728.0120|25.15|7695.9000|25.151|7972.8670|25.152|5432.8320|25.153|6087.0260|25.154|5735.1120|25.155|9533.7450|25.156|10490.0520|25.157|10238.8990|25.158|5585.0760|25.159|6214.2730|25.16|14492.1600|25.161|3346.4130|25.162|6315.6620|25.163|3321.5160|25.164|11197.9800|25.165|9260.7200|25.166|8355.1120

lens/test/testing_data/avax/depth/2025-02-22.csv

@@ -0,0 +1,4 @@
+pair|time|bid_0_price|bid_0_size|bid_1_price|bid_1_size|bid_2_price|bid_2_size|bid_3_price|bid_3_size|bid_4_price|bid_4_size|bid_5_price|bid_5_size|bid_6_price|bid_6_size|bid_7_price|bid_7_size|bid_8_price|bid_8_size|bid_9_price|bid_9_size|bid_10_price|bid_10_size|bid_11_price|bid_11_size|bid_12_price|bid_12_size|bid_13_price|bid_13_size|bid_14_price|bid_14_size|bid_15_price|bid_15_size|bid_16_price|bid_16_size|bid_17_price|bid_17_size|bid_18_price|bid_18_size|bid_19_price|bid_19_size|bid_20_price|bid_20_size|bid_21_price|bid_21_size|bid_22_price|bid_22_size|bid_23_price|bid_23_size|bid_24_price|bid_24_size|ask_0_price|ask_0_size|ask_1_price|ask_1_size|ask_2_price|ask_2_size|ask_3_price|ask_3_size|ask_4_price|ask_4_size|ask_5_price|ask_5_size|ask_6_price|ask_6_size|ask_7_price|ask_7_size|ask_8_price|ask_8_size|ask_9_price|ask_9_size|ask_10_price|ask_10_size|ask_11_price|ask_11_size|ask_12_price|ask_12_size|ask_13_price|ask_13_size|ask_14_price|ask_14_size|ask_15_price|ask_15_size|ask_16_price|ask_16_size|ask_17_price|ask_17_size|ask_18_price|ask_18_size|ask_19_price|ask_19_size|ask_20_price|ask_20_size|ask_21_price|ask_21_size|ask_22_price|ask_22_size|ask_23_price|ask_23_size|ask_24_price|ask_24_size
+AVAXUSDT|1740182400055000|24.531|883.1160|24.53|3802.1500|24.529|1545.3270|24.528|6082.9440|24.527|7259.9920|24.526|5690.0320|24.525|6008.6250|24.524|7357.2000|24.523|6057.1810|24.522|6056.9340|24.521|4242.1330|24.52|5443.4400|24.519|3334.5840|24.518|9880.7540|24.517|5491.8080|24.516|3236.1120|24.515|1838.6250|24.514|2990.7080|24.513|5588.9640|24.512|15687.6800|24.511|11765.2800|24.51|11421.6600|24.509|12279.0090|24.508|10538.4400|24.507|7058.0160|24.532|760.4920|24.533|4415.9400|24.534|24.5340|24.535|5544.9100|24.536|2134.6320|24.537|9397.6710|24.538|7508.6280|24.539|6650.0690|24.54|9570.6000|24.541|7116.8900|24.542|8516.0740|24.543|7927.3890|24.544|17892.5760|24.545|3559.0250|24.546|3657.3540|24.547|4123.8960|24.548|8100.8400|24.549|3019.5270|24.55|3461.5500|24.551|4787.4450|24.552|5867.9280|24.553|2602.6180|24.554|2087.0900|24.555|5745.8700|24.556|21069.0480
+AVAXUSDT|1740182401127000|24.531|1005.7710|24.53|3802.1500|24.529|1545.3270|24.528|6082.9440|24.527|7259.9920|24.526|5690.0320|24.525|6008.6250|24.524|7357.2000|24.523|6057.1810|24.522|6056.9340|24.521|4242.1330|24.52|5443.4400|24.519|3334.5840|24.518|9880.7540|24.517|5491.8080|24.516|3236.1120|24.515|1838.6250|24.514|2990.7080|24.513|5588.9640|24.512|15687.6800|24.511|11765.2800|24.51|11421.6600|24.509|12279.0090|24.508|10538.4400|24.507|7058.0160|24.532|760.4920|24.533|4415.9400|24.534|24.5340|24.535|5544.9100|24.536|2134.6320|24.537|9397.6710|24.538|7508.6280|24.539|6650.0690|24.54|9570.6000|24.541|7116.8900|24.542|8516.0740|24.543|7927.3890|24.544|17892.5760|24.545|3559.0250|24.546|3657.3540|24.547|4123.8960|24.548|8100.8400|24.549|3019.5270|24.55|3461.5500|24.551|4787.4450|24.552|5867.9280|24.553|2602.6180|24.554|2087.0900|24.555|5745.8700|24.556|21069.0480
+AVAXUSDT|1740182401863000|24.531|1005.7710|24.53|3802.1500|24.529|1545.3270|24.528|6082.9440|24.527|7186.4110|24.526|5690.0320|24.525|6008.6250|24.524|7357.2000|24.523|5738.3820|24.522|6056.9340|24.521|4242.1330|24.52|5443.4400|24.519|3334.5840|24.518|9880.7540|24.517|5491.8080|24.516|3236.1120|24.515|1838.6250|24.514|2990.7080|24.513|5588.9640|24.512|15687.6800|24.511|11765.2800|24.51|11421.6600|24.509|12279.0090|24.508|10538.4400|24.507|7058.0160|24.532|760.4920|24.533|4415.9400|24.534|24.5340|24.535|5544.9100|24.536|2134.6320|24.537|9397.6710|24.538|7508.6280|24.539|6650.0690|24.54|9570.6000|24.541|7116.8900|24.542|8417.9060|24.543|7927.3890|24.544|17892.5760|24.545|3559.0250|24.546|3338.2560|24.547|4123.8960|24.548|8100.8400|24.549|3019.5270|24.55|3461.5500|24.551|4787.4450|24.552|5867.9280|24.553|2602.6180|24.554|2087.0900|24.555|5745.8700|24.556|21069.0480

lens/test/testing_data/avax/depth/2025-02-23.csv

@@ -0,0 +1,5 @@
+pair|time|bid_0_price|bid_0_size|bid_1_price|bid_1_size|bid_2_price|bid_2_size|bid_3_price|bid_3_size|bid_4_price|bid_4_size|bid_5_price|bid_5_size|bid_6_price|bid_6_size|bid_7_price|bid_7_size|bid_8_price|bid_8_size|bid_9_price|bid_9_size|bid_10_price|bid_10_size|bid_11_price|bid_11_size|bid_12_price|bid_12_size|bid_13_price|bid_13_size|bid_14_price|bid_14_size|bid_15_price|bid_15_size|bid_16_price|bid_16_size|bid_17_price|bid_17_size|bid_18_price|bid_18_size|bid_19_price|bid_19_size|bid_20_price|bid_20_size|bid_21_price|bid_21_size|bid_22_price|bid_22_size|bid_23_price|bid_23_size|bid_24_price|bid_24_size|ask_0_price|ask_0_size|ask_1_price|ask_1_size|ask_2_price|ask_2_size|ask_3_price|ask_3_size|ask_4_price|ask_4_size|ask_5_price|ask_5_size|ask_6_price|ask_6_size|ask_7_price|ask_7_size|ask_8_price|ask_8_size|ask_9_price|ask_9_size|ask_10_price|ask_10_size|ask_11_price|ask_11_size|ask_12_price|ask_12_size|ask_13_price|ask_13_size|ask_14_price|ask_14_size|ask_15_price|ask_15_size|ask_16_price|ask_16_size|ask_17_price|ask_17_size|ask_18_price|ask_18_size|ask_19_price|ask_19_size|ask_20_price|ask_20_size|ask_21_price|ask_21_size|ask_22_price|ask_22_size|ask_23_price|ask_23_size|ask_24_price|ask_24_size
+AVAXUSDT|1740268800054000|25.822|4570.4940|25.821|77.4630|25.82|542.2200|25.819|2556.0810|25.818|2297.8020|25.817|4053.2690|25.816|5137.3840|25.815|3226.8750|25.814|7795.8280|25.813|2787.8040|25.812|6220.6920|25.811|11253.5960|25.81|8852.8300|25.809|3484.2150|25.808|7303.6640|25.807|4438.8040|25.806|5599.9020|25.805|3483.6750|25.804|13959.9640|25.803|2451.2850|25.802|6218.2820|25.801|14500.1620|25.8|10629.6000|25.799|70508.6670|25.798|8306.9560|25.823|955.4510|25.824|1652.7360|25.825|852.2250|25.826|6069.1100|25.827|2892.6240|25.828|5062.2880|25.829|6793.0270|25.83|8084.7900|25.831|11133.1610|25.832|7413.7840|25.833|11134.0230|25.834|4030.1040|25.835|11574.0800|25.836|11109.4800|25.837|8552.0470|25.838|7312.1540|25.839|5452.0290|25.84|4573.6800|25.841|4703.0620|25.842|5556.0300|25.843|4574.2110|25.844|8864.4920|25.845|4161.0450|25.846|9123.6380|25.847|23417.3820
+AVAXUSDT|1740268802814000|25.822|4570.4940|25.821|77.4630|25.82|542.2200|25.819|2556.0810|25.818|2297.8020|25.817|4053.2690|25.816|5137.3840|25.815|3226.8750|25.814|7795.8280|25.813|2787.8040|25.812|6220.6920|25.811|11253.5960|25.81|8852.8300|25.809|3484.2150|25.808|7303.6640|25.807|4438.8040|25.806|5599.9020|25.805|3483.6750|25.804|13959.9640|25.803|2451.2850|25.802|6218.2820|25.801|14500.1620|25.8|10629.6000|25.799|70508.6670|25.798|8306.9560|25.823|955.4510|25.824|1652.7360|25.825|852.2250|25.826|6069.1100|25.827|2892.6240|25.828|5062.2880|25.829|6793.0270|25.83|8058.9600|25.831|11133.1610|25.832|7413.7840|25.833|11134.0230|25.834|4030.1040|25.835|11574.0800|25.836|11109.4800|25.837|8552.0470|25.838|7312.1540|25.839|5452.0290|25.84|4573.6800|25.841|4703.0620|25.842|5556.0300|25.843|4574.2110|25.844|8864.4920|25.845|4161.0450|25.846|9123.6380|25.847|23417.3820
+AVAXUSDT|1740268804538000|25.822|4518.8500|25.821|77.4630|25.82|542.2200|25.819|2556.0810|25.818|1833.0780|25.817|5344.1190|25.816|3252.8160|25.815|3226.8750|25.814|7770.0140|25.813|2787.8040|25.812|6220.6920|25.811|11227.7850|25.81|8827.0200|25.809|3484.2150|25.808|7303.6640|25.807|4438.8040|25.806|5599.9020|25.805|2425.6700|25.804|13959.9640|25.803|2451.2850|25.802|6218.2820|25.801|14500.1620|25.8|10629.6000|25.799|70534.4660|25.798|8306.9560|25.823|955.4510|25.824|1652.7360|25.825|852.2250|25.826|6069.1100|25.827|2892.6240|25.828|5062.2880|25.829|5139.9710|25.83|6328.3500|25.831|11133.1610|25.832|7413.7840|25.833|11134.0230|25.834|4030.1040|25.835|11548.2450|25.836|10954.4640|25.837|8552.0470|25.838|7312.1540|25.839|5426.1900|25.84|4573.6800|25.841|4625.5390|25.842|5556.0300|25.843|4574.2110|25.844|8864.4920|25.845|4161.0450|25.846|9123.6380|25.847|23417.3820
+AVAXUSDT|1740268804592000|25.823|309.8760|25.817|5034.3150|25.816|3691.6880|25.815|3330.1350|25.814|6737.4540|25.813|3871.9500|25.812|6220.6920|25.811|7278.7020|25.81|7330.0400|25.809|3742.3050|25.808|7303.6640|25.807|4438.8040|25.806|5341.8420|25.805|2425.6700|25.804|13959.9640|25.803|2451.2850|25.802|6218.2820|25.801|14500.1620|25.8|10629.6000|25.799|70534.4660|25.798|1315.6980|25.797|7326.3480|25.796|2502.2120|25.795|4024.0200|25.794|6525.8820|25.83|439.1100|25.831|4804.5660|25.832|5553.8800|25.833|7594.9020|25.834|4004.2700|25.835|7828.0050|25.836|3436.1880|25.837|5942.5100|25.838|7027.9360|25.839|2325.5100|25.84|4470.3200|25.841|3772.7860|25.842|5556.0300|25.843|4574.2110|25.844|5814.9000|25.845|4419.4950|25.846|9123.6380|25.847|23417.3820|25.848|9822.2400|25.849|11115.0700|25.85|38697.4500|25.851|161517.0480|25.852|3696.8360|25.853|7057.8690|25.854|8583.5280

lens/test/testing_data/avax/trades/2025-02-21.csv

@@ -0,0 +1,5 @@
+symbol|side|time|price|volume
+AVAXUSDT|1|1740096004449000|25.142|75.4260
+AVAXUSDT|0|1740096004468000|25.141|905.0760
+AVAXUSDT|0|1740096004468000|25.141|377.1150
+AVAXUSDT|0|1740096004468000|25.141|25.1410

lens/test/testing_data/avax/trades/2025-02-22.csv

@@ -0,0 +1,5 @@
+symbol|side|time|price|volume
+AVAXUSDT|1|1740182400055000|24.531|318.9030
+AVAXUSDT|1|1740182400055000|24.531|24.5310
+AVAXUSDT|1|1740182404487000|24.532|49.0640
+AVAXUSDT|1|1740182404516000|24.532|24.5320

lens/test/testing_data/avax/trades/2025-02-23.csv

@@ -0,0 +1,5 @@
+symbol|side|time|price|volume
+AVAXUSDT|0|1740268804522000|25.822|51.6440
+AVAXUSDT|1|1740268804543000|25.823|25.8230
+AVAXUSDT|1|1740268804543000|25.823|103.2920
+AVAXUSDT|1|1740268804543000|25.823|25.8230

lens/test/testing_data/metadata.json

@@ -0,0 +1,6 @@
+{
+  "version": 0,
+  "coins": ["avax"],
+  "start": "2025-02-21",
+  "end": "2023-02-23"
+}