Merge branch 'develop'

pythonlessons · pythonlessons · commit 0325e4bf554b · 2023-11-29T11:22:04.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,13 @@
-## [0.1.0] - 
+## [0.2.0] - 2023-11-..
+### Added:
+- Created `reward.simpleReward` function to calculate reward based on the action and the difference between the current price and the previous price
+- Created `scalers.MinMaxScaler` object to transform the price data to a range between 0 and 1 and prepare it for the neural networks input
+- Created `state.Observations` object to hold the observations of the agent with set window size
+- Updated `render.PygameRender` object to render the agent's actions
+- Updated `state.State` to hold current state `assets`, `balance` and `allocation_percentage` on specific State
+
+
+## [0.1.0] - 2023-10-17
 ### Initial Release:
 - created the project
 - created code to create random sinusoidal price data
diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@ Reinforcement Learning package for Finance
 
 # Environment Structure:
 <p align="center">
-  <img src="Tutorials\Documents\01_FinRock.jpg">
+  <img src="Tutorials\Documents\02_FinRock.jpg">
 </p>
 
 ### Install requirements
@@ -20,4 +20,13 @@ python bin/create_sinusoid_data.py
 ### Run environment:
 ```
 python playing.py
-```
+```
+
+### Environment Render:
+<p align="center">
+  <img src="Tutorials\Documents\02_FinRock_render.png">
+</p>
+
+## Links to YouTube videos:
+- [Introduction to FinRock package](https://youtu.be/xU_YJB7vilA)
+- [Complete Trading Simulation Backbone](https://youtu.be/1z5geob8Yho)
diff --git a/Tutorials/02_Trading_simulation_backbone.md b/Tutorials/02_Trading_simulation_backbone.md
@@ -0,0 +1,37 @@
+# Complete Trading Simulation Backbone
+
+### Environment Structure:
+<p align="center">
+  <img src="Documents\02_FinRock.jpg">
+</p>
+
+### Link to YouTube video:
+https://youtu.be/1z5geob8Yho
+
+### Link to tutorial code:
+https://github.com/pythonlessons/FinRock/tree/0.2.0
+
+### Download tutorial code:
+https://github.com/pythonlessons/FinRock/archive/refs/tags/0.2.0.zip
+
+
+### Install requirements:
+```
+pip install -r requirements.txt
+pip install pygame
+```
+
+### Create sinusoid data:
+```
+python bin/create_sinusoid_data.py
+```
+
+### Run environment:
+```
+python playing.py
+```
+
+### Environment Render:
+<p align="center">
+  <img src="Documents\02_FinRock_render.png">
+</p>
diff --git a/Tutorials/Documents/02_FinRock.jpg b/Tutorials/Documents/02_FinRock.jpg
diff --git a/Tutorials/Documents/02_FinRock_render.png b/Tutorials/Documents/02_FinRock_render.png
diff --git a/finrock/__init__.py b/finrock/__init__.py
@@ -1 +1 @@
-__version__ = "0.1.0"
+__version__ = "0.2.0"
diff --git a/finrock/data_feeder.py b/finrock/data_feeder.py
@@ -3,20 +3,35 @@
 from finrock.state import State
 
 class PdDataFeeder:
-    def __init__(self, df: pd.DataFrame):
+    def __init__(
+            self, 
+            df: pd.DataFrame,
+            min: float = None,
+            max: float = None
+            ) -> None:
         self._df = df
+        self._min = min
+        self._max = max
 
-        assert isinstance(self._df, pd.DataFrame) == True
-        assert 'timestamp' in self._df.columns
-        assert 'open' in self._df.columns
-        assert 'high' in self._df.columns
-        assert 'low' in self._df.columns
-        assert 'close' in self._df.columns
+        assert isinstance(self._df, pd.DataFrame) == True, "df must be a pandas.DataFrame"
+        assert 'timestamp' in self._df.columns, "df must have 'timestamp' column"
+        assert 'open' in self._df.columns, "df must have 'open' column"
+        assert 'high' in self._df.columns, "df must have 'high' column"
+        assert 'low' in self._df.columns, "df must have 'low' column"
+        assert 'close' in self._df.columns, "df must have 'close' column"
+
+    @property
+    def min(self) -> float:
+        return self._min or self._df['low'].min()
+    
+    @property
+    def max(self) -> float:
+        return self._max or self._df['high'].max()
 
     def __len__(self) -> int:
         return len(self._df)
     
-    def __getitem__(self, idx: int) -> State:
+    def __getitem__(self, idx: int, args=None) -> State:
         data = self._df.iloc[idx]
 
         state = State(
diff --git a/finrock/render.py b/finrock/render.py
@@ -12,6 +12,8 @@ class ColorTheme:
     down_candle = red
     wick = white
     text = white
+    buy = green
+    sell = red
 
 class PygameRender:
     def __init__(
@@ -127,7 +129,33 @@ def render(self, info: dict):
             # Draw candlestick body
             self.pygame.draw.rect(canvas, candle_color, (candle_offset, candle_body_y, self.candle_width, candle_body_height))
 
+            # Compare with previous state to determine whether buy or sell action was taken and draw arrow
+            index = self._states.index(state)
+            if index > 0:
+                last_state = self._states[index - 1]
+                if last_state.allocation_percentage < state.allocation_percentage:
+                    # buy
+                    self.pygame.draw.polygon(canvas, self.color_theme.buy, [
+                        (candle_offset + self.candle_width // 2, candle_y_low + 10), 
+                        (candle_offset + self.candle_width // 2 - 5, candle_y_low + 20), 
+                        (candle_offset + self.candle_width // 2 + 5, candle_y_low + 20)
+                        ])
+                elif last_state.allocation_percentage > state.allocation_percentage:
+                    # sell
+                    self.pygame.draw.polygon(canvas, self.color_theme.sell, [
+                        (candle_offset + self.candle_width // 2, candle_y_high - 10), 
+                        (candle_offset + self.candle_width // 2 - 5, candle_y_high - 20), 
+                        (candle_offset + self.candle_width // 2 + 5, candle_y_high - 20)
+                        ])
+
             # Move to the next candle
             candle_offset += self.candle_width + self.candle_spacing
 
+        # Draw max and min ohlc values on the chart
+        font = self.pygame.font.SysFont('Noto Sans', 15)
+        label_y_low = font.render(str(max_low), True, self.color_theme.text)
+        label_y_high = font.render(str(max_high), True, self.color_theme.text)
+        canvas.blit(label_y_low, (self.candle_spacing + 5, self.chart_height + 35))
+        canvas.blit(label_y_high, (self.candle_spacing + 5, 5))
+
         return canvas
diff --git a/finrock/reward.py b/finrock/reward.py
@@ -0,0 +1,28 @@
+from .state import Observations
+
+
+def simpleReward(observations: Observations) -> float:
+    
+    assert isinstance(observations, Observations) == True, "observations must be an instance of Observations"
+
+    last_state, next_state = observations[-2:]
+
+    # buy
+    if next_state.allocation_percentage > last_state.allocation_percentage:
+        # check whether it was good or bad to buy
+        order_size = next_state.allocation_percentage - last_state.allocation_percentage
+        reward = (next_state.close - last_state.close) / last_state.close * order_size
+
+    # sell
+    elif next_state.allocation_percentage < last_state.allocation_percentage:
+        # check whether it was good or bad to sell
+        order_size = last_state.allocation_percentage - next_state.allocation_percentage
+        reward = -1 * (next_state.close - last_state.close) / last_state.close * order_size
+
+    # hold
+    else:
+        # check whether it was good or bad to hold
+        ratio = -1 if not last_state.allocation_percentage else last_state.allocation_percentage
+        reward = (next_state.close - last_state.close) / last_state.close * ratio
+        
+    return reward
diff --git a/finrock/scalers.py b/finrock/scalers.py
@@ -0,0 +1,25 @@
+import numpy as np
+from .state import Observations
+
+class MinMaxScaler:
+    def __init__(self, min: float, max: float):
+        self._min = min
+        self._max = max
+    
+    def transform(self, observations: Observations) -> np.ndarray:
+
+        assert isinstance(observations, Observations) == True, "observations must be an instance of Observations"
+
+        transformed_data = []
+        for state in observations:
+            open = (state.open - self._min) / (self._max - self._min)
+            high = (state.high - self._min) / (self._max - self._min)
+            low = (state.low - self._min) / (self._max - self._min)
+            close = (state.close - self._min) / (self._max - self._min)
+            
+            transformed_data.append([open, high, low, close, state.allocation_percentage])
+
+        return np.array(transformed_data)
+    
+    def __call__(self, observations) -> np.ndarray:
+        return self.transform(observations)
diff --git a/finrock/state.py b/finrock/state.py
@@ -1,3 +1,4 @@
+import typing
 from datetime import datetime
 
 class State:
@@ -20,4 +21,87 @@ def __init__(
         try:
             self.date = datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S')
         except ValueError:
-            raise ValueError(f'received invalid timestamp date format: {timestamp}, expected: YYYY-MM-DD HH:MM:SS')
+            raise ValueError(f'received invalid timestamp date format: {timestamp}, expected: YYYY-MM-DD HH:MM:SS')
+        
+        self._balance = 0.0 # balance in cash
+        self._assets = 0.0 # balance in assets
+        self._allocation_percentage = 0.0 # percentage of assets allocated to this state
+        
+    @property
+    def balance(self):
+        return self._balance
+    
+    @balance.setter
+    def balance(self, value: float):
+        self._balance = value
+
+    @property
+    def assets(self):
+        return self._assets
+    
+    @assets.setter
+    def assets(self, value: float):
+        self._assets = value
+
+    @property
+    def account_value(self):
+        return self.balance + self.assets * self.close
+
+    @property
+    def allocation_percentage(self):
+        return self._allocation_percentage
+    
+    @allocation_percentage.setter
+    def allocation_percentage(self, value: float):
+        assert 0.0 <= value <= 1.0, f'allocation_percentage value must be between 0.0 and 1.0, received: {value}'
+        self._allocation_percentage = value
+    
+
+class Observations:
+    def __init__(
+            self, 
+            window_size: int,
+            observations: typing.List[State]=[],
+        ):
+        self._observations = observations
+        self._window_size = window_size
+
+        assert isinstance(self._observations, list) == True, "observations must be a list"
+        assert len(self._observations) <= self._window_size, f'observations length must be <= window_size, received: {len(self._observations)}'
+        assert all(isinstance(observation, State) for observation in self._observations) == True, "observations must be a list of State objects"
+
+    def __len__(self) -> int:
+        return len(self._observations)
+    
+    @property
+    def window_size(self) -> int:
+        return self._window_size
+    
+    @property
+    def observations(self) -> typing.List[State]:
+        return self._observations
+    
+    @property
+    def full(self) -> bool:
+        return len(self._observations) == self._window_size
+    
+    def __getitem__(self, idx: int) -> State:
+        try:
+            return self._observations[idx]
+        except IndexError:
+            raise IndexError(f'index out of range: {idx}, observations length: {len(self._observations)}')
+        
+    def __iter__(self) -> State:
+        """ Create a generator that iterate over the Sequence."""
+        for index in range(len(self)):
+            yield self[index]
+
+    def reset(self) -> None:
+        self._observations = []
+    
+    def append(self, state: State) -> None:
+        assert isinstance(state, State) == True, "state must be a State object"
+        self._observations.append(state)
+
+        if len(self._observations) > self._window_size:
+            self._observations.pop(0)
diff --git a/finrock/trading_env.py b/finrock/trading_env.py
diff --git a/playing.py b/playing.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.1.0"`
	`1`	`+__version__ = "0.2.0"`