[ update ]: remove asyncio features

Author: belajarqywok

.github/workflows/gru_pipeline.yaml

@@ -74,7 +74,7 @@ jobs:
           fetch-depth: 1
 
       - name: Install Libraries
-        run: pip install -r requirements.txt
+        run: pip install -r dev.requirements.txt
 
       - name: Download Artifact (datasets)
         uses: actions/download-artifact@v3

.github/workflows/lstm_gru_pipeline.yaml

@@ -74,7 +74,7 @@ jobs:
           fetch-depth: 1
 
       - name: Install Libraries
-        run: pip install -r requirements.txt
+        run: pip install -r dev.requirements.txt
 
       - name: Download Artifact (datasets)
         uses: actions/download-artifact@v3

.github/workflows/lstm_pipeline.yaml

@@ -74,7 +74,7 @@ jobs:
           fetch-depth: 1
 
       - name: Install Libraries
-        run: pip install -r requirements.txt
+        run: pip install -r dev.requirements.txt
 
       - name: Download Artifact (datasets)
         uses: actions/download-artifact@v3

dev.requirements.txt

@@ -1,51 +1,57 @@
 absl-py==2.1.0
+aioredis==2.0.1
 annotated-types==0.7.0
 anyio==4.4.0
 astunparse==1.6.3
+async-timeout==4.0.3
 certifi==2024.7.4
 charset-normalizer==3.3.2
 click==8.1.7
+Cython==3.0.10
 dnspython==2.6.1
-email_validator==2.1.1
-exceptiongroup==1.2.1
+email_validator==2.2.0
+exceptiongroup==1.2.2
 fastapi==0.111.0
-fastapi-cli==0.0.4
+fastapi-cli==0.0.5
 flatbuffers==24.3.25
-gast==0.5.4
+gast==0.6.0
 google-pasta==0.2.0
-grpcio==1.64.0
+grpcio==1.65.4
 h11==0.14.0
 h5py==3.11.0
 httpcore==1.0.5
 httptools==0.6.1
 httpx==0.27.0
 idna==3.7
-importlib_metadata==7.1.0
 Jinja2==3.1.4
 joblib==1.4.2
 keras==3.3.3
 libclang==18.1.1
+llvmlite==0.43.0
 Markdown==3.6
 markdown-it-py==3.0.0
 MarkupSafe==2.1.5
 mdurl==0.1.2
 ml-dtypes==0.3.2
 namex==0.0.8
+numba==0.60.0
 numpy==1.26.4
 opt-einsum==3.3.0
-optree==0.11.0
-orjson==3.10.3
-packaging==24.0
+optree==0.12.1
+packaging==24.1
 pandas==2.2.2
 protobuf==4.25.3
 pydantic==2.7.2
+pydantic-settings==2.4.0
 pydantic_core==2.18.3
 Pygments==2.18.0
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
 python-multipart==0.0.9
 pytz==2024.1
 PyYAML==6.0.1
+redis==3.5.3
+redis-py-cluster==2.1.3
 requests==2.32.3
 rich==13.7.1
 scikit-learn==1.5.0
@@ -57,18 +63,18 @@ starlette==0.37.2
 tensorboard==2.16.2
 tensorboard-data-server==0.7.2
 tensorflow==2.16.1
-tensorflow-io-gcs-filesystem==0.31.0
+tensorflow-io-gcs-filesystem==0.37.1
 termcolor==2.4.0
 threadpoolctl==3.5.0
 typer==0.12.3
-typing_extensions==4.12.1
+typing_extensions==4.12.2
 tzdata==2024.1
 ujson==5.10.0
 urllib3==2.2.2
 uvicorn==0.30.1
 uvloop==0.19.0
+valkey==6.0.0b1
 watchfiles==0.22.0
 websockets==12.0
 Werkzeug==3.0.3
 wrapt==1.16.0
-zipp==3.19.1

training/data_processor.pyx

@@ -6,7 +6,7 @@ filterwarnings('ignore')
 
 
 """ Get Datasets """
-async def get_datasets(str datasets_path):
+def get_datasets(str datasets_path):
   cdef list items = os.listdir(datasets_path)
   cdef list csv_files = []
   cdef str item
@@ -19,7 +19,7 @@ async def get_datasets(str datasets_path):
 
 
 """ Create Sequences """
-async def create_sequences(df, int sequence_length):
+def create_sequences(df, int sequence_length):
   cdef list labels = []
   cdef list sequences = []
   cdef int i
@@ -34,7 +34,7 @@ async def create_sequences(df, int sequence_length):
 
 
 """ Pre-Process Data """
-async def preprocess_data(dataframe):
+def preprocess_data(dataframe):
   cdef str col
 
   for col in dataframe.columns:
@@ -48,7 +48,7 @@ async def preprocess_data(dataframe):
 
 
 """ Scale Data """
-async def scale_data(dataframe, scaler_cls):
+def scale_data(dataframe, scaler_cls):
   scaler = scaler_cls()
   dataframe['Close'] = scaler.fit_transform(dataframe[['Close']])
   return scaler, dataframe

training/main.pyx

@@ -24,43 +24,43 @@ from training.model_builder import (
 from warnings import filterwarnings
 filterwarnings('ignore')
 
-async def main(algorithm: str, sequence_length: int, epochs: int, batch_size: int):
+def main(algorithm: str, sequence_length: int, epochs: int, batch_size: int):
     datasets_path = './datasets'
     models_path   = './models'
     posttrained   = './posttrained'
     pickle_file   = './pickles'
 
 
-    for dataset in await get_datasets(datasets_path):
+    for dataset in get_datasets(datasets_path):
         print(f"[TRAINING] {dataset.replace('.csv', '')} ")
 
         dataframe = pd.read_csv(os.path.join(datasets_path, dataset), index_col='Date')[['Close']]
         model_file = os.path.join(models_path, f"{dataset.replace('.csv', '')}.keras")
 
         # dataframe = preprocess_data(dataframe)
         dataframe.dropna(inplace = True)
-        standard_scaler, dataframe = await scale_data(dataframe, StandardScaler)
-        minmax_scaler, dataframe = await scale_data(dataframe, MinMaxScaler)
+        standard_scaler, dataframe = scale_data(dataframe, StandardScaler)
+        minmax_scaler, dataframe = scale_data(dataframe, MinMaxScaler)
 
-        sequences, labels = await create_sequences(dataframe, sequence_length)
+        sequences, labels = create_sequences(dataframe, sequence_length)
         input_shape = (sequences.shape[1], sequences.shape[2])
 
         if algorithm == "GRU":
-            model = await gru_model(input_shape)
+            model = gru_model(input_shape)
 
         elif algorithm == "LSTM":
-            model = await lstm_model(input_shape)
+            model = lstm_model(input_shape)
 
         elif algorithm == "LSTM_GRU":
-            model = await lstm_gru_model(input_shape)
+            model = lstm_gru_model(input_shape)
 
-        else: model = await lstm_model(input_shape)
+        else: model = lstm_model(input_shape)
 
         train_size = int(len(sequences) * 0.8)
         X_train, X_test = sequences[:train_size], sequences[train_size:]
         y_train, y_test = labels[:train_size], labels[train_size:]
 
-        await train({
+        train({
 			'model': model,
 			'model_file': model_file,
 			'sequence_length': sequence_length,
@@ -70,7 +70,7 @@ async def main(algorithm: str, sequence_length: int, epochs: int, batch_size: in
 
         dataframe_json = {'Date': dataframe.index.tolist(), 'Close': dataframe['Close'].tolist()}
 
-        await save_json(
+        save_json(
           os.path.join(posttrained, f'{dataset.replace(".csv", "")}-posttrained.json'),
           dataframe_json
         )

training/model_builder.pyx

@@ -6,7 +6,7 @@ filterwarnings('ignore')
 
 
 """ GRU (Gated Recurrent Units) Model """
-async def gru_model(input_shape):
+def gru_model(input_shape):
   cdef object model = Sequential([
     GRU(50, return_sequences = True, input_shape = input_shape),
     Dropout(0.2),
@@ -28,7 +28,7 @@ async def gru_model(input_shape):
 
 
 """ LSTM (Long Short-Term Memory) Model """
-async def lstm_model(input_shape):
+def lstm_model(input_shape):
   cdef object model = Sequential([
     LSTM(50, return_sequences = True, input_shape = input_shape),
     Dropout(0.2),
@@ -53,7 +53,7 @@ async def lstm_model(input_shape):
   LSTM (Long Short-Term Memory) and
   GRU (Gated Recurrent Units) Model
 """
-async def lstm_gru_model(input_shape):
+def lstm_gru_model(input_shape):
   cdef object model = Sequential([
     LSTM(50, return_sequences = True, input_shape = input_shape),
     Dropout(0.2),

training/post_processor.pyx

@@ -6,12 +6,12 @@ filterwarnings('ignore')
 
 
 """ Inverse Transform """
-async def inverse_transform(object scaler, data):
+def inverse_transform(object scaler, data):
 	return scaler.inverse_transform(data)
 
 
 """ save json """
-async def save_json(str filename, data):
+def save_json(str filename, data):
 	with open(filename, 'w') as f:
 		json.dump(data, f)
 

training/trainer.pyx

@@ -5,7 +5,7 @@ filterwarnings('ignore')
 
 
 """ Trainer """
-async def train(dict configuration, X_train, y_train, X_test, y_test):
+def train(dict configuration, X_train, y_train, X_test, y_test):
 	cdef object early_stopping = EarlyStopping(
 		monitor = 'val_loss',
 		patience = 5,

trainingcli.py

@@ -1,4 +1,4 @@
-import asyncio
+# import asyncio
 import argparse
 from training.main import main as training
 
@@ -14,17 +14,25 @@ def main() -> None:
     parser.add_argument('-s', '--sequences', type = int, required = True, help = 'sequences length')
 
     args = parser.parse_args()
-    event_loop = asyncio.get_event_loop()
-
-    event_loop.run_until_complete(
-        training(
-            epochs     = args.epochs,
-            batch_size = args.batchs,
-            algorithm  = args.algorithm,
-            sequence_length = args.sequences
-        )
+
+    training(
+        epochs     = args.epochs,
+        batch_size = args.batchs,
+        algorithm  = args.algorithm,
+        sequence_length = args.sequences
     )
 
+    # event_loop = asyncio.get_event_loop()
+
+    # event_loop.run_until_complete(
+    #     training(
+    #         epochs     = args.epochs,
+    #         batch_size = args.batchs,
+    #         algorithm  = args.algorithm,
+    #         sequence_length = args.sequences
+    #     )
+    # )
+
 
 if __name__ == "__main__": main()