Merge branch 'main' into 120-add-use-examples-efficientnet-fine-tuning-on-cifar-100

Author: sashakolpakov

.github/workflows/automerge.yml

@@ -13,12 +13,14 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-latest
+    # container:
+    #   image: python:33.11.9
     steps:
     - uses: actions/checkout@v3
     - name: Test Cerebros
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10.4"
+        python-version: "3.11"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -31,28 +33,16 @@ jobs:
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-#    - name: Test by running.
-#      run: python3 cerebros.py
-#    - name: Test distributed by running.
-#      run: python3 test_simple_cerebros_gridsearch.py
-#    - name: Test distributed random search wine by running.
-#      run: python3 random_search.py
-    - name: Test image classifier EfficientNetv2S - CIFAR100
-      run: python3 cifar_100_EfficientNet.py
-      timeout-minutes: 600
-#    - name: Test CerebrosRealNeuronNetwork
-#      run: python3 realnn-regression-example-ames-no-preproc.py
-#      timeout-minutes: 45
     - name: Test distributed random search Ames by running
       run: python3 regression-example-ames-no-preproc.py
     - name: Test distributed random search Ames by running - Val set
       run: python3 regression-example-ames-no-preproc-val-set.py
-    - name: Test text classifier - random search - ham-spam
-      run: python3 text-class-ham-or-spam.py
-      timeout-minutes: 90
-    - name: Test image classifier - small subset of CIFAR10
+    # - name: Test text classifier - random search - ham-spam
+    #   run: python3 text-class-ham-or-spam.py
+    #   timeout-minutes: 90
+    - name: Test image classifier - small subset of CIFAR10 # add back
       timeout-minutes: 90
       run: python3 cifar10-example.py
     - name: Phishing email detection with GPT2 embedding
-      timeout-minutes: 180
+      timeout-minutes: 420
       run: python3 phishing_email_detection_gpt2.py

cerebros/neuralnetworkfuture/neural_network_future.py

@@ -332,8 +332,10 @@ def compile_neural_network(self):
         self.materialized_neural_network.compile(
             loss=self.loss,
             metrics=self.metrics,
-            optimizer=tf.keras.optimizers.Adam(
-                    learning_rate=self.learning_rate),
+            optimizer=tf.keras.optimizers.AdamW(
+                learning_rate=self.learning_rate,
+                weight_decay=0.004  # Add weight decay parameter
+            ),
             jit_compile=jit_compile)
 
     def util_parse_connectivity_csv(self):

cerebros/simplecerebrosrandomsearch/simple_cerebros_random_search.py

@@ -2,6 +2,7 @@
 import numpy as np
 import pandas as pd
 import tensorflow as tf
+from tqdm import tqdm
 from cerebros.denseautomlstructuralcomponent.\
     dense_automl_structural_component \
     import DenseAutoMlStructuralComponent, DenseLateralConnectivity, \
@@ -502,7 +503,7 @@ def run_moity_permutations(self, spec, subtrial_number, lock):
         with open(neural_network_spec_file, 'w') as f:
             f.write(str(spec))
         next_model_name =\
-            f"{self.project_name}/models/tr_{str(self.trial_number).zfill(16)}_subtrial_{str(subtrial_number).zfill(16)}"\
+            f"{self.project_name}/models/tr_{str(self.trial_number).zfill(16)}_subtrial_{str(subtrial_number).zfill(16)}.keras"\
             .lower()
         neural_network.save(next_model_name)
         oracle_0['trial_number'] = self.trial_number
@@ -519,7 +520,10 @@ def run_moity_permutations(self, spec, subtrial_number, lock):
 
     def run_random_search(self):
         processes = []
-        for i in np.arange(self.number_of_architecture_moities_to_try):
+        for i in tqdm(np.arange(self.number_of_architecture_moities_to_try),
+                      desc="Global task progress",
+                      colour="#16ceeb"):
+
             self.parse_neural_network_structural_spec_random()
             spec = self.get_neural_network_spec()
 

cerebros/units/units.py

@@ -83,8 +83,13 @@ def __init__(self,
                  train_data_dtype=tf.float32,
                  *args,
                  **kwargs):
-
-        self.input_shape = input_shape
+        if isinstance(input_shape, int):
+            self.input_shape = (input_shape,)
+        elif isinstance(input_shape, str):
+            self.input_shape = (int(input_shape),)
+        else:
+            _input_shape = [int(ax) for ax in input_shape]
+            self.input_shape = tuple(_input_shape)
         self.neural_network_layer = []
         self.base_models = base_models
         self.train_data_dtype = train_data_dtype

cicd-requirements.txt

@@ -1,6 +1,6 @@
-tensorflow-datasets==4.9.3
-gmpy2==2.1.5
-matplotlib==3.6.2
-tensorflow-text==2.12.0
-keras-nlp==0.6.2
-scikit-learn==1.3.2
+matplotlib==3.8.4
+tensorflow-text==2.19.0
+keras-nlp==0.9.1
+scikit-learn==1.4.1.post1
+tensorflow-hub==0.16.1
+transformers==4.51.1

cifar10-example.py

@@ -88,8 +88,8 @@ def make_dataset(dataset):
 
 last_relevant_layer = base_new.layers[-2]
 # last_relevant_layer_extracted = last_relevant_layer #.output[0][0][0]
-base_embedding = tf.keras.Model(inputs=base_new.layers[0].input,
-                                outputs=last_relevant_layer.output)
+base_embedding = tf.keras.Model(inputs=base_new.input,
+                               outputs=last_relevant_layer.output)
 
 
 image_input_0 = tf.keras.layers.Input(shape=INPUT_SHAPES[0])