Skip to content

Commit 6c7924a

Browse files
david-throwerdavid-thrower
authored
Merge pull request #257 from david-thrower/255-copy-of-branch-254-updated-hpo-script-for-cicd-scale-testing
255 copy of branch 254 updated hpo script for cicd scale testing
2 parents 1fa8cf9 + 428c0ec commit 6c7924a

File tree

3 files changed

+66
-14
lines changed

3 files changed

+66
-14
lines changed

‎.github/workflows/automerge.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ name: Python application
55

66
on:
77
push:
8-
branches: [ "main", "254-more-optimizations-to-notgpt-hpo-script" ]
8+
branches: [ "main", "255-copy-of-branch-254-updated-hpo-script-for-cicd-scale-testing" ]
99

1010
permissions:
1111
contents: read

‎cerebros/neuralnetworkfuture/neural_network_future.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ def compile_neural_network(self):
341341
weight_decay=0.004, # Add weight decay parameter
342342
gradient_accumulation_steps=self.gradient_accumulation_steps
343343
),
344-
jit_compile=jit_compile)
344+
jit_compile=True) # jit_compile)
345345
elif self.gradient_accumulation_steps == 1:
346346
self.materialized_neural_network.compile(
347347
loss=self.loss,
@@ -350,7 +350,7 @@ def compile_neural_network(self):
350350
learning_rate=self.learning_rate,
351351
weight_decay=0.004, # Add weight decay parameter
352352
),
353-
jit_compile=jit_compile)
353+
jit_compile=True) # jit_compile=jit_compile)
354354
else:
355355
raise ValueError("gradient_accumulation_steps must be an int >= 0. You set it as {self.gradient_accumulation_steps} type {type(self.gradient_accumulation_steps)}")
356356

‎generative-proof-of-concept-CPU-preprocessing-in-memory.py‎

Lines changed: 63 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,9 @@ def objective(trial: optuna.Trial) -> float:
6161
### Non - HP tuning parameters (Optimize to RAM / CPU / GPU capacity)
6262

6363
# Number of text samples to create: # Number of text samples (of approximately max_seq_len) to create
64-
# Raises RAM in a linear fashion
65-
66-
SAMPLES_TO_CREATE = 681
64+
# Raises RAM in a linear fashion
65+
66+
SAMPLES_TO_CREATE = 20 # 681
6767

6868
# How many tokens to provide before expecting the next token to be predicted.
6969
# Half this = double RAM (inversely proportional to RAM requirement)
@@ -86,7 +86,7 @@ def objective(trial: optuna.Trial) -> float:
8686

8787
GENERATION_PROMPT_LEN = 25
8888
MAX_NEW_TOKENS = MAX_SEQ_LENGTH - GENERATION_PROMPT_LEN
89-
RESULT_CUTOFF = 20 # Only print out verbose text samples when perplexity is < RESULT_CUTOFF
89+
RESULT_CUTOFF = 20 # 100 # <---<< In production 100 # Only print out verbose text samples when perplexity is < RESULT_CUTOFF
9090

9191
if GENERATION_PROMPT_LEN + MAX_NEW_TOKENS > MAX_SEQ_LENGTH:
9292
raise ValueError("Sequence length overflow: Generated text length (GENERATION_PROMPT_LEN + MAX_NEW_TOKENS) "
@@ -117,10 +117,11 @@ def objective(trial: optuna.Trial) -> float:
117117

118118
epochs = trial.suggest_int('epochs', 50, 75)
119119

120-
batch_size = 10 # trial.suggest_int('batch_size', 5, 10)
121-
120+
batch_size = 5 # trial.suggest_int('batch_size', 5, 10)
121+
122122
gradient_accumulation_steps = trial.suggest_int('gradient_accumulation_steps', 1, 7)
123123

124+
124125
# Level constraints - ensure max >= min by setting min of max to value of min
125126
minimum_levels = 2 # trial.suggest_int('minimum_levels', 1, 3)
126127
maximum_levels = 2 # trial.suggest_int('maximum_levels', minimum_levels, 3)
@@ -1159,8 +1160,52 @@ def test_text(test_prompt: str, max_new_tokens: int, sample_number: int, result:
11591160
'repetition_penalty': None,
11601161
'presence_penalty': 1.4,
11611162
'frequency_penalty': 1.4
1163+
},
1164+
{
1165+
'max_new_tokens': max_new_tokens,
1166+
'temperature': 0.6,
1167+
'top_k': 40,
1168+
'top_p': 0.96,
1169+
'repetition_penalty': None,
1170+
'presence_penalty': 1.4,
1171+
'frequency_penalty': 1.4
1172+
},
1173+
{
1174+
'max_new_tokens': max_new_tokens,
1175+
'temperature': 0.7,
1176+
'top_k': 45,
1177+
'top_p': 0.97,
1178+
'repetition_penalty': None,
1179+
'presence_penalty': 1.4,
1180+
'frequency_penalty': 1.3
1181+
}, #
1182+
{
1183+
'max_new_tokens': max_new_tokens,
1184+
'temperature': 0.6,
1185+
'top_k': 75,
1186+
'top_p': 0.99,
1187+
'repetition_penalty': None,
1188+
'presence_penalty': 1.4,
1189+
'frequency_penalty': 1.4
1190+
},
1191+
{
1192+
'max_new_tokens': max_new_tokens,
1193+
'temperature': 0.65,
1194+
'top_k': 75,
1195+
'top_p': 0.985,
1196+
'repetition_penalty': None,
1197+
'presence_penalty': 1.4,
1198+
'frequency_penalty': 1.4
1199+
},
1200+
{
1201+
'max_new_tokens': max_new_tokens,
1202+
'temperature': 0.8,
1203+
'top_k': 75,
1204+
'top_p': 0.99,
1205+
'repetition_penalty': None,
1206+
'presence_penalty': 0.7,
1207+
'frequency_penalty': 0.7
11621208
}
1163-
11641209
]
11651210
# Default cases, no params
11661211
response_1 = response = complete_text_greedy(text=test_prompt, max_new_tokens=max_new_tokens)
@@ -1191,11 +1236,18 @@ def test_text(test_prompt: str, max_new_tokens: int, sample_number: int, result:
11911236

11921237
prompt_samples = [
11931238
"I saw the sun and it was as shining on the",
1194-
"And God said to Moses:",
1195-
"In the beginning God created the ",
1196-
"And the earth was without form, and",
1239+
# "And God said to Moses:",
1240+
# "In the beginning God created the ",
1241+
# "And the earth was without form, and",
11971242
"And God said, Let there be light: and there ",
1198-
"Shall we all go to the river and"
1243+
# "Shall we all go to the river and"
1244+
# "Try to",
1245+
# "You must go and",
1246+
"In the beginning God created the heavens",
1247+
# "The earth was formless and empty, with darkness over",
1248+
# "God called the light 'day' and the darkness 'night,' marking evening and morning",
1249+
# "God called the expanse 'sky,' and there was",
1250+
# "The earth brought forth grass, seed-bearing"
11991251
]
12001252

12011253

0 commit comments

Comments
 (0)