Minor refactoring items

Author: energydrink9

src/stem_continuation_dataset_generator/cluster.py

@@ -6,7 +6,6 @@
 from stem_continuation_dataset_generator.constants import DASK_CLUSTER_NAME
 
 NUM_WORKERS = [4, 50]
-BUCKET = 's3://stem-continuation-dataset'
 
 
 def get_client(

src/stem_continuation_dataset_generator/constants.py

@@ -1,8 +1,8 @@
 import os
 
 DATASET_TAGS = ['medium']
-CLEARML_DATASET_TRAINING_NAME = 'stem-continuation-dataset'
-CLEARML_DATASET_TRAINING_VERSION = '1.0.0'
+CLEARML_DATASET_NAME = 'stem_continuation_dataset'
+CLEARML_DATASET_VERSION = '1.0.0'
 DEFAULT_STEM_NAME = 'drum'
 STORAGE_BUCKET_NAME = 'stem-continuation-dataset'
 DASK_CLUSTER_NAME = 'stem-continuation-dataset-generator-cluster'

src/stem_continuation_dataset_generator/dataset.py

@@ -1,6 +1,6 @@
 
 from clearml import Dataset
-from stem_continuation_dataset_generator.constants import CLEARML_DATASET_TRAINING_NAME, CLEARML_DATASET_TRAINING_VERSION
+from stem_continuation_dataset_generator.constants import CLEARML_DATASET_NAME, CLEARML_DATASET_VERSION
 from stem_continuation_dataset_generator.utils.constants import get_clearml_project_name
 
 
@@ -16,8 +16,8 @@ def get_remote_dataset_by_id(id: str):
 def get_remote_dataset_by_tag(tag: str):
     dataset = Dataset.get(
         dataset_project=get_clearml_project_name(),
-        dataset_name=CLEARML_DATASET_TRAINING_NAME,
-        dateset_version=CLEARML_DATASET_TRAINING_VERSION,
+        dataset_name=CLEARML_DATASET_NAME,
+        dateset_version=CLEARML_DATASET_VERSION,
         dataset_tags=[tag],
         only_completed=False,  # True 
         only_published=False, 

src/stem_continuation_dataset_generator/steps/encode.py

@@ -55,10 +55,6 @@ def encode_all(source_directory: str, output_directory: str):
     ))
 
     print('Encoding audio tracks')
-    
-    # for i in range(len(params_list) - 1, 0, -1):
-    #     print(f'Processing {i} of {len(params_list)} {round(cast(float, i) / len(params_list) * 100)}')
-    #     encode(params_list[i])
 
     futures = client.map(encode, params_list, retries=2, batch_size=8)
     progress(futures)

src/stem_continuation_dataset_generator/steps/upload.py

@@ -6,7 +6,7 @@
 import multiprocessing
 import multiprocessing.pool
 
-from stem_continuation_dataset_generator.constants import CLEARML_DATASET_TRAINING_VERSION, DATASET_TAGS, get_split_files_path
+from stem_continuation_dataset_generator.constants import CLEARML_DATASET_VERSION, DATASET_TAGS, get_split_files_path
 from stem_continuation_dataset_generator.utils.utils import upload_dataset
 
 
@@ -61,7 +61,7 @@ def upload(split_files_path: str, tags: List[str]):
                 list(tqdm(pool.imap(download_file, inputs), total=len(inputs)))
 
             print(f'Uploading {set} dataset to ClearML')
-            upload_dataset(path=local_directory, version=CLEARML_DATASET_TRAINING_VERSION, tags=tags + ['final'], dataset_set=set)
+            upload_dataset(path=local_directory, version=CLEARML_DATASET_VERSION, tags=tags + ['final'], dataset_set=set)
 
 
 if __name__ == '__main__':

src/stem_continuation_dataset_generator/utils/utils.py

@@ -1,7 +1,7 @@
 from clearml import Dataset
 import numpy as np
 
-from stem_continuation_dataset_generator.constants import CLEARML_DATASET_TRAINING_NAME
+from stem_continuation_dataset_generator.constants import CLEARML_DATASET_NAME
 from stem_continuation_dataset_generator.utils.constants import get_clearml_project_name
 
 
@@ -10,7 +10,7 @@ def upload_dataset(path: str, version: str, tags: list[str] = [], dataset_set=No
     tags = [f'{dataset_set}-set'] + tags if dataset_set is not None else tags
     dataset = Dataset.create(
         dataset_project=get_clearml_project_name(), 
-        dataset_name=CLEARML_DATASET_TRAINING_NAME,
+        dataset_name=CLEARML_DATASET_NAME,
         dataset_version=version,
         dataset_tags=tags,
     )