make variable name consistent

Author: R-Palazzo

sdgym/_run_benchmark/run_benchmark.py

@@ -7,13 +7,13 @@
 
 
 def main():
-    aws_key = os.getenv('AWS_ACCESS_KEY_ID')
-    aws_secret = os.getenv('AWS_SECRET_ACCESS_KEY')
+    aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
+    aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
     for synthesizer in ['GaussianCopulaSynthesizer', 'TVAESynthesizer']:
         benchmark_single_table_aws(
             output_destination=run_benchmark.OUTPUT_DESTINATION_AWS,
-            aws_access_key_id=aws_key,
-            aws_secret_access_key=aws_secret,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
             synthesizers=[synthesizer],
             sdv_datasets=datasets,
             compute_privacy_score=False,

sdgym/_run_benchmark/upload_benchmark_results.py

@@ -87,14 +87,16 @@ def upload_results(aws_access_key_id, aws_secret_access_key, run_name, s3_client
 
 
 def main():
-    aws_key = os.getenv('AWS_ACCESS_KEY_ID')
-    aws_secret = os.getenv('AWS_SECRET_ACCESS_KEY')
-    run_name, s3_client, bucket, prefix = get_run_name_and_s3_vars(aws_key, aws_secret)
+    aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
+    aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
+    run_name, s3_client, bucket, prefix = get_run_name_and_s3_vars(
+        aws_access_key_id, aws_secret_access_key
+    )
     if upload_already_done(s3_client, bucket, prefix, run_name):
         LOGGER.info('Benchmark results have already been uploaded. Exiting.')
         sys.exit(0)
 
-    upload_results(aws_key, aws_secret, run_name, s3_client, bucket, prefix)
+    upload_results(aws_access_key_id, aws_secret_access_key, run_name, s3_client, bucket, prefix)
 
 
 if __name__ == '__main__':

sdgym/benchmark.py

@@ -242,11 +242,25 @@ def _generate_job_args_list(
     synthesizers = get_synthesizers(synthesizers + custom_synthesizers)
 
     # Get list of dataset paths
-    sdv_datasets = [] if sdv_datasets is None else get_dataset_paths(datasets=sdv_datasets)
+    aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
+    aws_secret_access_key_key = os.getenv('AWS_SECRET_ACCESS_KEY')
+    sdv_datasets = (
+        []
+        if sdv_datasets is None
+        else get_dataset_paths(
+            datasets=sdv_datasets,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key_key,
+        )
+    )
     additional_datasets = (
         []
         if additional_datasets_folder is None
-        else get_dataset_paths(bucket=additional_datasets_folder)
+        else get_dataset_paths(
+            bucket=additional_datasets_folder,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key_key,
+        )
     )
     datasets = sdv_datasets + additional_datasets
     synthesizer_names = [synthesizer['name'] for synthesizer in synthesizers]

sdgym/cli/__main__.py

@@ -98,12 +98,16 @@ def _download_datasets(args):
     datasets = args.datasets
     if not datasets:
         datasets = sdgym.datasets.get_available_datasets(
-            args.bucket, args.aws_key, args.aws_secret
+            args.bucket, args.aws_access_key_id, args.aws_secret_access_key
         )['name']
 
     for dataset in tqdm.tqdm(datasets):
         sdgym.datasets.load_dataset(
-            dataset, args.datasets_path, args.bucket, args.aws_key, args.aws_secret
+            dataset,
+            args.datasets_path,
+            args.bucket,
+            args.aws_access_key_id,
+            args.aws_secret_access_key,
         )
 
 
@@ -114,7 +118,9 @@ def _list_downloaded(args):
 
 
 def _list_available(args):
-    datasets = sdgym.datasets.get_available_datasets(args.bucket, args.aws_key, args.aws_secret)
+    datasets = sdgym.datasets.get_available_datasets(
+        args.bucket, args.aws_access_key_id, args.aws_secret_access_key
+    )
     _print_table(datasets, args.sort, args.reverse, {'size': humanfriendly.format_size})
 
 
@@ -125,16 +131,16 @@ def _list_synthesizers(args):
 
 def _collect(args):
     sdgym.cli.collect.collect_results(
-        args.input_path, args.output_file, args.aws_key, args.aws_secret
+        args.input_path, args.output_file, args.aws_access_key_id, args.aws_secret_access_key
     )
 
 
 def _summary(args):
     sdgym.cli.summary.make_summary_spreadsheet(
         args.input_path,
         output_path=args.output_file,
-        aws_key=args.aws_key,
-        aws_secret=args.aws_secret,
+        aws_access_key_id=args.aws_access_key_id,
+        aws_secret_access_key=args.aws_secret_access_key,
     )
 
 

sdgym/cli/collect.py

@@ -4,7 +4,9 @@
 from sdgym.s3 import write_csv
 
 
-def collect_results(input_path, output_file=None, aws_key=None, aws_secret=None):
+def collect_results(
+    input_path, output_file=None, aws_access_key_id=None, aws_secret_access_key=None
+):
     """Collect the results in the given input directory.
 
     Write all the results into one csv file.
@@ -15,15 +17,15 @@ def collect_results(input_path, output_file=None, aws_key=None, aws_secret=None)
         output_file (str):
             If ``output_file`` is provided, the consolidated results will be written there.
             Otherwise, they will be written to ``input_path``/results.csv.
-        aws_key (str):
-            If an ``aws_key`` is provided, the given access key id will be used to read from
-            and/or write to any s3 paths.
-        aws_secret (str):
-            If an ``aws_secret`` is provided, the given secret access key will be used to read
-            from and/or write to any s3 paths.
+        aws_access_key_id (str):
+            If an ``aws_access_key_id`` is provided, the given access key id will be used
+            to read from and/or write to any s3 paths.
+        aws_secret_access_key (str):
+            If an ``aws_secret_access_key`` is provided, the given secret access key will
+            be used to read from and/or write to any s3 paths.
     """
     print(f'Reading results from {input_path}')  # noqa: T201
-    scores = read_csv_from_path(input_path, aws_key, aws_secret)
+    scores = read_csv_from_path(input_path, aws_access_key_id, aws_secret_access_key)
     scores = scores.drop_duplicates()
 
     if output_file:
@@ -32,4 +34,4 @@ def collect_results(input_path, output_file=None, aws_key=None, aws_secret=None)
         output = f'{input_path}/results.csv'
 
     print(f'Storing results at {output}')  # noqa: T201
-    write_csv(scores, output, aws_key, aws_secret)
+    write_csv(scores, output, aws_access_key_id, aws_secret_access_key)

sdgym/cli/summary.py

@@ -289,7 +289,11 @@ def _add_summary(data, modality, baselines, writer):
 
 
 def make_summary_spreadsheet(
-    results_csv_path, output_path=None, baselines=None, aws_key=None, aws_secret=None
+    results_csv_path,
+    output_path=None,
+    baselines=None,
+    aws_access_key_id=None,
+    aws_secret_access_key=None,
 ):
     """Create a spreadsheet document organizing information from results.
 
@@ -307,7 +311,7 @@ def make_summary_spreadsheet(
             Optional dict mapping modalities to a list of baseline
             model names. If not provided, a default dict is used.
     """
-    results = read_csv(results_csv_path, aws_key, aws_secret)
+    results = read_csv(results_csv_path, aws_access_key_id, aws_secret_access_key)
     data = preprocess(results)
     baselines = baselines or MODALITY_BASELINES
     output_path = output_path or re.sub('.csv$', '.xlsx', results_csv_path)
@@ -319,4 +323,4 @@ def make_summary_spreadsheet(
         _add_summary(df, modality, modality_baselines, writer)
 
     writer.save()
-    write_file(output.getvalue(), output_path, aws_key, aws_secret)
+    write_file(output.getvalue(), output_path, aws_access_key_id, aws_secret_access_key)

sdgym/cli/utils.py

@@ -11,25 +11,25 @@
 from sdgym.s3 import get_s3_client, is_s3_path, parse_s3_path
 
 
-def read_file(path, aws_key, aws_secret):
+def read_file(path, aws_access_key_id, aws_secret_access_key):
     """Read file from path.
 
     The path can either be a local path or an s3 directory.
 
     Args:
         path (str):
             The path to the file.
-        aws_key (str):
+        aws_access_key_id (str):
             The access key id that will be used to communicate with s3, if provided.
-        aws_secret (str):
+        aws_secret_access_key (str):
             The secret access key that will be used to communicate with s3, if provided.
 
     Returns:
         bytes:
             The content of the file in bytes.
     """
     if is_s3_path(path):
-        s3 = get_s3_client(aws_key, aws_secret)
+        s3 = get_s3_client(aws_access_key_id, aws_secret_access_key)
         bucket_name, key = parse_s3_path(path)
         obj = s3.get_object(Bucket=bucket_name, Key=key)
         contents = obj['Body'].read()
@@ -40,28 +40,28 @@ def read_file(path, aws_key, aws_secret):
     return contents
 
 
-def read_csv(path, aws_key, aws_secret):
+def read_csv(path, aws_access_key_id, aws_secret_access_key):
     """Read csv file from path.
 
     The path can either be a local path or an s3 directory.
 
     Args:
         path (str):
             The path to the csv file.
-        aws_key (str):
+        aws_access_key_id (str):
             The access key id that will be used to communicate with s3, if provided.
-        aws_secret (str):
+        aws_secret_access_key (str):
             The secret access key that will be used to communicate with s3, if provided.
 
     Returns:
         pandas.DataFrame:
             A DataFrame containing the contents of the csv file.
     """
-    contents = read_file(path, aws_key, aws_secret)
+    contents = read_file(path, aws_access_key_id, aws_secret_access_key)
     return pd.read_csv(io.BytesIO(contents))
 
 
-def read_csv_from_path(path, aws_key, aws_secret):
+def read_csv_from_path(path, aws_access_key_id, aws_secret_access_key):
     """Read all csv content within a path.
 
     All csv content within a path will be read and returned in a
@@ -70,9 +70,9 @@ def read_csv_from_path(path, aws_key, aws_secret):
     Args:
         path (str):
             The path to read from, which can be either local or an s3 path.
-        aws_key (str):
+        aws_access_key_id (str):
             The access key id that will be used to communicate with s3, if provided.
-        aws_secret (str):
+        aws_secret_access_key (str):
             The secret access key that will be used to communicate with s3, if provided.
 
     Returns:
@@ -81,13 +81,17 @@ def read_csv_from_path(path, aws_key, aws_secret):
     """
     csv_contents = []
     if is_s3_path(path):
-        s3 = get_s3_client(aws_key, aws_secret)
+        s3 = get_s3_client(aws_access_key_id, aws_secret_access_key)
         bucket_name, key_prefix = parse_s3_path(path)
         resp = s3.list_objects(Bucket=bucket_name, Prefix=key_prefix)
         csv_files = [f for f in resp['Contents'] if f['Key'].endswith('.csv')]
         for csv_file in csv_files:
             csv_file_key = csv_file['Key']
-            csv_contents.append(read_csv(f's3://{bucket_name}/{csv_file_key}', aws_key, aws_secret))
+            csv_contents.append(
+                read_csv(
+                    f's3://{bucket_name}/{csv_file_key}', aws_access_key_id, aws_secret_access_key
+                )
+            )
 
     else:
         run_path = pathlib.Path(path)