Feat: multi-targets support: unit tests

Author: ligangty

charon/pkgs/maven.py

@@ -303,9 +303,12 @@ def handle_maven_uploading(
         # Question: should we exit here?
 
     # 4. Do uploading
-    logger.info("Start uploading files to s3")
     s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
     targets_ = [(target[1], remove_prefix(target[2], "/")) for target in targets]
+    logger.info(
+        "Start uploading files to s3 buckets: %s",
+        [target[0] for target in targets_]
+    )
     failed_files = s3_client.upload_files(
         file_paths=valid_mvn_paths,
         targets=targets_,
@@ -317,8 +320,8 @@ def handle_maven_uploading(
     succeeded = True
     for target in targets:
         # 5. Do manifest uploading
-        logger.info("Start uploading manifest to s3")
         manifest_folder = target[0]
+        logger.info("Start uploading manifest to s3 bucket %s", manifest_bucket_name)
         if not manifest_bucket_name:
             logger.warning(
                 'Warning: No manifest bucket is provided, will ignore the process of manifest '
@@ -347,7 +350,7 @@ def handle_maven_uploading(
         # 7. Upload all maven-metadata.xml
         if META_FILE_GEN_KEY in meta_files:
             logger.info("Start updating maven-metadata.xml to s3 bucket %s", bucket_)
-            (_, _failed_metas) = s3_client.upload_metadatas(
+            _failed_metas = s3_client.upload_metadatas(
                 meta_file_paths=meta_files[META_FILE_GEN_KEY],
                 target=(bucket_, prefix__),
                 product=None,
@@ -373,7 +376,7 @@ def handle_maven_uploading(
                     __hash_decorate_metadata(top_level, ARCHETYPE_CATALOG_FILENAME)
                 )
                 logger.info("Start updating archetype-catalog.xml to s3 bucket %s", bucket_)
-                (_, _failed_metas) = s3_client.upload_metadatas(
+                _failed_metas = s3_client.upload_metadatas(
                     meta_file_paths=archetype_files,
                     target=(bucket_, prefix__),
                     product=None,
@@ -394,7 +397,7 @@ def handle_maven_uploading(
             logger.info("Index files generation done.\n")
 
             logger.info("Start updating index files to s3 bucket %s", bucket_)
-            (_, _failed_metas) = s3_client.upload_metadatas(
+            _failed_metas = s3_client.upload_metadatas(
                 meta_file_paths=created_indexes,
                 target=(bucket_, prefix__),
                 product=None,
@@ -454,10 +457,10 @@ def handle_maven_del(
     succeeded = True
     for target in targets:
         prefix_ = remove_prefix(target[2], "/")
-        logger.info("Start deleting files from s3")
         s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
         bucket = target[1]
-        (_, failed_files) = s3_client.delete_files(
+        logger.info("Start deleting files from s3 bucket %s", bucket)
+        failed_files = s3_client.delete_files(
             valid_mvn_paths,
             target=(bucket, prefix_),
             product=prod_key,
@@ -467,12 +470,18 @@ def handle_maven_del(
 
         # 4. Delete related manifest from s3
         manifest_folder = target[0]
-        logger.info("Start deleting manifest from s3")
+        logger.info(
+            "Start deleting manifest from s3 bucket %s in folder %s",
+            manifest_bucket_name, manifest_folder
+        )
         s3_client.delete_manifest(prod_key, manifest_folder, manifest_bucket_name)
         logger.info("Manifest deletion is done\n")
 
         # 5. Use changed GA to scan s3 for metadata refreshment
-        logger.info("Start generating maven-metadata.xml files for all changed GAs")
+        logger.info(
+            "Start generating maven-metadata.xml files for all changed GAs in s3 bucket %s",
+            bucket
+        )
         meta_files = _generate_metadatas(
             s3=s3_client, bucket=bucket,
             poms=valid_poms, root=top_level,
@@ -483,7 +492,7 @@ def handle_maven_del(
 
         # 6. Upload all maven-metadata.xml. We need to delete metadata files
         # firstly for all affected GA, and then replace the theirs content.
-        logger.info("Start updating maven-metadata.xml to s3")
+        logger.info("Start updating maven-metadata.xml to s3 bucket %s", bucket)
         all_meta_files = []
         for _, files in meta_files.items():
             all_meta_files.extend(files)
@@ -495,7 +504,7 @@ def handle_maven_del(
         )
         failed_metas = meta_files.get(META_FILE_FAILED, [])
         if META_FILE_GEN_KEY in meta_files:
-            (_, _failed_metas) = s3_client.upload_metadatas(
+            _failed_metas = s3_client.upload_metadatas(
                 meta_file_paths=meta_files[META_FILE_GEN_KEY],
                 target=(bucket, prefix_),
                 product=None,
@@ -519,8 +528,8 @@ def handle_maven_del(
             archetype_files = [os.path.join(top_level, ARCHETYPE_CATALOG_FILENAME)]
             archetype_files.extend(__hash_decorate_metadata(top_level, ARCHETYPE_CATALOG_FILENAME))
             if archetype_action < 0:
-                logger.info("Start updating archetype-catalog.xml to s3")
-                (_, _failed_metas) = s3_client.delete_files(
+                logger.info("Start updating archetype-catalog.xml to s3 bucket %s", bucket)
+                _failed_metas = s3_client.delete_files(
                     file_paths=archetype_files,
                     target=(bucket, prefix_),
                     product=None,
@@ -529,7 +538,7 @@ def handle_maven_del(
                 if len(_failed_metas) > 0:
                     failed_metas.extend(_failed_metas)
             elif archetype_action > 0:
-                (_, _failed_metas) = s3_client.upload_metadatas(
+                _failed_metas = s3_client.upload_metadatas(
                     meta_file_paths=archetype_files,
                     target=(bucket, prefix_),
                     product=None,
@@ -546,8 +555,8 @@ def handle_maven_del(
             )
             logger.info("Index files generation done.\n")
 
-            logger.info("Start updating index to s3")
-            (_, _failed_index_files) = s3_client.upload_metadatas(
+            logger.info("Start updating index to s3 bucket %s", bucket)
+            _failed_index_files = s3_client.upload_metadatas(
                 meta_file_paths=created_indexes,
                 target=(bucket, prefix_),
                 product=None,
@@ -799,7 +808,7 @@ def _generate_upload_archetype_catalog(
                 "Error: Can not generate archtype-catalog.xml due to: %s", e
             )
             return 0
-        if not existed
+        if not existed:
             __gen_all_digest_files(local)
             # If there is no catalog in the bucket, just upload what we have locally
             return True

charon/pkgs/npm.py

@@ -96,9 +96,12 @@ def handle_npm_uploading(
     valid_dirs = __get_path_tree(valid_paths, target_dir)
 
     # main_target = targets[0]
-    logger.info("Start uploading files to s3")
     client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
     targets_ = [(target[1], remove_prefix(target[2], "/")) for target in targets]
+    logger.info(
+        "Start uploading files to s3 buckets: %s",
+        [target[0] for target in targets_]
+    )
     failed_files = client.upload_files(
         file_paths=valid_paths,
         targets=targets_,
@@ -110,7 +113,7 @@ def handle_npm_uploading(
     succeeded = True
     for target in targets:
         manifest_folder = target[0]
-        logger.info("Start uploading manifest to s3")
+        logger.info("Start uploading manifest to s3 bucket %s", manifest_bucket_name)
         if not manifest_bucket_name:
             logger.warning(
                 'Warning: No manifest bucket is provided, will ignore the process of manifest '
@@ -125,15 +128,18 @@ def handle_npm_uploading(
 
         bucket_ = target[1]
         prefix__ = remove_prefix(target[2], "/")
-        logger.info("Start generating package.json for package: %s", package_metadata.name)
+        logger.info(
+            "Start generating package.json for package: %s in s3 bucket %s",
+            package_metadata.name, bucket_
+        )
         meta_files = _gen_npm_package_metadata_for_upload(
             client, bucket_, target_dir, package_metadata, prefix__
         )
         logger.info("package.json generation done\n")
 
         failed_metas = []
         if META_FILE_GEN_KEY in meta_files:
-            _, _failed_metas = client.upload_metadatas(
+            _failed_metas = client.upload_metadatas(
                 meta_file_paths=[meta_files[META_FILE_GEN_KEY]],
                 target=(bucket_, prefix__),
                 product=None,
@@ -145,14 +151,14 @@ def handle_npm_uploading(
         # this step generates index.html for each dir and add them to file list
         # index is similar to metadata, it will be overwritten everytime
         if do_index:
-            logger.info("Start generating index files to s3")
+            logger.info("Start generating index files to s3 bucket %s", bucket_)
             created_indexes = indexing.generate_indexes(
                 PACKAGE_TYPE_NPM, target_dir, valid_dirs, client, bucket_, prefix__
             )
             logger.info("Index files generation done.\n")
 
-            logger.info("Start updating index files to s3")
-            (_, _failed_metas) = client.upload_metadatas(
+            logger.info("Start updating index files to s3 bucket %s", bucket_)
+            _failed_metas = client.upload_metadatas(
                 meta_file_paths=created_indexes,
                 target=(bucket_, prefix__),
                 product=None,
@@ -197,32 +203,37 @@ def handle_npm_del(
 
     valid_dirs = __get_path_tree(valid_paths, target_dir)
 
-    logger.info("Start deleting files from s3")
     client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
     succeeded = True
     for target in targets:
         bucket = target[1]
         prefix_ = remove_prefix(target[2], "/")
-        
-        _, failed_files = client.delete_files(
+        logger.info("Start deleting files from s3 bucket %s", bucket)
+        failed_files = client.delete_files(
             file_paths=valid_paths,
             target=(bucket, prefix_),
             product=product, root=target_dir
         )
         logger.info("Files deletion done\n")
 
         manifest_folder = target[0]
-        logger.info("Start deleting manifest from s3")
+        logger.info(
+            "Start deleting manifest from s3 bucket %s",
+            manifest_bucket_name
+        )
         client.delete_manifest(product, manifest_folder, manifest_bucket_name)
         logger.info("Manifest deletion is done\n")
 
-        logger.info("Start generating package.json for package: %s", package_name_path)
+        logger.info(
+            "Start generating package.json for package: %s in bucket %s",
+            package_name_path, bucket
+        )
         meta_files = _gen_npm_package_metadata_for_del(
             client, bucket, target_dir, package_name_path, prefix_
         )
         logger.info("package.json generation done\n")
 
-        logger.info("Start uploading package.json to s3")
+        logger.info("Start uploading package.json to s3 bucket %s", bucket)
         all_meta_files = []
         for _, file in meta_files.items():
             all_meta_files.append(file)
@@ -233,7 +244,7 @@ def handle_npm_del(
         )
         failed_metas = []
         if META_FILE_GEN_KEY in meta_files:
-            _, _failed_metas = client.upload_metadatas(
+            _failed_metas = client.upload_metadatas(
                 meta_file_paths=[meta_files[META_FILE_GEN_KEY]],
                 target=(bucket, prefix_),
                 product=None,
@@ -243,14 +254,17 @@ def handle_npm_del(
         logger.info("package.json uploading done")
 
         if do_index:
-            logger.info("Start generating index files for all changed entries")
+            logger.info(
+                "Start generating index files for all changed entries for bucket %s",
+                bucket
+            )
             created_indexes = indexing.generate_indexes(
                 PACKAGE_TYPE_NPM, target_dir, valid_dirs, client, bucket, prefix_
             )
             logger.info("Index files generation done.\n")
 
-            logger.info("Start updating index to s3")
-            (_, _failed_index_files) = client.upload_metadatas(
+            logger.info("Start updating index to s3 bucket %s", bucket)
+            _failed_index_files = client.upload_metadatas(
                 meta_file_paths=created_indexes,
                 target=(bucket, prefix_),
                 product=None,

charon/storage.py

@@ -156,7 +156,7 @@ async def path_upload_handler(
                 main_file_object: s3.Object = main_bucket.Object(main_path_key)
                 existed = False
                 try:
-                    existed = await self.__run_async(self.__file_exists, file_object)
+                    existed = await self.__run_async(self.__file_exists, main_file_object)
                 except (ClientError, HTTPClientError) as e:
                     logger.error(
                         "Error: file existence check failed due to error: %s", e
@@ -258,8 +258,8 @@ async def handle_existed(
             )
             if checksum != "" and checksum.strip() != file_sha1:
                 logger.warning('Error: checksum check failed. The file %s is '
-                             'different from the one in S3 bucket %s. Product: %s',
-                             path_key, bucket_name, product)
+                               'different from the one in S3 bucket %s. Product: %s',
+                               path_key, bucket_name, product)
                 failed_paths.append(file_path)
                 return False
             (prods, no_error) = await self.__run_async(
@@ -293,7 +293,7 @@ async def __copy_between_bucket(
     ) -> bool:
         logger.info(
             "Copying file %s from bucket %s to target %s as %s",
-            source_key, source, target, target_key)
+            source_key, source, target.name, target_key)
         copy_source = {
             'Bucket': source,
             'Key': source_key
@@ -319,7 +319,7 @@ def upload_metadatas(
         self, meta_file_paths: List[str],
         target: Tuple[str, str],
         product: Optional[str] = None, root="/"
-    ) -> Tuple[List[str], List[str]]:
+    ) -> List[str]:
         """ Upload a list of metadata files to s3 bucket. This function is very similar to
         upload_files, except:
             * The metadata files will always be overwritten for each uploading
@@ -328,7 +328,6 @@ def upload_metadatas(
         """
         bucket_name = target[0]
         bucket = self.__get_bucket(bucket_name)
-        uploaded_files = []
 
         async def path_upload_handler(
             full_file_path: str, path: str, index: int,
@@ -404,7 +403,6 @@ async def path_upload_handler(
                                 failed.append(full_file_path)
                                 return
                     logger.debug('Updated metadata %s to bucket %s', path, bucket_name)
-                    uploaded_files.append(path_key)
                 except (ClientError, HTTPClientError) as e:
                     logger.error(
                         "ERROR: file %s not uploaded to bucket"
@@ -413,11 +411,11 @@ async def path_upload_handler(
                     )
                     failed.append(full_file_path)
 
-        return (uploaded_files, self.__do_path_cut_and(
+        return self.__do_path_cut_and(
             file_paths=meta_file_paths,
             path_handler=self.__path_handler_count_wrapper(path_upload_handler),
             root=root
-        ))
+        )
 
     def upload_manifest(
             self, manifest_name: str, manifest_full_path: str, target: str,
@@ -441,7 +439,7 @@ def upload_manifest(
     def delete_files(
         self, file_paths: List[str], target: Tuple[str, str],
         product: Optional[str], root="/"
-    ) -> Tuple[List[str], List[str]]:
+    ) -> List[str]:
         """ Deletes a list of files to s3 bucket. * Use the cut down file path as s3 key. The cut
         down way is move root from the file path if it starts with root. Example: if file_path is
         /tmp/maven-repo/org/apache/.... and root is /tmp/maven-repo Then the key will be
@@ -455,8 +453,6 @@ def delete_files(
         bucket_name = target[0]
         bucket = self.__get_bucket(bucket_name)
 
-        deleted_files = []
-
         async def path_delete_handler(
             full_file_path: str, path: str, index: int,
             total: int, failed: List[str]
@@ -527,7 +523,6 @@ async def path_delete_handler(
                                     failed.append(full_file_path)
                                     return
                             logger.info("Deleted %s from bucket %s", path, bucket_name)
-                            deleted_files.append(path)
                             return
                         except (ClientError, HTTPClientError) as e:
                             logger.error(
@@ -550,7 +545,7 @@ async def path_delete_handler(
             root=root
         )
 
-        return (deleted_files, failed_files)
+        return failed_files
 
     def delete_manifest(self, product_key: str, target: str, manifest_bucket_name: str):
         if not manifest_bucket_name: