Merge pull request #154 from yma88/main

Convert npm version-level package.json as metadata partially

Author: yma955

charon/pkgs/npm.py

@@ -102,7 +102,7 @@ def handle_npm_uploading(
 
         logger.info("Start uploading files to s3 buckets: %s", bucket_)
         failed_files = client.upload_files(
-            file_paths=valid_paths,
+            file_paths=[valid_paths[0]],
             targets=[(bucket_, prefix__)],
             product=product,
             root=target_dir
@@ -126,6 +126,21 @@ def handle_npm_uploading(
             )
             logger.info("Manifest uploading is done\n")
 
+        logger.info(
+            "Start generating version-level package.json for package: %s in s3 bucket %s",
+            package_metadata.name, bucket_
+        )
+        failed_metas = []
+        _version_metadata_path = valid_paths[1]
+        _failed_metas = client.upload_metadatas(
+            meta_file_paths=[_version_metadata_path],
+            target=(bucket_, prefix__),
+            product=product,
+            root=target_dir
+        )
+        failed_metas.extend(_failed_metas)
+        logger.info("version-level package.json uploading done")
+
         logger.info(
             "Start generating package.json for package: %s in s3 bucket %s",
             package_metadata.name, bucket_
@@ -135,7 +150,6 @@ def handle_npm_uploading(
         )
         logger.info("package.json generation done\n")
 
-        failed_metas = []
         if META_FILE_GEN_KEY in meta_files:
             _failed_metas = client.upload_metadatas(
                 meta_file_paths=[meta_files[META_FILE_GEN_KEY]],

charon/storage.py

@@ -383,15 +383,18 @@ async def path_upload_handler(
                             # NOTE: This should not happen for most cases, as most
                             # of the metadata file does not have product info. Just
                             # leave for requirement change in future
-                            (prods, no_error) = await self.__run_async(
-                                self.__get_prod_info,
-                                path_key, bucket_name
-                            )
-                            if not no_error:
-                                failed.append(full_file_path)
-                                return
-                            if no_error and product not in prods:
-                                prods.append(product)
+                            # This is now used for npm version-level package.json
+                            prods = [product]
+                            if existed:
+                                (prods, no_error) = await self.__run_async(
+                                    self.__get_prod_info,
+                                    path_key, bucket_name
+                                )
+                                if not no_error:
+                                    failed.append(full_file_path)
+                                    return
+                                if no_error and product not in prods:
+                                    prods.append(product)
                             updated = await self.__update_prod_info(
                                 path_key, bucket_name, prods
                             )

tests/test_npm_dist_gen.py

@@ -109,3 +109,46 @@ def test_dist_gen_in_multi_targets(self):
         self.assertIn("\"dist\"", merged_meta_content_client)
         self.assertIn("\"tarball\": \"https://npm2.registry.redhat.com/@babel/code-frame/-/code"
                       "-frame-7.14.5.tgz\"", merged_meta_content_client)
+
+    def test_overlapping_registry_dist_gen(self):
+        targets_ = [(None, TEST_BUCKET, None, "npm1.registry.redhat.com")]
+        test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz")
+        product_7_14_5 = "code-frame-7.14.5"
+        handle_npm_uploading(
+            test_tgz, product_7_14_5,
+            targets=targets_,
+            dir_=self.tempdir, do_index=False
+        )
+        test_bucket = self.mock_s3.Bucket(TEST_BUCKET)
+        meta_obj_client_7_14_5 = test_bucket.Object(CODE_FRAME_7_14_5_META)
+        meta_content_client_7_14_5 = str(meta_obj_client_7_14_5.get()["Body"].read(), "utf-8")
+        self.assertIn("\"dist\"", meta_content_client_7_14_5)
+        self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code"
+                      "-frame-7.14.5.tgz\"", meta_content_client_7_14_5)
+
+        merged_meta_obj_client = test_bucket.Object(CODE_FRAME_META)
+        merged_meta_content_client = str(merged_meta_obj_client.get()["Body"].read(), "utf-8")
+        self.assertIn("\"dist\"", merged_meta_content_client)
+        self.assertIn("\"tarball\": \"https://npm1.registry.redhat.com/@babel/code-frame/-/code"
+                      "-frame-7.14.5.tgz\"", merged_meta_content_client)
+
+        targets_overlapping_ = [(None, TEST_BUCKET, None, "npm1.overlapping.registry.redhat.com")]
+        test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz")
+        product_7_14_5 = "code-frame-7.14.5"
+        handle_npm_uploading(
+            test_tgz, product_7_14_5,
+            targets=targets_overlapping_,
+            dir_=self.tempdir, do_index=False
+        )
+
+        meta_obj_client_7_14_5 = test_bucket.Object(CODE_FRAME_7_14_5_META)
+        meta_content_client_7_14_5 = str(meta_obj_client_7_14_5.get()["Body"].read(), "utf-8")
+        self.assertIn("\"dist\"", meta_content_client_7_14_5)
+        self.assertIn("\"tarball\": \"https://npm1.overlapping.registry.redhat.com/@babel/code"
+                      "-frame/-/code-frame-7.14.5.tgz\"", meta_content_client_7_14_5)
+
+        merged_meta_obj_client = test_bucket.Object(CODE_FRAME_META)
+        merged_meta_content_client = str(merged_meta_obj_client.get()["Body"].read(), "utf-8")
+        self.assertIn("\"dist\"", merged_meta_content_client)
+        self.assertIn("\"tarball\": \"https://npm1.overlapping.registry.redhat.com/@babel/code"
+                      "-frame/-/code-frame-7.14.5.tgz\"", merged_meta_content_client)