Azure: Allow modular publishing

This commit introduces a new feature for Azure named "modular publish",
which allows the library to only submit the changed technical config for
a given offer/plan to "preview/live" instead of the whole offer like
before.

The new feature is disabled by default in order to keep the library
consistent with its past behavior, being possible to activate it by
setting the value `modular_push = True` on `AzurePublishingMetadata`.

Refers to SPSTRAT-604

Signed-off-by: Jonathan Gangi <[email protected]>

Author: JAVGan

cloudpub/ms_azure/service.py

@@ -417,7 +417,9 @@ def diff_offer(self, product: Product, first_target="preview") -> DeepDiff:
         remote = self.get_product(product.id, first_target=first_target)
         return DeepDiff(remote.to_json(), product.to_json(), exclude_regex_paths=self.DIFF_EXCLUDES)
 
-    def submit_to_status(self, product_id: str, status: str) -> ConfigureStatus:
+    def submit_to_status(
+        self, product_id: str, status: str, resources: Optional[List[AzureResource]] = None
+    ) -> ConfigureStatus:
         """
         Send a submission request to Microsoft with a new Product status.
 
@@ -426,6 +428,8 @@ def submit_to_status(self, product_id: str, status: str) -> ConfigureStatus:
                 The product ID to submit the new status.
             status (str)
                 The new status: 'preview' or 'live'
+            resources (optional(list(AzureRerouce)))
+                Additional resources for modular push.
         Returns:
             The response from configure request.
         """
@@ -446,9 +450,12 @@ def submit_to_status(self, product_id: str, status: str) -> ConfigureStatus:
 
         # Update the status with the expected one
         submission.target.targetType = status
+        cfg_res: List[AzureResource] = [submission]
+        if resources:
+            log.info("Performing a modular push to \"%s\" for \"%s\"", status, product_id)
+            cfg_res = resources + cfg_res
         log.debug("Set the status \"%s\" to submission.", status)
-
-        return self.configure(resources=[submission])
+        return self.configure(resources=cfg_res)
 
     @retry(
         wait=wait_fixed(300),
@@ -501,6 +508,54 @@ def get_plan_tech_config(self, product: Product, plan: PlanSummary) -> VMIPlanTe
         )
         return tconfigs[0]  # It should have only one VMIPlanTechConfig per plan.
 
+    def get_modular_resources_to_publish(
+        self, product: Product, tech_config: VMIPlanTechConfig
+    ) -> List[AzureResource]:
+        """Return the required resources for a modular publishing.
+
+        According to Microsoft docs:
+        "For a modular publish, all resources are required except for the product level details
+        (for example, listing, availability, packages, reseller) as applicable to your
+        product type."
+
+        Args:
+            product (Product): The original product to filter the resources from
+            tech_config (VMIPlanTechConfig): The updated tech config to publish
+
+        Returns:
+            List[AzureResource]: _description_
+        """
+        # The following resources shouldn't be required:
+        # -> customer-leads
+        # -> test-drive
+        # -> property
+        # -> *listing*
+        # -> reseller
+        # -> price-and-availability-*
+        # NOTE: The "submission" resource will be already added by the "submit_to_status" method
+        #
+        # With that it needs only the related "product" and "plan" resources alongisde the
+        # updated tech_config
+        product_id = tech_config.product_id
+        plan_id = tech_config.plan_id
+        prod_res = cast(
+            List[ProductSummary],
+            [
+                prd
+                for prd in self.filter_product_resources(product=product, resource="product")
+                if prd.id == product_id
+            ],
+        )[0]
+        plan_res = cast(
+            List[PlanSummary],
+            [
+                pln
+                for pln in self.filter_product_resources(product=product, resource="plan")
+                if pln.id == plan_id
+            ],
+        )[0]
+        return [prod_res, plan_res, tech_config]
+
     def _is_submission_in_preview(self, current: ProductSubmission) -> bool:
         """Return True if the latest submission state is "preview", False otherwise.
 
@@ -528,17 +583,21 @@ def _is_submission_in_preview(self, current: ProductSubmission) -> bool:
         stop=stop_after_attempt(3),
         reraise=True,
     )
-    def _publish_preview(self, product: Product, product_name: str) -> None:
+    def _publish_preview(
+        self, product: Product, product_name: str, resources: Optional[List[AzureResource]] = None
+    ) -> None:
         """
         Submit the product to 'preview'  if it's not already in this state.
 
         This is required to execute the validation pipeline on Azure side.
 
         Args:
             product
-                The product with changes to publish live
+                The product with changes to publish to preview
             product_name
                 The product name to display in logs.
+            resources:
+                Additional resources for modular push.
         """
         # We just want to set the ProductSubmission to 'preview' if it's not in this status.
         #
@@ -553,14 +612,14 @@ def _publish_preview(self, product: Product, product_name: str) -> None:
             log.info("The product \"%s\" is already set to preview", product_name)
             return
 
-        res = self.submit_to_status(product_id=product.id, status='preview')
+        res = self.submit_to_status(product_id=product.id, status='preview', resources=resources)
 
         if res.job_result != 'succeeded' or not self.get_submission_state(
             product.id, state="preview"
         ):
             errors = "\n".join(res.errors)
             failure_msg = (
-                f"Failed to submit the product {product.id} to preview. "
+                f"Failed to submit the product {product_name} ({product.id}) to preview. "
                 f"Status: {res.job_result} Errors: {errors}"
             )
             raise RuntimeError(failure_msg)
@@ -587,7 +646,7 @@ def _publish_live(self, product: Product, product_name: str) -> None:
         if res.job_result != 'succeeded' or not self.get_submission_state(product.id, state="live"):
             errors = "\n".join(res.errors)
             failure_msg = (
-                f"Failed to submit the product {product.id} to live. "
+                f"Failed to submit the product {product_name} ({product.id}) to live. "
                 f"Status: {res.job_result} Errors: {errors}"
             )
             raise RuntimeError(failure_msg)
@@ -705,7 +764,13 @@ def publish(self, metadata: AzurePublishingMetadata) -> None:
                 logdiff(self.diff_offer(product))
                 self.ensure_can_publish(product.id)
 
-                self._publish_preview(product, product_name)
+                # According to the documentation we only need to pass the
+                # required resources for modular publish on "preview"
+                # https://learn.microsoft.com/en-us/partner-center/marketplace-offers/product-ingestion-api#method-2-publish-specific-draft-resources-also-known-as-modular-publish  # noqa: E501
+                modular_resources = None
+                if metadata.modular_push:
+                    modular_resources = self.get_modular_resources_to_publish(product, tech_config)
+                self._publish_preview(product, product_name, resources=modular_resources)
                 self._publish_live(product, product_name)
 
         log.info(

cloudpub/ms_azure/utils.py

@@ -54,6 +54,11 @@ def __init__(
             check_base_sas_only (bool, optional):
                 Indicates to skip checking SAS parameters when set as ``True``.
                 Default to ``False``
+            modular_push (bool, optional):
+                Indicate whether to perform a modular push or not.
+                The modular push causes the effect to only publish
+                the changed plan instead of the whole offer to preview/live.
+                Default to ``False``.
             **kwargs
                 Arguments for :class:`~cloudpub.common.PublishingMetadata`.
         """
@@ -64,6 +69,7 @@ def __init__(
         self.recommended_sizes = recommended_sizes or []
         self.legacy_sku_id = kwargs.pop("legacy_sku_id", None)
         self.check_base_sas_only = kwargs.pop("check_base_sas_only", False)
+        self.modular_push = kwargs.pop("modular_push", None) or False
 
         if generation == "V1" or not support_legacy:
             self.legacy_sku_id = None

tests/ms_azure/test_service.py

@@ -868,7 +868,10 @@ def test_publish_preview_success_on_retry(
         azure_service._publish_preview(product_obj, "test-product")
 
         mock_subst.assert_has_calls(
-            [mock.call(product_id=product_obj.id, status="preview") for _ in range(3)]
+            [
+                mock.call(product_id=product_obj.id, status="preview", resources=None)
+                for _ in range(3)
+            ]
         )
         mock_getsubst.assert_has_calls(
             [mock.call(product_obj.id, state="preview") for _ in range(3)]
@@ -900,7 +903,7 @@ def test_publish_preview_fail_on_retry(
         # Remove the retry sleep
         azure_service._publish_preview.retry.sleep = mock.Mock()  # type: ignore
         expected_err = (
-            f"Failed to submit the product {product_obj.id} to preview. "
+            f"Failed to submit the product test-product \\({product_obj.id}\\) to preview. "
             "Status: failed Errors: failure1\nfailure2"
         )
 
@@ -960,7 +963,7 @@ def test_publish_live_fail_on_retry(
         # Remove the retry sleep
         azure_service._publish_live.retry.sleep = mock.Mock()  # type: ignore
         expected_err = (
-            f"Failed to submit the product {product_obj.id} to live. "
+            f"Failed to submit the product test-product \\({product_obj.id}\\) to live. "
             "Status: failed Errors: failure1\nfailure2"
         )
 
@@ -1417,7 +1420,7 @@ def test_publish_live_x64_only(
         mock_diff_offer.assert_called_once_with(product_obj)
         mock_configure.assert_called_once_with(resources=[technical_config_obj])
         submit_calls = [
-            mock.call(product_id=product_obj.id, status="preview"),
+            mock.call(product_id=product_obj.id, status="preview", resources=None),
             mock.call(product_id=product_obj.id, status="live"),
         ]
         mock_submit.assert_has_calls(submit_calls)
@@ -1509,7 +1512,7 @@ def test_publish_live_arm64_only(
         mock_diff_offer.assert_called_once_with(product_obj)
         mock_configure.assert_called_once_with(resources=[technical_config_obj])
         submit_calls = [
-            mock.call(product_id=product_obj.id, status="preview"),
+            mock.call(product_id=product_obj.id, status="preview", resources=None),
             mock.call(product_id=product_obj.id, status="live"),
         ]
         mock_submit.assert_has_calls(submit_calls)
@@ -1637,3 +1640,105 @@ def test_publish_live_when_state_is_preview(
             'Updating the technical configuration for "example-product/plan-1" on "preview".'
             not in caplog.text
         )
+
+    @mock.patch("cloudpub.ms_azure.AzureService.configure")
+    def test_publish_live_modular_push(
+        self,
+        mock_configure: mock.MagicMock,
+        token: Dict[str, Any],
+        auth_dict: Dict[str, Any],
+        configure_success_response: Dict[str, Any],
+        product: Dict[str, Any],
+        products_list: Dict[str, Any],
+        product_summary: Dict[str, Any],
+        technical_config: Dict[str, Any],
+        submission: Dict[str, Any],
+        product_summary_obj: ProductSummary,
+        plan_summary_obj: PlanSummary,
+        metadata_azure_obj: mock.MagicMock,
+        gen2_image: Dict[str, Any],
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        """Ensure a modular publish works as intended."""
+        # Prepare testing data
+        metadata_azure_obj.keepdraft = False
+        metadata_azure_obj.destination = "example-product/plan-1"
+        metadata_azure_obj.modular_push = True
+
+        # Set the complementary submission states
+        submission_preview = deepcopy(submission)
+        submission_preview.update({"target": {"targetType": "preview"}})
+        submission_live = deepcopy(submission)
+        submission_live.update({"target": {"targetType": "live"}})
+        mock_configure.return_value = ConfigureStatus.from_json(configure_success_response)
+
+        # Expected results
+        new_dv = {
+            "version_number": metadata_azure_obj.disk_version,
+            "vm_images": [
+                {
+                    "imageType": "x64Gen2",
+                    "source": {
+                        "sourceType": "sasUri",
+                        "osDisk": {"uri": "https://foo.com/bar/image.vhd"},
+                        "dataDisks": [],
+                    },
+                }
+            ],
+            "lifecycle_state": "generallyAvailable",
+        }
+        dvs = technical_config["vmImageVersions"] + [new_dv]
+        new_tc = deepcopy(technical_config)
+        new_tc["vmImageVersions"] = dvs
+        expected_tc = VMIPlanTechConfig.from_json(new_tc)
+        expected_modular_resources = [
+            product_summary_obj,
+            plan_summary_obj,
+            expected_tc,
+            ProductSubmission.from_json(submission_preview),
+        ]
+
+        # Constants
+        login_url = "https://login.microsoftonline.com/foo/oauth2/token"
+        base_url = "https://graph.microsoft.com/rp/product-ingestion"
+        product_id = str(product_summary['id']).split("/")[-1]
+
+        # Test
+        with caplog.at_level(logging.INFO):
+            with requests_mock.Mocker() as m:
+                m.post(login_url, json=token)
+                m.get(f"{base_url}/product", json=products_list)
+                m.get(f"{base_url}/resource-tree/product/{product_id}", json=product)
+                m.get(
+                    f"{base_url}/submission/{product_id}",
+                    [
+                        {"json": {"value": [submission]}},  # ensure_can_publish call "preview"
+                        {"json": {"value": [submission]}},  # ensure_can_publish call "live"
+                        {"json": {"value": [submission]}},  # push_preview: call submit_status
+                        {"json": {"value": [submission_preview]}},  # push_preview: check result
+                        {"json": {"value": [submission_preview]}},  # push_live: call submit_status
+                        {"json": {"value": [submission_live]}},  # push_live: check result
+                    ],
+                )
+                azure_svc = AzureService(auth_dict)
+                azure_svc.publish(metadata=metadata_azure_obj)
+
+        # Present messages
+        assert "The DiskVersion doesn't exist, creating one from scratch." in caplog.text
+        assert (
+            "Found the following offer diff before publishing:\n"
+            "Item root['resources'][1]['vmImageVersions'][1] added to iterable."
+        ) in caplog.text
+        assert (
+            'Updating the technical configuration for "example-product/plan-1" on "preview".'
+            in caplog.text
+        )
+        assert (
+            'Performing a modular push to "preview" for "ffffffff-ffff-ffff-ffff-ffffffffffff"'
+            in caplog.text
+        )
+
+        # Configure request
+        mock_configure.assert_has_calls(
+            [mock.call(resources=[expected_tc]), mock.call(resources=expected_modular_resources)]
+        )