Search for the catalog entities folder in the 'extensions' folder from the catalog index image, with a fallback to 'marketplace' (for backward compatibility)

Author: rm3l

docker/install-dynamic-plugins.py

@@ -1002,17 +1002,23 @@ def extract_catalog_index(catalog_index_image: str, catalog_index_mount: str, ca
     print("\t==> Successfully extracted dynamic-plugins.default.yaml from catalog index image", flush=True)
 
     print(f"\t==> Extracting extensions catalog entities to {catalog_entities_parent_dir}", flush=True)
-    marketplace_dir_from_catalog_index = os.path.join(catalog_index_temp_dir, 'catalog-entities', 'marketplace')
-    if os.path.isdir(marketplace_dir_from_catalog_index):
+
+    extensions_dir_from_catalog_index = os.path.join(catalog_index_temp_dir, 'catalog-entities', 'extensions')
+    if not os.path.isdir(extensions_dir_from_catalog_index):
+        # fallback to 'catalog-entities/marketplace' directory for backward compatibility
+        extensions_dir_from_catalog_index = os.path.join(catalog_index_temp_dir, 'catalog-entities', 'marketplace')
+
+    if os.path.isdir(extensions_dir_from_catalog_index):
         os.makedirs(catalog_entities_parent_dir, exist_ok=True)
         catalog_entities_dest = os.path.join(catalog_entities_parent_dir, 'catalog-entities')
         # Ensure the destination directory is is sync with the catalog entities from the index image
         if os.path.exists(catalog_entities_dest):
             shutil.rmtree(catalog_entities_dest, ignore_errors=True, onerror=None)
-        shutil.copytree(marketplace_dir_from_catalog_index, catalog_entities_dest, dirs_exist_ok=True)
+        shutil.copytree(extensions_dir_from_catalog_index, catalog_entities_dest, dirs_exist_ok=True)
         print("\t==> Successfully extracted extensions catalog entities from index image", flush=True)
     else:
-        print(f"\t==> WARNING: Catalog index image {catalog_index_image} does not have a 'catalog-entities/marketplace' directory", flush=True)
+        print(f"\t==> WARNING: Catalog index image {catalog_index_image} does not have neither 'catalog-entities/extensions/' nor 'catalog-entities/marketplace/' directory",
+            flush=True)
 
     return default_plugins_file
 

docker/test_install-dynamic-plugins.py

@@ -2312,7 +2312,7 @@ def mock_oci_image(self, tmp_path):
 """
         yaml_file.write_text(yaml_content)
 
-        # Create catalog entities directory structure
+        # Create catalog entities directory structure (using marketplace for backward compatibility)
         catalog_entities_dir = layer_content_dir / "catalog-entities" / "marketplace"
         catalog_entities_dir.mkdir(parents=True)
         entity_file = catalog_entities_dir / "test-entity.yaml"
@@ -2334,6 +2334,67 @@ def mock_oci_image(self, tmp_path):
             "entity_file": str(entity_file)
         }
 
+    @pytest.fixture
+    def mock_oci_image_with_extensions(self, tmp_path):
+        """Create a mock OCI image structure with extensions directory (new format)."""
+        import tarfile
+
+        # Create a temporary directory for the OCI image
+        oci_dir = tmp_path / "oci-image-extensions"
+        oci_dir.mkdir()
+
+        # Create manifest.json
+        manifest = {
+            "schemaVersion": 2,
+            "mediaType": "application/vnd.oci.image.manifest.v1+json",
+            "config": {
+                "mediaType": "application/vnd.oci.image.config.v1+json",
+                "digest": "sha256:test456",
+                "size": 100
+            },
+            "layers": [
+                {
+                    "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+                    "digest": "sha256:def789ghi012",
+                    "size": 1000
+                }
+            ]
+        }
+        manifest_path = oci_dir / "manifest.json"
+        manifest_path.write_text(json.dumps(manifest))
+
+        # Create a layer tarball with dynamic-plugins.default.yaml and catalog entities
+        layer_content_dir = tmp_path / "layer-content-extensions"
+        layer_content_dir.mkdir()
+
+        yaml_file = layer_content_dir / "dynamic-plugins.default.yaml"
+        yaml_content = """plugins:
+  - package: '@backstage/plugin-catalog'
+    integrity: sha512-test
+"""
+        yaml_file.write_text(yaml_content)
+
+        # Create catalog entities directory structure using extensions (new format)
+        catalog_entities_dir = layer_content_dir / "catalog-entities" / "extensions"
+        catalog_entities_dir.mkdir(parents=True)
+        entity_file = catalog_entities_dir / "test-entity.yaml"
+        entity_file.write_text("apiVersion: backstage.io/v1alpha1\nkind: Component\nmetadata:\n  name: test-extensions")
+
+        # Create the layer tarball
+        layer_tarball = oci_dir / "def789ghi012"
+        with create_test_tarball(layer_tarball) as tar:
+            tar.add(str(yaml_file), arcname="dynamic-plugins.default.yaml")
+            # Add catalog entities directory structure recursively
+            tar.add(str(layer_content_dir / "catalog-entities"), arcname="catalog-entities", recursive=True)
+
+        return {
+            "oci_dir": str(oci_dir),
+            "manifest_path": str(manifest_path),
+            "layer_tarball": str(layer_tarball),
+            "yaml_content": yaml_content,
+            "entity_file": str(entity_file)
+        }
+
     def test_extract_catalog_index_skopeo_not_found(self, tmp_path, mocker):
         """Test that function raises InstallException when skopeo is not available."""
         mocker.patch('shutil.which', return_value=None)
@@ -2675,8 +2736,87 @@ def test_extract_catalog_index_removes_existing_destination(self, tmp_path, mock
         assert not (entities_dir / "old-file.yaml").exists(), "Old file should not exist"
         assert not (entities_dir / "old-subdir").exists(), "Old subdirectory should not exist"
 
+    def test_extract_catalog_index_uses_extensions_directory(self, tmp_path, mocker, mock_oci_image_with_extensions, capsys):
+        """Test that extraction prefers extensions directory over marketplace."""
+        catalog_mount = tmp_path / "catalog-mount"
+        catalog_mount.mkdir()
+        catalog_entities_parent_dir = tmp_path / "entities-extensions"
+
+        mocker.patch('shutil.which', return_value='/usr/bin/skopeo')
+
+        mock_result = mocker.Mock()
+        mock_result.returncode = 0
+        mock_subprocess_run = create_mock_skopeo_copy(
+            mock_oci_image_with_extensions['manifest_path'],
+            mock_oci_image_with_extensions['layer_tarball'],
+            mock_result
+        )
+        mocker.patch('subprocess.run', side_effect=mock_subprocess_run)
+
+        result = install_dynamic_plugins.extract_catalog_index(
+            "quay.io/test/catalog-index-extensions:1.9",
+            str(catalog_mount),
+            str(catalog_entities_parent_dir)
+        )
+
+        # Verify the function returned a path
+        assert result is not None
+        assert result.endswith('dynamic-plugins.default.yaml')
+
+        # Verify catalog entities were extracted from extensions directory
+        entities_dir = catalog_entities_parent_dir / "catalog-entities"
+        assert entities_dir.exists()
+        entity_file = entities_dir / "test-entity.yaml"
+        assert entity_file.exists()
+        assert "kind: Component" in entity_file.read_text()
+        assert "test-extensions" in entity_file.read_text()
+
+        # Verify success messages were printed
+        captured = capsys.readouterr()
+        assert 'Successfully extracted dynamic-plugins.default.yaml' in captured.out
+        assert 'Successfully extracted extensions catalog entities' in captured.out
+
+    def test_extract_catalog_index_falls_back_to_marketplace(self, tmp_path, mocker, mock_oci_image, capsys):
+        """Test that extraction falls back to marketplace directory when extensions doesn't exist."""
+        catalog_mount = tmp_path / "catalog-mount"
+        catalog_mount.mkdir()
+        catalog_entities_parent_dir = tmp_path / "entities-marketplace"
+
+        mocker.patch('shutil.which', return_value='/usr/bin/skopeo')
+
+        mock_result = mocker.Mock()
+        mock_result.returncode = 0
+        mock_subprocess_run = create_mock_skopeo_copy(
+            mock_oci_image['manifest_path'],
+            mock_oci_image['layer_tarball'],
+            mock_result
+        )
+        mocker.patch('subprocess.run', side_effect=mock_subprocess_run)
+
+        result = install_dynamic_plugins.extract_catalog_index(
+            "quay.io/test/catalog-index-marketplace:1.9",
+            str(catalog_mount),
+            str(catalog_entities_parent_dir)
+        )
+
+        # Verify the function returned a path
+        assert result is not None
+        assert result.endswith('dynamic-plugins.default.yaml')
+
+        # Verify catalog entities were extracted from marketplace directory (fallback)
+        entities_dir = catalog_entities_parent_dir / "catalog-entities"
+        assert entities_dir.exists()
+        entity_file = entities_dir / "test-entity.yaml"
+        assert entity_file.exists()
+        assert "kind: Component" in entity_file.read_text()
+
+        # Verify success messages were printed
+        captured = capsys.readouterr()
+        assert 'Successfully extracted dynamic-plugins.default.yaml' in captured.out
+        assert 'Successfully extracted extensions catalog entities' in captured.out
+
     def test_extract_catalog_index_without_catalog_entities(self, tmp_path, mocker, capsys):
-        """Test that extraction succeeds with warning if catalog-entities directory doesn't exist in image."""
+        """Test that extraction succeeds with warning if neither extensions nor marketplace directory exists."""
         import tarfile
 
         catalog_mount = tmp_path / "catalog-mount"
@@ -2727,11 +2867,12 @@ def test_extract_catalog_index_without_catalog_entities(self, tmp_path, mocker,
         assert result is not None
         assert result.endswith('dynamic-plugins.default.yaml')
 
-        # Verify warning was printed
+        # Verify warning was printed with both directory names
         captured = capsys.readouterr()
         assert 'WARNING' in captured.out
-        assert 'does not have a' in captured.out
-        assert 'catalog-entities/marketplace' in captured.out
+        assert 'does not have neither' in captured.out
+        assert 'catalog-entities/extensions/' in captured.out
+        assert 'catalog-entities/marketplace/' in captured.out
 
         # Verify catalog entities directory was not created
         entities_dir = catalog_entities_parent_dir / "catalog-entities"