Support associating export units with packages in subdirectories (1.12)

.github/actions/spelling/expect.txt

@@ -47,6 +47,7 @@ awgpm
 awgs
 azurewebsites
 Baz
+bbb
 bcp
 BEBOM
 BEFACEF
@@ -74,6 +75,7 @@ buildtrees
 cancelledbyuser
 casemap
 casemappings
+ccc
 cch
 centralus
 certmgr
@@ -395,6 +397,7 @@ packageinusebyapplication
 PACL
 PARAMETERMAP
 pathparts
+pathtree
 Patil
 pbstr
 pcb

src/AppInstallerCLICore/Workflows/ConfigurationFlow.cpp

@@ -17,6 +17,7 @@
 #include <AppInstallerStrings.h>
 #include <winget/ExperimentalFeature.h>
 #include <winget/SelfManagement.h>
+#include <winget/PathTree.h>
 #include <winrt/Microsoft.Management.Configuration.h>
 
 using namespace AppInstaller::CLI::Execution;
@@ -1678,6 +1679,80 @@ namespace AppInstaller::CLI::Workflow
             }
         }
 
+        // Contains a tree of all unit processors by their path.
+        struct UnitProcessorTree
+        {
+        private:
+            struct SourceAndPackage
+            {
+                PackageCollection::Source Source;
+                PackageCollection::Package Package;
+            };
+
+            struct Node
+            {
+                // Packages whose installed location is at this node
+                std::vector<SourceAndPackage> Packages;
+
+                // Units whose location is at this node.
+                std::vector<IConfigurationUnitProcessorDetails> Units;
+            };
+
+            Filesystem::PathTree<Node> m_pathTree;
+
+            Node& FindNodeForFilePath(const winrt::hstring& filePath)
+            {
+                std::filesystem::path path{ std::wstring{ filePath } };
+                return m_pathTree.FindOrInsert(path.parent_path());
+            }
+
+        public:
+            UnitProcessorTree(std::vector<IConfigurationUnitProcessorDetails>&& unitProcessors)
+            {
+                for (auto&& unit : unitProcessors)
+                {
+                    IConfigurationUnitProcessorDetails3 unitProcessor3;
+                    if (unit.try_as(unitProcessor3))
+                    {
+                        winrt::hstring unitPath = unitProcessor3.Path();
+                        AICLI_LOG(Config, Verbose, << "Found unit `" << Utility::ConvertToUTF8(unit.UnitType()) << "` at: " << Utility::ConvertToUTF8(unitPath));
+                        Node& node = FindNodeForFilePath(unitPath);
+                        node.Units.emplace_back(std::move(unit));
+                    }
+                }
+            }
+
+            void PlacePackage(const PackageCollection::Source& source, const PackageCollection::Package& package)
+            {
+                Node* node = m_pathTree.Find(package.InstalledLocation);
+                if (node)
+                {
+                    node->Packages.emplace_back(SourceAndPackage{ source, package });
+                }
+            }
+
+            std::vector<IConfigurationUnitProcessorDetails> GetResourcesForPackage(const PackageCollection::Package& package) const
+            {
+                std::vector<IConfigurationUnitProcessorDetails> result;
+
+                m_pathTree.VisitIf(
+                    package.InstalledLocation,
+                    [&](const Node& node)
+                    {
+                        for (const auto& unit : node.Units)
+                        {
+                            result.emplace_back(unit);
+                        }
+                    },
+                    [](const Node& node)
+                    {
+                        return node.Packages.empty();
+                    });
+
+                return result;
+            }
+        };
+
         void ProcessPackagesForConfigurationExportAll(Execution::Context& context)
         {
             ConfigurationContext& configContext = context.Get<Data::ConfigurationContext>();
@@ -1718,6 +1793,17 @@ namespace AppInstaller::CLI::Workflow
                 }
             }
 
+            // Build a tree of the unit processors and place packages onto it to indicate nearest ownership.
+            UnitProcessorTree unitProcessorTree{ std::move(unitProcessors) };
+
+            for (const auto& source : context.Get<Execution::Data::PackageCollection>().Sources)
+            {
+                for (const auto& package : source.Packages)
+                {
+                    unitProcessorTree.PlacePackage(source, package);
+                }
+            }
+
             for (const auto& source : context.Get<Execution::Data::PackageCollection>().Sources)
             {
                 // Create WinGetSource unit for non well known source.
@@ -1730,34 +1816,28 @@ namespace AppInstaller::CLI::Workflow
 
                 for (const auto& package : source.Packages)
                 {
+                    AICLI_LOG(Config, Verbose, << "Exporting package `" << package.Id << "` at: " << package.InstalledLocation);
+
                     auto packageUnit = anon::CreateWinGetPackageUnit(package, source, context.Args.Contains(Args::Type::IncludeVersions), sourceUnit, packageUnitType);
                     configContext.Set().Units().Append(packageUnit);
 
                     // Try package settings export.
-                    for (auto itr = unitProcessors.begin(); itr != unitProcessors.end(); /* itr incremented in the logic */)
+                    auto unitsForPackage = unitProcessorTree.GetResourcesForPackage(package);
+                    for (const auto& unit : unitsForPackage)
                     {
-                        IConfigurationUnitProcessorDetails3 unitProcessor3;
-                        itr->try_as(unitProcessor3);
-                        if (Filesystem::IsParentPath(std::filesystem::path{ std::wstring{ unitProcessor3.Path() } }, package.InstalledLocation))
-                        {
-                            ConfigurationUnit configUnit = anon::CreateConfigurationUnitFromUnitType(
-                                unitProcessor3.UnitType(),
-                                Utility::ConvertToUTF8(packageUnit.Identifier()));
+                        winrt::hstring unitType = unit.UnitType();
+                        AICLI_LOG(Config, Verbose, << "  exporting unit `" << Utility::ConvertToUTF8(unitType));
 
-                            auto exportedUnits = anon::ExportUnit(context, configUnit);
-                            anon::AddDependentUnit(exportedUnits, packageUnit);
+                        ConfigurationUnit configUnit = anon::CreateConfigurationUnitFromUnitType(
+                            unitType,
+                            Utility::ConvertToUTF8(packageUnit.Identifier()));
 
-                            for (auto exportedUnit : exportedUnits)
-                            {
-                                configContext.Set().Units().Append(exportedUnit);
-                            }
+                        auto exportedUnits = anon::ExportUnit(context, configUnit);
+                        anon::AddDependentUnit(exportedUnits, packageUnit);
 
-                            // Remove the unit processor from the list after export.
-                            itr = unitProcessors.erase(itr);
-                        }
-                        else
+                        for (const auto& exportedUnit : exportedUnits)
                         {
-                            itr++;
+                            configContext.Set().Units().Append(exportedUnit);
                         }
                     }
                 }

src/AppInstallerCLICore/pch.h

@@ -30,6 +30,7 @@
 #include <mutex>
 #include <numeric>
 #include <optional>
+#include <queue>
 #include <set>
 #include <sstream>
 #include <string>

src/AppInstallerCLIE2ETests/ConfigureCommand.cs

@@ -364,7 +364,7 @@ public void ConfigureFromTestRepo_DSCv3()
         public void ConfigureFindUnitProcessors()
         {
             // Find all unit processors.
-            var result = TestCommon.RunAICLICommand("test config-find-unit-processors", string.Empty, timeOut: 120000);
+            var result = TestCommon.RunAICLICommand("test config-find-unit-processors", string.Empty, timeOut: 300000);
             Assert.AreEqual(0, result.ExitCode);
             Assert.True(result.StdOut.Contains("Microsoft/OSInfo"));
 

src/AppInstallerCLIE2ETests/ConfigureExportCommand.cs

@@ -31,7 +31,9 @@ public void BaseSetup()
             var installDir = TestCommon.GetRandomTestDir();
             TestCommon.RunAICLICommand("install", $"AppInstallerTest.TestPackageExport -v 1.0.0.0 --silent -l {installDir}");
             this.previousPathValue = System.Environment.GetEnvironmentVariable("PATH");
-            System.Environment.SetEnvironmentVariable("PATH", this.previousPathValue + ";" + installDir);
+
+            // The installer puts DSCv3 resources in both locations
+            System.Environment.SetEnvironmentVariable("PATH", this.previousPathValue + ";" + installDir + ";" + installDir + "\\SubDirectory");
             DSCv3ResourceTestBase.EnsureTestResourcePresence();
         }
 
@@ -146,7 +148,7 @@ public void ExportAll()
         {
             var exportDir = TestCommon.GetRandomTestDir();
             var exportFile = Path.Combine(exportDir, "exported.yml");
-            var result = TestCommon.RunAICLICommand(Command, $"--all -o {exportFile}", timeOut: 1200000);
+            var result = TestCommon.RunAICLICommand(Command, $"--all --verbose -o {exportFile}", timeOut: 1200000);
             Assert.AreEqual(Constants.ErrorCode.S_OK, result.ExitCode);
             Assert.True(File.Exists(exportFile));
 
@@ -175,6 +177,10 @@ public void ExportAll()
             Assert.True(showResult.StdOut.Contains("AppInstallerTest/TestResource"));
             Assert.True(showResult.StdOut.Contains($"Dependencies: {Constants.TestSourceName}_AppInstallerTest.TestPackageExport"));
             Assert.True(showResult.StdOut.Contains("data: TestData"));
+
+            Assert.True(showResult.StdOut.Contains("AppInstallerTest/TestResource.SubDirectory"));
+            Assert.True(showResult.StdOut.Contains($"Dependencies: {Constants.TestSourceName}_AppInstallerTest.TestPackageExport"));
+            Assert.True(showResult.StdOut.Contains("data: TestData"));
         }
 
         /// <summary>

src/AppInstallerCLITests/Downloader.cpp

@@ -82,7 +82,7 @@ TEST_CASE("HttpStream_ReadLastFullPage", "[HttpStream]")
 
     for (size_t i = 0; i < 10; ++i)
     {
-        stream = GetReadOnlyStreamFromURI("https://cdn.winget.microsoft.com/cache/source2.msix");
+        stream = GetReadOnlyStreamFromURI("https://aka.ms/win32-x64-user-stable");
 
         stat = { 0 };
         REQUIRE(stream->Stat(&stat, STATFLAG_NONAME) == S_OK);
@@ -96,7 +96,7 @@ TEST_CASE("HttpStream_ReadLastFullPage", "[HttpStream]")
     }
 
     {
-        INFO("https://cdn.winget.microsoft.com/cache/source2.msix gave back a 0 byte file");
+        INFO("https://aka.ms/win32-x64-user-stable gave back a 0 byte file");
         REQUIRE(stream);
     }
 

src/AppInstallerCLITests/Filesystem.cpp

@@ -3,6 +3,7 @@
 #include "pch.h"
 #include "TestCommon.h"
 #include <winget/Filesystem.h>
+#include <winget/PathTree.h>
 #include <AppInstallerStrings.h>
 
 using namespace AppInstaller::Utility;
@@ -113,3 +114,112 @@ TEST_CASE("GetExecutablePathForProcess", "[filesystem]")
     REQUIRE(thisExecutable.has_extension());
     REQUIRE(thisExecutable.filename() == L"AppInstallerCLITests.exe");
 }
+
+TEST_CASE("PathTree_InsertAndFind", "[filesystem][pathtree]")
+{
+    PathTree<bool> pathTree;
+
+    std::filesystem::path path1 = L"C:\\test";
+    std::filesystem::path path1sub = L"C:\\test\\sub";
+    std::filesystem::path path2 = L"C:\\diff";
+    std::filesystem::path path3 = L"D:\\test";
+
+    REQUIRE(nullptr == pathTree.Find(path1));
+    pathTree.FindOrInsert(path1) = true;
+
+    REQUIRE(nullptr != pathTree.Find(path1));
+    REQUIRE(*pathTree.Find(path1));
+
+    REQUIRE(nullptr == pathTree.Find(path1sub));
+    REQUIRE(nullptr == pathTree.Find(path2));
+    REQUIRE(nullptr == pathTree.Find(path3));
+}
+
+TEST_CASE("PathTree_InsertAndFind_Negative", "[filesystem][pathtree]")
+{
+    PathTree<bool> pathTree;
+    pathTree.FindOrInsert(L"C:\\a\\aa\\aaa");
+
+    REQUIRE(nullptr == pathTree.Find({}));
+    REQUIRE_THROWS_HR(pathTree.FindOrInsert({}), E_INVALIDARG);
+}
+
+size_t CountVisited(const PathTree<bool>& pathTree, const std::filesystem::path& path, std::function<bool(const bool&)> predicate)
+{
+    size_t result = 0;
+    pathTree.VisitIf(path, [&](const bool&) { ++result; }, predicate);
+    return result;
+}
+
+TEST_CASE("PathTree_VisitIf_Count", "[filesystem][pathtree]")
+{
+    PathTree<bool> pathTree;
+
+    pathTree.FindOrInsert(L"C:\\a\\aa\\aaa") = true;
+    pathTree.FindOrInsert(L"C:\\a\\aa\\bbb") = true;
+    pathTree.FindOrInsert(L"C:\\a\\aa\\ccc") = false;
+    pathTree.FindOrInsert(L"C:\\a\\aa") = true;
+
+    pathTree.FindOrInsert(L"C:\\a\\bb\\aaa") = false;
+    pathTree.FindOrInsert(L"C:\\a\\bb\\bbb") = true;
+    pathTree.FindOrInsert(L"C:\\a\\bb\\ccc") = false;
+    pathTree.FindOrInsert(L"C:\\a\\bb") = true;
+
+    pathTree.FindOrInsert(L"C:\\a\\cc\\aaa") = true;
+    pathTree.FindOrInsert(L"C:\\a\\cc\\bbb") = false;
+    pathTree.FindOrInsert(L"C:\\a\\cc\\ccc") = false;
+    pathTree.FindOrInsert(L"C:\\a\\cc") = false;
+
+    pathTree.FindOrInsert(L"C:\\a") = true;
+    pathTree.FindOrInsert(L"C:\\b") = false;
+    pathTree.FindOrInsert(L"D:\\a") = false;
+
+    auto always = [](const bool&) { return true; };
+    auto never = [](const bool&) { return false; };
+    auto if_input = [](const bool& b) { return b; };
+
+    REQUIRE(0 == CountVisited(pathTree, {}, always));
+
+    REQUIRE(15 == CountVisited(pathTree, L"C:\\", always));
+    REQUIRE(2 == CountVisited(pathTree, L"D:\\", always));
+    REQUIRE(0 == CountVisited(pathTree, L"E:\\", always));
+
+    REQUIRE(1 == CountVisited(pathTree, L"C:\\", never));
+    REQUIRE(1 == CountVisited(pathTree, L"D:\\", never));
+    REQUIRE(0 == CountVisited(pathTree, L"E:\\", never));
+
+    REQUIRE(7 == CountVisited(pathTree, L"C:\\", if_input));
+    REQUIRE(6 == CountVisited(pathTree, L"C:\\a", if_input));
+    REQUIRE(2 == CountVisited(pathTree, L"C:\\a\\cc", if_input));
+    REQUIRE(1 == CountVisited(pathTree, L"D:\\", if_input));
+    REQUIRE(0 == CountVisited(pathTree, L"E:\\", if_input));
+}
+
+TEST_CASE("PathTree_VisitIf_Correct", "[filesystem][pathtree]")
+{
+    PathTree<std::pair<bool, bool>> pathTree;
+
+    pathTree.FindOrInsert(L"C:\\a\\aa\\aaa") = { true, true };
+    pathTree.FindOrInsert(L"C:\\a\\aa\\bbb") = { true, true };
+    pathTree.FindOrInsert(L"C:\\a\\aa\\ccc") = { false, false };
+    pathTree.FindOrInsert(L"C:\\a\\aa") = { true, true };
+
+    pathTree.FindOrInsert(L"C:\\a\\bb\\aaa") = { false, false };
+    pathTree.FindOrInsert(L"C:\\a\\bb\\bbb") = { true, true };
+    pathTree.FindOrInsert(L"C:\\a\\bb\\ccc") = { false, false };
+    pathTree.FindOrInsert(L"C:\\a\\bb") = { true, true };
+
+    pathTree.FindOrInsert(L"C:\\a\\cc\\aaa") = { true, true };
+    pathTree.FindOrInsert(L"C:\\a\\cc\\bbb") = { false, false };
+    pathTree.FindOrInsert(L"C:\\a\\cc\\ccc") = { false, false };
+    pathTree.FindOrInsert(L"C:\\a\\cc") = { false, false };
+
+    pathTree.FindOrInsert(L"C:\\a") = { true, true };
+    pathTree.FindOrInsert(L"C:\\b") = { false, false };
+    pathTree.FindOrInsert(L"C:") = { true, false };
+
+    auto check_input = [](const std::pair<bool, bool>& p) { REQUIRE(p.first); };
+    auto if_input = [](const std::pair<bool, bool>& p) { return p.second; };
+
+    pathTree.VisitIf(L"C:", check_input, if_input);
+}

src/AppInstallerSharedLib/AppInstallerSharedLib.vcxproj

@@ -346,6 +346,7 @@
     <ClInclude Include="Public\winget\LocIndependent.h" />
     <ClInclude Include="Public\winget\ManagedFile.h" />
     <ClInclude Include="Public\winget\ModuleCountBase.h" />
+    <ClInclude Include="Public\winget\PathTree.h" />
     <ClInclude Include="Public\winget\Registry.h" />
     <ClInclude Include="Public\winget\Resources.h" />
     <ClInclude Include="Public\winget\Runtime.h" />

src/AppInstallerSharedLib/AppInstallerSharedLib.vcxproj.filters

@@ -146,6 +146,9 @@
     <ClInclude Include="Public\winget\DetectMismatch.h">
       <Filter>Public\winget</Filter>
     </ClInclude>
+    <ClInclude Include="Public\winget\PathTree.h">
+      <Filter>Public\winget</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="pch.cpp">