Cache information responses from REST sources (#5726)

## Change
Store and use the `/information` response from REST sources per their
`cache-control` header.

At the base of this change is a new type of setting stream: `Encrypted`.
These settings are encrypted to the user and machine that the data is
stored on.

Based on the log file from running with and without a cached value, the
time between close log lines dropped from 189 ms to 8 ms for the msstore
source. This time saving should be in effect for the large majority of
uses of WinGet.

Author: JohnMcPMS

.github/actions/spelling/allow.txt

@@ -53,6 +53,7 @@ createpintable
 createportabletable
 createtables
 CRTDECL
+CRYPTPROTECT
 CTLs
 curated
 CURSORPOSITON

azure-pipelines.yml

@@ -332,7 +332,7 @@ jobs:
   - powershell: |
       Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force
       Install-Module Microsoft.WinGet.Client -Repository PSGallery -Force
-      Repair-WingetPackageManager -AllUsers -Latest
+      try { Repair-WingetPackageManager -AllUsers -Latest -Verbose } catch { $_.Exception }
       Install-WinGetPackage -Id Microsoft.Sysinternals.PsTools -Source winget
     displayName: Install Sysinternals PsTools Using Winget
     condition: succeededOrFailed()

src/AppInstallerCLIE2ETests/ConfigureCommand.cs

@@ -364,7 +364,7 @@ public void ConfigureFromTestRepo_DSCv3()
         public void ConfigureFindUnitProcessors()
         {
             // Find all unit processors.
-            var result = TestCommon.RunAICLICommand("test config-find-unit-processors", string.Empty);
+            var result = TestCommon.RunAICLICommand("test config-find-unit-processors", string.Empty, timeOut: 120000);
             Assert.AreEqual(0, result.ExitCode);
             Assert.True(result.StdOut.Contains("Microsoft/OSInfo"));
 

src/AppInstallerCLITests/Downloader.cpp

@@ -109,3 +109,63 @@ TEST_CASE("HttpStream_ReadLastFullPage", "[HttpStream]")
     REQUIRE(stream->Read(buffer.get(), static_cast<ULONG>(HttpStream::HttpLocalCache::PAGE_SIZE), &read) >= S_OK);
     REQUIRE(read == (stat.cbSize.QuadPart % HttpStream::HttpLocalCache::PAGE_SIZE));
 }
+
+TEST_CASE("CacheControl", "[Downloader]")
+{
+    SECTION("Empty")
+    {
+        CacheControlPolicy test{ L"" };
+        REQUIRE(!test.Present);
+    }
+    SECTION("Space")
+    {
+        CacheControlPolicy test{ L" " };
+        REQUIRE(!test.Present);
+    }
+    SECTION("Standard")
+    {
+        CacheControlPolicy test{ L"public, max-age=77287" };
+        REQUIRE(test.Present);
+        REQUIRE(test.Public);
+        REQUIRE(!test.NoCache);
+        REQUIRE(!test.NoStore);
+        REQUIRE(test.MaxAge == 77287);
+    }
+    SECTION("All")
+    {
+        CacheControlPolicy test{ L"public, no-cache, no-store, max-age = 77" };
+        REQUIRE(test.Present);
+        REQUIRE(test.Public);
+        REQUIRE(test.NoCache);
+        REQUIRE(test.NoStore);
+        REQUIRE(test.MaxAge == 77);
+    }
+    SECTION("Casing")
+    {
+        CacheControlPolicy test{ L"Public, Max-Age=42" };
+        REQUIRE(test.Present);
+        REQUIRE(test.Public);
+        REQUIRE(!test.NoCache);
+        REQUIRE(!test.NoStore);
+        REQUIRE(test.MaxAge == 42);
+    }
+    SECTION("Unknown")
+    {
+        CacheControlPolicy test{ L"public, max-age=77287, not-real" };
+        REQUIRE(test.Present);
+        REQUIRE(test.Public);
+        REQUIRE(!test.NoCache);
+        REQUIRE(!test.NoStore);
+        REQUIRE(test.MaxAge == 77287);
+    }
+    SECTION("MaxAge Negative")
+    {
+        CacheControlPolicy test{ L"max-age=-1" };
+        REQUIRE(test.MaxAge == CacheControlPolicy::MaximumMaxAge);
+    }
+    SECTION("MaxAge not a number")
+    {
+        CacheControlPolicy test{ L"max-age=FOO" };
+        REQUIRE(test.MaxAge == 0);
+    }
+}

src/AppInstallerCLITests/RestClient.cpp

@@ -4,7 +4,9 @@
 #include "TestCommon.h"
 #include "TestRestRequestHandler.h"
 #include <Rest/RestClient.h>
+#include <Rest/RestInformationCache.h>
 #include <Rest/Schema/IRestClient.h>
+#include <Rest/Schema/InformationResponseDeserializer.h>
 #include <AppInstallerVersions.h>
 #include <AppInstallerErrors.h>
 #include <AppInstallerRuntime.h>
@@ -488,3 +490,182 @@ TEST_CASE("RestClientCreate_1.7_Success", "[RestSource]")
     REQUIRE(information.Authentication.MicrosoftEntraIdInfo->Resource == "GUID");
     REQUIRE(information.Authentication.MicrosoftEntraIdInfo->Scope == "test");
 }
+
+// Simulate the msstore cache round trip using real world data.
+TEST_CASE("RestInformationCache_RoundTrip", "[RestInformationCache]")
+{
+    Settings::Stream{ Settings::Stream::RestInformationCache }.Remove();
+
+    std::wstring endpoint = L"https://test-url-com/information";
+    CacheControlPolicy cacheControl{ L"public, max-age=77287" };
+    auto response = web::json::value::parse(
+R"delimiter({
+    "$type": "Microsoft.Marketplace.Storefront.StoreEdgeFD.BusinessLogic.Response.PackageMetadata.PackageMetadataResponse, StoreEdgeFD",
+    "Data": {
+        "$type": "Microsoft.Marketplace.Storefront.StoreEdgeFD.BusinessLogic.Response.PackageMetadata.PackageMetadataData, StoreEdgeFD",
+        "SourceIdentifier": "StoreEdgeFD",
+        "SourceAgreements": {
+            "$type": "Microsoft.Marketplace.Storefront.StoreEdgeFD.BusinessLogic.Response.PackageMetadata.SourceAgreements, StoreEdgeFD",
+            "AgreementsIdentifier": "StoreEdgeFD",
+            "Agreements": [
+                {
+                    "$type": "Microsoft.Marketplace.Storefront.StoreEdgeFD.BusinessLogic.Response.PackageManifest.AgreementDetail, StoreEdgeFD",
+                    "AgreementLabel": "Terms of Transaction",
+                    "AgreementUrl": "https://aka.ms/microsoft-store-terms-of-transaction"
+                }
+            ]
+        },
+        "ServerSupportedVersions": [ "1.0.0", "1.1.0", "1.6.0" ],
+        "RequiredQueryParameters": [ "market" ],
+        "RequiredPackageMatchFields": [ "market" ]
+    }
+})delimiter");
+
+    RestInformationCache cache;
+    cache.Cache(endpoint, {}, {}, cacheControl, response);
+    auto cachedValue = cache.Get(endpoint, {}, {});
+
+    REQUIRE(cachedValue.has_value());
+
+    InformationResponseDeserializer deserializer;
+    const auto expected = deserializer.Deserialize(response);
+    const auto& actual = cachedValue.value();
+
+    REQUIRE(expected.SourceIdentifier == actual.SourceIdentifier);
+    REQUIRE(expected.SourceAgreementsIdentifier == actual.SourceAgreementsIdentifier);
+    REQUIRE(expected.SourceAgreements.size() == actual.SourceAgreements.size());
+    REQUIRE(1 == actual.SourceAgreements.size());
+    REQUIRE(expected.SourceAgreements[0].Label == actual.SourceAgreements[0].Label);
+    REQUIRE(expected.SourceAgreements[0].Text == actual.SourceAgreements[0].Text);
+    REQUIRE(expected.SourceAgreements[0].Url == actual.SourceAgreements[0].Url);
+
+    REQUIRE(expected.ServerSupportedVersions.size() == actual.ServerSupportedVersions.size());
+    for (const auto& expectedVersion : expected.ServerSupportedVersions)
+    {
+        REQUIRE(std::find(actual.ServerSupportedVersions.begin(), actual.ServerSupportedVersions.end(), expectedVersion) != actual.ServerSupportedVersions.end());
+    }
+
+    REQUIRE(expected.RequiredQueryParameters.size() == actual.RequiredQueryParameters.size());
+    REQUIRE(1 == actual.RequiredQueryParameters.size());
+    REQUIRE(expected.RequiredQueryParameters[0] == actual.RequiredQueryParameters[0]);
+
+    REQUIRE(expected.RequiredPackageMatchFields.size() == actual.RequiredPackageMatchFields.size());
+    REQUIRE(1 == actual.RequiredPackageMatchFields.size());
+    REQUIRE(expected.RequiredPackageMatchFields[0] == actual.RequiredPackageMatchFields[0]);
+}
+
+web::json::value CreateInformationResponse(std::string_view identifier)
+{
+    std::ostringstream stream;
+    stream << R"({ "Data": { "SourceIdentifier": ")" << identifier << R"(", "ServerSupportedVersions": [ "1.0.0" ] } })";
+
+    return web::json::value::parse(stream.str());
+}
+
+TEST_CASE("RestInformationCache_Get", "[RestInformationCache]")
+{
+    Settings::Stream{ Settings::Stream::RestInformationCache }.Remove();
+
+    std::wstring endpoint1 = L"https://test-url1-com/information";
+    std::wstring endpoint2 = L"https://test-url2-com/information";
+    std::wstring endpointNotPresent = L"https://test-url-not-present-com/information";
+    std::string header = "Header";
+    std::string caller = "Caller";
+    std::string publicEndpoint1Identifier = "Identifier1";
+    std::string privateEndpoint1Identifier = "Identifier2";
+    std::string privateEndpoint2Identifier = "Identifier3";
+    auto publicEndpoint1Response = CreateInformationResponse(publicEndpoint1Identifier);
+    auto privateEndpoint1Response = CreateInformationResponse(privateEndpoint1Identifier);
+    auto privateEndpoint2Response = CreateInformationResponse(privateEndpoint2Identifier);
+
+    RestInformationCache cache;
+
+    // Cache:
+    //  1. public and private for same endpoint
+    cache.Cache(endpoint1, header, caller, { L"public" }, publicEndpoint1Response);
+    cache.Cache(endpoint1, header, caller, {}, privateEndpoint1Response);
+    //  2. another endpoint with private data (same headers)
+    cache.Cache(endpoint2, header, caller, {}, privateEndpoint2Response);
+
+    SECTION("Same headers prefers private")
+    {
+        auto cachedValue = cache.Get(endpoint1, header, caller);
+        REQUIRE(cachedValue.has_value());
+        REQUIRE(privateEndpoint1Identifier == cachedValue->SourceIdentifier);
+    }
+    SECTION("Different headers falls back to public")
+    {
+        auto cachedValue = cache.Get(endpoint1, "Different", "Different");
+        REQUIRE(cachedValue.has_value());
+        REQUIRE(publicEndpoint1Identifier == cachedValue->SourceIdentifier);
+    }
+    SECTION("Second endpoint")
+    {
+        auto cachedValue = cache.Get(endpoint2, header, caller);
+        REQUIRE(cachedValue.has_value());
+        REQUIRE(privateEndpoint2Identifier == cachedValue->SourceIdentifier);
+    }
+    SECTION("Second endpoint different headers")
+    {
+        auto cachedValue = cache.Get(endpoint2, {}, {});
+        REQUIRE(!cachedValue.has_value());
+    }
+    SECTION("Missing endpoint")
+    {
+        auto cachedValue = cache.Get(endpointNotPresent, header, caller);
+        REQUIRE(!cachedValue.has_value());
+    }
+}
+
+TEST_CASE("RestInformationCache_Cache_NoStore", "[RestInformationCache]")
+{
+    Settings::Stream{ Settings::Stream::RestInformationCache }.Remove();
+
+    std::wstring endpoint = L"https://test-url-com/information";
+
+    RestInformationCache cache;
+    cache.Cache(endpoint, {}, {}, { L"no-store" }, CreateInformationResponse("Identifier"));
+
+    auto cachedValue = cache.Get(endpoint, {}, {});
+    REQUIRE(!cachedValue.has_value());
+}
+
+TEST_CASE("RestInformationCache_Cache_Expiration", "[RestInformationCache]")
+{
+    Settings::Stream{ Settings::Stream::RestInformationCache }.Remove();
+
+    std::wstring endpoint = L"https://test-url-com/information";
+
+    RestInformationCache cache;
+    cache.Cache(endpoint, {}, {}, { L"max-age=2" }, CreateInformationResponse("Identifier"));
+
+    auto cachedValue = cache.Get(endpoint, {}, {});
+    REQUIRE(cachedValue.has_value());
+
+    std::this_thread::sleep_for(5s);
+
+    cachedValue = cache.Get(endpoint, {}, {});
+    REQUIRE(!cachedValue.has_value());
+}
+
+TEST_CASE("RestInformationCache_Cache_Overwrite", "[RestInformationCache]")
+{
+    Settings::Stream{ Settings::Stream::RestInformationCache }.Remove();
+
+    std::wstring endpoint = L"https://test-url-com/information";
+    std::string identifier1 = "Identifier1";
+    std::string identifier2 = "Identifier2";
+
+    RestInformationCache cache;
+    cache.Cache(endpoint, {}, {}, {}, CreateInformationResponse(identifier1));
+
+    auto cachedValue = cache.Get(endpoint, {}, {});
+    REQUIRE(cachedValue.has_value());
+    REQUIRE(identifier1 == cachedValue->SourceIdentifier);
+
+    cache.Cache(endpoint, {}, {}, {}, CreateInformationResponse(identifier2));
+
+    cachedValue = cache.Get(endpoint, {}, {});
+    REQUIRE(cachedValue.has_value());
+    REQUIRE(identifier2 == cachedValue->SourceIdentifier);
+}

src/AppInstallerCLITests/Settings.cpp

@@ -281,3 +281,29 @@ TEST_CASE("AttemptSetOnNewValue", "[settings]")
     settingValue = ReadEntireStream(*result);
     REQUIRE(value2 == settingValue);
 }
+
+TEST_CASE("SetAndReadSettingEncrypted", "[settings]")
+{
+    StreamDefinition name{ Type::Encrypted, "encrypted_test_setting" };
+    std::string value = "This is the test setting value";
+
+    Stream stream{ name };
+    REQUIRE(stream.Set(value));
+
+    auto result = stream.Get();
+    REQUIRE(static_cast<bool>(result));
+
+    std::string settingValue = ReadEntireStream(*result);
+    REQUIRE(value == settingValue);
+
+    // Ensure that the data is encrypted
+    name.Type = Type::StandardFile;
+
+    Stream streamDirect{ name };
+
+    auto resultDirect = streamDirect.Get();
+    REQUIRE(static_cast<bool>(resultDirect));
+
+    std::string settingValueDirect = ReadEntireStream(*resultDirect);
+    REQUIRE(value != settingValueDirect);
+}

src/AppInstallerCLITests/Strings.cpp

@@ -117,6 +117,9 @@ TEST_CASE("Trim", "[strings]")
     REQUIRE(Trim(str.assign("Multiple words")) == "Multiple words");
     REQUIRE(Trim(str.assign("         Multiple words")) == "Multiple words");
     REQUIRE(Trim(str.assign("Much after is taken \f\n\r\t\v\v\t\r\n\f ")) == "Much after is taken");
+
+    REQUIRE(Trim(L" Test") == L"Test");
+    REQUIRE(Trim(L"   ") == L"");
 }
 
 TEST_CASE("CaseInsensitiveStartsWith", "[strings]")
@@ -297,6 +300,20 @@ TEST_CASE("SplitWithSeparator", "[strings]")
     REQUIRE(test4.size() == 2);
     REQUIRE(test4[0] == "trim");
     REQUIRE(test4[1] == "spaces");
+
+    std::vector<std::wstring_view> test5 = Split(L" trim /spaces /  ", '/', true);
+    REQUIRE(test5.size() == 3);
+    REQUIRE(test5[0] == L"trim");
+    REQUIRE(test5[1] == L"spaces");
+    REQUIRE(test5[2] == L"");
+
+    std::vector<std::wstring_view> test6 = Split(L" ", '/', true);
+    REQUIRE(test6.size() == 1);
+    REQUIRE(test6[0] == L"");
+
+    std::vector<std::string> test7 = Split("", ';');
+    REQUIRE(test7.size() == 1);
+    REQUIRE(test7[0] == "");
 }
 
 TEST_CASE("ConvertGuid", "[strings]")

src/AppInstallerCLITests/TestRestRequestHandler.cpp

@@ -23,6 +23,7 @@ std::shared_ptr<TestRestRequestHandler> GetTestRestRequestHandler(
             }
 
             response.headers().set_content_type(mimeType);
+            response.headers().set_cache_control(L"no-store");
             response.set_status_code(statusCode);
             return pplx::task_from_result(response);
         });
@@ -38,6 +39,7 @@ std::shared_ptr<TestRestRequestHandler> GetTestRestRequestHandler(
             response.set_body(utf16string{});
 
             response.headers().set_content_type(web::http::details::mime_types::application_json);
+            response.headers().set_cache_control(L"no-store");
             response.set_status_code(handler(request));
             return pplx::task_from_result(response);
         });
@@ -65,6 +67,7 @@ std::shared_ptr<TestRestRequestHandler> GetHeaderVerificationHandler(
             }
 
             response.headers().set_content_type(web::http::details::mime_types::application_json);
+            response.headers().set_cache_control(L"no-store");
             response.set_status_code(statusCode);
             return pplx::task_from_result(response);
         });