Merge pull request #13975 from obsidiansystems/prep-json-0

Misc prep changes for #13942

Author: Ericson2314

src/libutil-tests/hash.cc

@@ -1,13 +1,17 @@
 #include <regex>
 
 #include <gtest/gtest.h>
+#include <nlohmann/json.hpp>
 
 #include "nix/util/hash.hh"
+#include "nix/util/tests/characterization.hh"
 
 namespace nix {
 
-class BLAKE3HashTest : public virtual ::testing::Test
+class HashTest : public CharacterizationTest
 {
+    std::filesystem::path unitTestData = getUnitTestData() / "hash";
+
 public:
 
     /**
@@ -16,8 +20,14 @@ class BLAKE3HashTest : public virtual ::testing::Test
      */
     ExperimentalFeatureSettings mockXpSettings;
 
-private:
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+};
 
+class BLAKE3HashTest : public HashTest
+{
     void SetUp() override
     {
         mockXpSettings.set("experimental-features", "blake3-hashes");
@@ -137,6 +147,46 @@ TEST(hashString, testKnownSHA512Hashes2)
         "c7d329eeb6dd26545e96e55b874be909");
 }
 
+/* ----------------------------------------------------------------------------
+ * parsing hashes
+ * --------------------------------------------------------------------------*/
+
+TEST(hashParseExplicitFormatUnprefixed, testKnownSHA256Hashes1_correct)
+{
+    // values taken from: https://tools.ietf.org/html/rfc4634
+    auto s = "abc";
+
+    auto hash = hashString(HashAlgorithm::SHA256, s);
+    ASSERT_EQ(
+        hash,
+        Hash::parseExplicitFormatUnprefixed(
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+            HashAlgorithm::SHA256,
+            HashFormat::Base16));
+}
+
+TEST(hashParseExplicitFormatUnprefixed, testKnownSHA256Hashes1_wrongAlgo)
+{
+    // values taken from: https://tools.ietf.org/html/rfc4634
+    ASSERT_THROW(
+        Hash::parseExplicitFormatUnprefixed(
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+            HashAlgorithm::SHA1,
+            HashFormat::Base16),
+        BadHash);
+}
+
+TEST(hashParseExplicitFormatUnprefixed, testKnownSHA256Hashes1_wrongBase)
+{
+    // values taken from: https://tools.ietf.org/html/rfc4634
+    ASSERT_THROW(
+        Hash::parseExplicitFormatUnprefixed(
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+            HashAlgorithm::SHA256,
+            HashFormat::Nix32),
+        BadHash);
+}
+
 /* ----------------------------------------------------------------------------
  * parseHashFormat, parseHashFormatOpt, printHashFormat
  * --------------------------------------------------------------------------*/

src/libutil/hash.cc

@@ -99,22 +99,37 @@ struct DecodeNamePair
 
 } // namespace
 
+static DecodeNamePair baseExplicit(HashFormat format)
+{
+    switch (format) {
+    case HashFormat::Base16:
+        return {base16::decode, "base16"};
+    case HashFormat::Nix32:
+        return {BaseNix32::decode, "nix32"};
+    case HashFormat::Base64:
+        return {base64::decode, "Base64"};
+    case HashFormat::SRI:
+        assert(false);
+    }
+}
+
 /**
  * Given the expected size of the message once decoded it, figure out
  * which encoding we are using by looking at the size of the encoded
  * message.
  */
-static DecodeNamePair baseFromSize(std::string_view rest, HashAlgorithm algo)
+static HashFormat baseFromSize(std::string_view rest, HashAlgorithm algo)
 {
     auto hashSize = regularHashSize(algo);
+
     if (rest.size() == base16::encodedLength(hashSize))
-        return {base16::decode, "base16"};
+        return HashFormat::Base16;
 
     if (rest.size() == BaseNix32::encodedLength(hashSize))
-        return {BaseNix32::decode, "nix32"};
+        return HashFormat::Nix32;
 
     if (rest.size() == base64::encodedLength(hashSize))
-        return {base64::decode, "Base64"};
+        return HashFormat::Base64;
 
     throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(algo));
 }
@@ -135,7 +150,8 @@ static Hash parseLowLevel(std::string_view rest, HashAlgorithm algo, DecodeNameP
         e.addTrace({}, "While decoding hash '%s'", rest);
     }
     if (d.size() != res.hashSize)
-        throw BadHash("invalid %s hash '%s' %d %d", pair.encodingName, rest);
+        throw BadHash(
+            "invalid %s hash '%s', length %d != expected length %d", pair.encodingName, rest, d.size(), res.hashSize);
     assert(res.hashSize);
     memcpy(res.hash, d.data(), res.hashSize);
 
@@ -189,7 +205,7 @@ static Hash parseAnyHelper(std::string_view rest, auto resolveAlgo)
         } else {
             /* Otherwise, decide via the length of the hash (for the
                given algorithm) what base encoding it is. */
-            return baseFromSize(rest, algo);
+            return baseExplicit(baseFromSize(rest, algo));
         }
     }();
 
@@ -224,7 +240,12 @@ Hash Hash::parseAny(std::string_view original, std::optional<HashAlgorithm> optA
 
 Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo)
 {
-    return parseLowLevel(s, algo, baseFromSize(s, algo));
+    return parseExplicitFormatUnprefixed(s, algo, baseFromSize(s, algo));
+}
+
+Hash Hash::parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat format)
+{
+    return parseLowLevel(s, algo, baseExplicit(format));
 }
 
 Hash Hash::random(HashAlgorithm algo)

src/libutil/include/nix/util/experimental-features.hh

@@ -3,6 +3,7 @@
 
 #include "nix/util/error.hh"
 #include "nix/util/types.hh"
+#include "nix/util/json-non-null.hh"
 
 #include <nlohmann/json_fwd.hpp>
 
@@ -89,6 +90,13 @@ public:
     MissingExperimentalFeature(ExperimentalFeature missingFeature);
 };
 
+/**
+ * `ExperimentalFeature` is always rendered as a string.
+ */
+template<>
+struct json_avoids_null<ExperimentalFeature> : std::true_type
+{};
+
 /**
  * Semi-magic conversion to and from json.
  * See the nlohmann/json readme for more details.

src/libutil/include/nix/util/hash.hh

@@ -90,6 +90,15 @@ struct Hash
      */
     static Hash parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo);
 
+    /**
+     * Like `parseNonSRIUnprefixed`, but the hash format has been
+     * explicitly given.
+     *
+     * @param explicitFormat cannot be SRI, but must be one of the
+     * "bases".
+     */
+    static Hash parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat explicitFormat);
+
     static Hash parseSRI(std::string_view original);
 
 public:

src/libutil/include/nix/util/json-non-null.hh

@@ -0,0 +1,55 @@
+#pragma once
+///@file
+
+#include <string>
+#include <vector>
+#include <set>
+#include <map>
+#include <list>
+
+namespace nix {
+
+/**
+ * For `adl_serializer<std::optional<T>>` below, we need to track what
+ * types are not already using `null`. Only for them can we use `null`
+ * to represent `std::nullopt`.
+ */
+template<typename T>
+struct json_avoids_null;
+
+/**
+ * Handle numbers in default impl
+ */
+template<typename T>
+struct json_avoids_null : std::bool_constant<std::is_integral<T>::value>
+{};
+
+template<>
+struct json_avoids_null<std::nullptr_t> : std::false_type
+{};
+
+template<>
+struct json_avoids_null<bool> : std::true_type
+{};
+
+template<>
+struct json_avoids_null<std::string> : std::true_type
+{};
+
+template<typename T>
+struct json_avoids_null<std::vector<T>> : std::true_type
+{};
+
+template<typename T>
+struct json_avoids_null<std::list<T>> : std::true_type
+{};
+
+template<typename T, typename Compare>
+struct json_avoids_null<std::set<T, Compare>> : std::true_type
+{};
+
+template<typename K, typename V, typename Compare>
+struct json_avoids_null<std::map<K, V, Compare>> : std::true_type
+{};
+
+} // namespace nix

src/libutil/include/nix/util/json-utils.hh

@@ -6,6 +6,7 @@
 
 #include "nix/util/error.hh"
 #include "nix/util/types.hh"
+#include "nix/util/json-non-null.hh"
 
 namespace nix {
 
@@ -59,56 +60,6 @@ Strings getStringList(const nlohmann::json & value);
 StringMap getStringMap(const nlohmann::json & value);
 StringSet getStringSet(const nlohmann::json & value);
 
-/**
- * For `adl_serializer<std::optional<T>>` below, we need to track what
- * types are not already using `null`. Only for them can we use `null`
- * to represent `std::nullopt`.
- */
-template<typename T>
-struct json_avoids_null;
-
-/**
- * Handle numbers in default impl
- */
-template<typename T>
-struct json_avoids_null : std::bool_constant<std::is_integral<T>::value>
-{};
-
-template<>
-struct json_avoids_null<std::nullptr_t> : std::false_type
-{};
-
-template<>
-struct json_avoids_null<bool> : std::true_type
-{};
-
-template<>
-struct json_avoids_null<std::string> : std::true_type
-{};
-
-template<typename T>
-struct json_avoids_null<std::vector<T>> : std::true_type
-{};
-
-template<typename T>
-struct json_avoids_null<std::list<T>> : std::true_type
-{};
-
-template<typename T, typename Compare>
-struct json_avoids_null<std::set<T, Compare>> : std::true_type
-{};
-
-template<typename K, typename V>
-struct json_avoids_null<std::map<K, V>> : std::true_type
-{};
-
-/**
- * `ExperimentalFeature` is always rendered as a string.
- */
-template<>
-struct json_avoids_null<ExperimentalFeature> : std::true_type
-{};
-
 } // namespace nix
 
 namespace nlohmann {

src/libutil/include/nix/util/meson.build

@@ -42,6 +42,7 @@ headers = files(
   'hash.hh',
   'hilite.hh',
   'json-impls.hh',
+  'json-non-null.hh',
   'json-utils.hh',
   'logging.hh',
   'lru-cache.hh',