Merge pull request #12421 from DeterminateSystems/self-input-attrs

Add `inputs.self.submodules` flake attribute

Author: edolstra

doc/manual/rl-next/self-submodules-attr.md

@@ -0,0 +1,12 @@
+---
+synopsis: "`inputs.self.submodules` flake attribute"
+prs: [12421]
+---
+
+Flakes in Git repositories can now declare that they need Git submodules to be enabled:
+```
+{
+  inputs.self.submodules = true;
+}
+```
+Thus, it's no longer needed for the caller of the flake to pass `submodules = true`.

src/libcmd/common-eval-args.cc

@@ -34,7 +34,9 @@ EvalSettings evalSettings {
                 // FIXME `parseFlakeRef` should take a `std::string_view`.
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
-                auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
+                auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
+                auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+                state.allowPath(storePath);
                 return state.rootPath(state.store->toRealPath(storePath));
             },
         },
@@ -183,7 +185,9 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
-        auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
+        auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
+        auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+        state.allowPath(storePath);
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 

src/libfetchers/fetchers.cc

@@ -186,6 +186,7 @@ bool Input::contains(const Input & other) const
     return false;
 }
 
+// FIXME: remove
 std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
 {
     if (!scheme)
@@ -200,10 +201,6 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
             auto narHash = store->queryPathInfo(storePath)->narHash;
             result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
-            // FIXME: we would like to mark inputs as final in
-            // getAccessorUnchecked(), but then we can't add
-            // narHash. Or maybe narHash should be excluded from the
-            // concept of "final" inputs?
             result.attrs.insert_or_assign("__final", Explicit<bool>(true));
 
             assert(result.isFinal());
@@ -284,6 +281,8 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
     try {
         auto [accessor, result] = getAccessorUnchecked(store);
 
+        result.attrs.insert_or_assign("__final", Explicit<bool>(true));
+
         checkLocks(*this, result);
 
         return {accessor, std::move(result)};

src/libflake/flake/flake.cc

@@ -12,6 +12,7 @@
 #include "flake/settings.hh"
 #include "value-to-json.hh"
 #include "local-fs-store.hh"
+#include "fetch-to-store.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -24,7 +25,7 @@ namespace flake {
 struct FetchedFlake
 {
     FlakeRef lockedRef;
-    StorePath storePath;
+    ref<SourceAccessor> accessor;
 };
 
 typedef std::map<FlakeRef, FetchedFlake> FlakeCache;
@@ -40,7 +41,7 @@ static std::optional<FetchedFlake> lookupInFlakeCache(
     return i->second;
 }
 
-static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
+static std::tuple<ref<SourceAccessor>, FlakeRef, FlakeRef> fetchOrSubstituteTree(
     EvalState & state,
     const FlakeRef & originalRef,
     bool useRegistries,
@@ -51,8 +52,8 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
 
     if (!fetched) {
         if (originalRef.input.isDirect()) {
-            auto [storePath, lockedRef] = originalRef.fetchTree(state.store);
-            fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath});
+            auto [accessor, lockedRef] = originalRef.lazyFetch(state.store);
+            fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
         } else {
             if (useRegistries) {
                 resolvedRef = originalRef.resolve(
@@ -64,8 +65,8 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
                     });
                 fetched = lookupInFlakeCache(flakeCache, originalRef);
                 if (!fetched) {
-                    auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store);
-                    fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath});
+                    auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store);
+                    fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
                 }
                 flakeCache.insert_or_assign(resolvedRef, *fetched);
             }
@@ -76,14 +77,27 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
         flakeCache.insert_or_assign(originalRef, *fetched);
     }
 
-    debug("got tree '%s' from '%s'",
-        state.store->printStorePath(fetched->storePath), fetched->lockedRef);
+    debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedRef);
 
-    state.allowPath(fetched->storePath);
+    return {fetched->accessor, resolvedRef, fetched->lockedRef};
+}
+
+static StorePath copyInputToStore(
+    EvalState & state,
+    fetchers::Input & input,
+    const fetchers::Input & originalInput,
+    ref<SourceAccessor> accessor)
+{
+    auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName());
+
+    state.allowPath(storePath);
+
+    auto narHash = state.store->queryPathInfo(storePath)->narHash;
+    input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
-    assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store));
+    assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
 
-    return {fetched->storePath, resolvedRef, fetched->lockedRef};
+    return storePath;
 }
 
 static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
@@ -101,12 +115,47 @@ static void expectType(EvalState & state, ValueType type,
             showType(type), showType(value.type()), state.positions[pos]);
 }
 
-static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+static std::pair<std::map<FlakeId, FlakeInput>, fetchers::Attrs> parseFlakeInputs(
     EvalState & state,
     Value * value,
     const PosIdx pos,
     const InputAttrPath & lockRootAttrPath,
-    const SourcePath & flakeDir);
+    const SourcePath & flakeDir,
+    bool allowSelf);
+
+static void parseFlakeInputAttr(
+    EvalState & state,
+    const Attr & attr,
+    fetchers::Attrs & attrs)
+{
+    // Allow selecting a subset of enum values
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wswitch-enum"
+    switch (attr.value->type()) {
+        case nString:
+            attrs.emplace(state.symbols[attr.name], attr.value->c_str());
+            break;
+        case nBool:
+            attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean() });
+            break;
+        case nInt: {
+            auto intValue = attr.value->integer().value;
+            if (intValue < 0)
+                state.error<EvalError>("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow();
+            attrs.emplace(state.symbols[attr.name], uint64_t(intValue));
+            break;
+        }
+        default:
+            if (attr.name == state.symbols.create("publicKeys")) {
+                experimentalFeatureSettings.require(Xp::VerifiedFetches);
+                NixStringContext emptyContext = {};
+                attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, attr.pos, emptyContext).dump());
+            } else
+                state.error<TypeError>("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
+                    state.symbols[attr.name], showType(*attr.value)).debugThrow();
+    }
+    #pragma GCC diagnostic pop
+}
 
 static FlakeInput parseFlakeInput(
     EvalState & state,
@@ -149,44 +198,14 @@ static FlakeInput parseFlakeInput(
                 expectType(state, nBool, *attr.value, attr.pos);
                 input.isFlake = attr.value->boolean();
             } else if (attr.name == sInputs) {
-                input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir);
+                input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first;
             } else if (attr.name == sFollows) {
                 expectType(state, nString, *attr.value, attr.pos);
                 auto follows(parseInputAttrPath(attr.value->c_str()));
                 follows.insert(follows.begin(), lockRootAttrPath.begin(), lockRootAttrPath.end());
                 input.follows = follows;
-            } else {
-                // Allow selecting a subset of enum values
-                #pragma GCC diagnostic push
-                #pragma GCC diagnostic ignored "-Wswitch-enum"
-                switch (attr.value->type()) {
-                    case nString:
-                        attrs.emplace(state.symbols[attr.name], attr.value->c_str());
-                        break;
-                    case nBool:
-                        attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean() });
-                        break;
-                    case nInt: {
-                        auto intValue = attr.value->integer().value;
-
-                        if (intValue < 0) {
-                            state.error<EvalError>("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow();
-                        }
-
-                        attrs.emplace(state.symbols[attr.name], uint64_t(intValue));
-                        break;
-                    }
-                    default:
-                        if (attr.name == state.symbols.create("publicKeys")) {
-                            experimentalFeatureSettings.require(Xp::VerifiedFetches);
-                            NixStringContext emptyContext = {};
-                            attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump());
-                        } else
-                            state.error<TypeError>("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
-                                state.symbols[attr.name], showType(*attr.value)).debugThrow();
-                }
-                #pragma GCC diagnostic pop
-            }
+            } else
+                parseFlakeInputAttr(state, attr, attrs);
         } catch (Error & e) {
             e.addTrace(
                 state.positions[attr.pos],
@@ -216,28 +235,39 @@ static FlakeInput parseFlakeInput(
     return input;
 }
 
-static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+static std::pair<std::map<FlakeId, FlakeInput>, fetchers::Attrs> parseFlakeInputs(
     EvalState & state,
     Value * value,
     const PosIdx pos,
     const InputAttrPath & lockRootAttrPath,
-    const SourcePath & flakeDir)
+    const SourcePath & flakeDir,
+    bool allowSelf)
 {
     std::map<FlakeId, FlakeInput> inputs;
+    fetchers::Attrs selfAttrs;
 
     expectType(state, nAttrs, *value, pos);
 
     for (auto & inputAttr : *value->attrs()) {
-        inputs.emplace(state.symbols[inputAttr.name],
-            parseFlakeInput(state,
-                state.symbols[inputAttr.name],
-                inputAttr.value,
-                inputAttr.pos,
-                lockRootAttrPath,
-                flakeDir));
+        auto inputName = state.symbols[inputAttr.name];
+        if (inputName == "self") {
+            if (!allowSelf)
+                throw Error("'self' input attribute not allowed at %s", state.positions[inputAttr.pos]);
+            expectType(state, nAttrs, *inputAttr.value, inputAttr.pos);
+            for (auto & attr : *inputAttr.value->attrs())
+                parseFlakeInputAttr(state, attr, selfAttrs);
+        } else {
+            inputs.emplace(inputName,
+                parseFlakeInput(state,
+                    inputName,
+                    inputAttr.value,
+                    inputAttr.pos,
+                    lockRootAttrPath,
+                    flakeDir));
+        }
     }
 
-    return inputs;
+    return {inputs, selfAttrs};
 }
 
 static Flake readFlake(
@@ -269,8 +299,11 @@ static Flake readFlake(
 
     auto sInputs = state.symbols.create("inputs");
 
-    if (auto inputs = vInfo.attrs()->get(sInputs))
-        flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootAttrPath, flakeDir);
+    if (auto inputs = vInfo.attrs()->get(sInputs)) {
+        auto [flakeInputs, selfAttrs] = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootAttrPath, flakeDir, true);
+        flake.inputs = std::move(flakeInputs);
+        flake.selfAttrs = std::move(selfAttrs);
+    }
 
     auto sOutputs = state.symbols.create("outputs");
 
@@ -301,10 +334,10 @@ static Flake readFlake(
                     state.symbols[setting.name],
                     std::string(state.forceStringNoCtx(*setting.value, setting.pos, "")));
             else if (setting.value->type() == nPath) {
-                NixStringContext emptyContext = {};
+                auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy);
                 flake.config.settings.emplace(
                     state.symbols[setting.name],
-                    state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true).toOwned());
+                    state.store->toRealPath(storePath));
             }
             else if (setting.value->type() == nInt)
                 flake.config.settings.emplace(
@@ -342,16 +375,54 @@ static Flake readFlake(
     return flake;
 }
 
+static FlakeRef applySelfAttrs(
+    const FlakeRef & ref,
+    const Flake & flake)
+{
+    auto newRef(ref);
+
+    std::set<std::string> allowedAttrs{"submodules"};
+
+    for (auto & attr : flake.selfAttrs) {
+        if (!allowedAttrs.contains(attr.first))
+            throw Error("flake 'self' attribute '%s' is not supported", attr.first);
+        newRef.input.attrs.insert_or_assign(attr.first, attr.second);
+    }
+
+    return newRef;
+}
+
 static Flake getFlake(
     EvalState & state,
     const FlakeRef & originalRef,
     bool useRegistries,
     FlakeCache & flakeCache,
     const InputAttrPath & lockRootAttrPath)
 {
-    auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+    // Fetch a lazy tree first.
+    auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
         state, originalRef, useRegistries, flakeCache);
 
+    // Parse/eval flake.nix to get at the input.self attributes.
+    auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {accessor}, lockRootAttrPath);
+
+    // Re-fetch the tree if necessary.
+    auto newLockedRef = applySelfAttrs(lockedRef, flake);
+
+    if (lockedRef != newLockedRef) {
+        debug("refetching input '%s' due to self attribute", newLockedRef);
+        // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
+        newLockedRef.input.attrs.erase("narHash");
+        auto [accessor2, resolvedRef2, lockedRef2] = fetchOrSubstituteTree(
+            state, newLockedRef, false, flakeCache);
+        accessor = accessor2;
+        lockedRef = lockedRef2;
+    }
+
+    // Copy the tree to the store.
+    auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, accessor);
+
+    // Re-parse flake.nix from the store.
     return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootAttrPath);
 }
 
@@ -707,8 +778,12 @@ LockedFlake lockFlake(
                                 if (auto resolvedPath = resolveRelativePath()) {
                                     return {*resolvedPath, *input.ref};
                                 } else {
-                                    auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+                                    auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
                                         state, *input.ref, useRegistries, flakeCache);
+
+                                    // FIXME: allow input to be lazy.
+                                    auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, accessor);
+
                                     return {state.rootPath(state.store->toRealPath(storePath)), lockedRef};
                                 }
                             }();