simple store path root, remote store rework

Author: sorki

hnix-store-core/hnix-store-core.cabal

@@ -18,6 +18,7 @@ cabal-version:       >=1.10
 
 library
   exposed-modules:     System.Nix.Base32
+                     , System.Nix.Build
                      , System.Nix.Hash
                      , System.Nix.Internal.Base32
                      , System.Nix.Internal.Hash
@@ -29,6 +30,7 @@ library
                      , System.Nix.StorePath
                      , System.Nix.StorePathMetadata
                      , System.Nix.Util
+                     , System.Nix.ValidPath
   build-depends:       base >=4.10 && <5
                      , base16-bytestring
                      , bytestring
@@ -42,8 +44,6 @@ library
                      , filepath
                      , hashable
                      , mtl
-                     , regex-base
-                     , regex-tdfa >= 1.3.1.0
                      , saltine
                      , time
                      , text
@@ -67,6 +67,7 @@ test-suite format-tests
        Arbitrary
        NarFormat
        Hash
+       StorePath
    hs-source-dirs:
        tests
    build-depends:
@@ -77,6 +78,7 @@ test-suite format-tests
      , binary
      , bytestring
      , containers
+     , filepath
      , directory
      , process
      , tasty

hnix-store-core/src/System/Nix/Build.hs

@@ -0,0 +1,55 @@
+{-# LANGUAGE RecordWildCards #-}
+{-|
+Description : Build related types
+Maintainer  : srk <[email protected]>
+|-}
+module System.Nix.Build (
+    BuildMode(..)
+  , BuildStatus(..)
+  , BuildResult(..)
+  , buildSuccess
+  ) where
+
+import           Data.Time                 (UTCTime)
+import           Data.Text                 (Text)
+import           Data.HashSet              (HashSet)
+
+-- keep the order of these Enums to match enums from reference implementations
+-- src/libstore/store-api.hh
+data BuildMode = Normal | Repair | Check
+  deriving (Eq, Ord, Enum, Show)
+
+data BuildStatus =
+    Built
+  | Substituted
+  | AlreadyValid
+  | PermanentFailure
+  | InputRejected
+  | OutputRejected
+  | TransientFailure -- possibly transient
+  | CachedFailure    -- no longer used
+  | TimedOut
+  | MiscFailure
+  | DependencyFailed
+  | LogLimitExceeded
+  | NotDeterministic
+  deriving (Eq, Ord, Enum, Show)
+
+
+-- | Result of the build
+data BuildResult = BuildResult
+  { -- | build status, MiscFailure should be default
+    status             :: !BuildStatus
+  , -- | possible build error message
+    errorMessage       :: !(Maybe Text)
+  , -- | How many times this build was performed
+    timesBuilt         :: !Integer
+  , -- | If timesBuilt > 1, whether some builds did not produce the same result
+    isNonDeterministic :: !Bool
+  ,  -- Start time of this build
+    startTime          :: !UTCTime
+  ,  -- Stop time of this build
+    stopTime           :: !UTCTime
+  } deriving (Eq, Ord, Show)
+
+buildSuccess BuildResult{..} = status == Built || status == Substituted || status == AlreadyValid

hnix-store-core/src/System/Nix/Internal/StorePath.hs

@@ -16,12 +16,13 @@ import System.Nix.Hash
   ( HashAlgorithm(Truncated, SHA256)
   , Digest
   , encodeBase32
+  , decodeBase32
   , SomeNamedDigest
   )
-import Text.Regex.Base.RegexLike (makeRegex, matchTest)
-import Text.Regex.TDFA.Text (Regex)
+
 import Data.Text (Text)
 import Data.Text.Encoding (encodeUtf8)
+import qualified Data.Text as T
 import GHC.TypeLits (Symbol, KnownSymbol, symbolVal)
 import Data.ByteString (ByteString)
 import qualified Data.ByteString as BS
@@ -30,6 +31,9 @@ import Data.Hashable (Hashable(..))
 import Data.HashSet (HashSet)
 import Data.Proxy (Proxy(..))
 
+import System.FilePath (splitFileName)
+
+import Data.Char
 -- | A path in a Nix store.
 --
 -- From the Nix thesis: A store path is the full path of a store
@@ -39,20 +43,25 @@ import Data.Proxy (Proxy(..))
 --
 -- See the 'StoreDir' haddocks for details on why we represent this at
 -- the type level.
-data StorePath (storeDir :: StoreDir) = StorePath
+data StorePath = StorePath
   { -- | The 160-bit hash digest reflecting the "address" of the name.
     -- Currently, this is a truncated SHA256 hash.
     storePathHash :: !(Digest StorePathHashAlgo)
   , -- | The (typically human readable) name of the path. For packages
     -- this is typically the package name and version (e.g.
     -- hello-1.2.3).
     storePathName :: !StorePathName
+  , -- | Root of the store
+    storePathRoot :: !FilePath
   } deriving (Eq, Ord)
 
-instance Hashable (StorePath storeDir) where
+instance Hashable StorePath where
   hashWithSalt s (StorePath {..}) =
     s `hashWithSalt` storePathHash `hashWithSalt` storePathName
 
+instance Show StorePath where
+  show p = BC.unpack $ storePathToRawFilePath p
+
 -- | The name portion of a Nix path.
 --
 -- 'unStorePathName' must only contain a-zA-Z0-9+._?=-, can't start
@@ -67,7 +76,7 @@ newtype StorePathName = StorePathName
 type StorePathHashAlgo = 'Truncated 20 'SHA256
 
 -- | A set of 'StorePath's.
-type StorePathSet storeDir = HashSet (StorePath storeDir)
+type StorePathSet = HashSet StorePath
 
 -- | An address for a content-addressable store path, i.e. one whose
 -- store path hash is purely a function of its contents (as opposed to
@@ -99,103 +108,67 @@ data NarHashMode
     -- file if so desired.
     Recursive
 
--- | A type-level representation of the root directory of a Nix store.
---
--- The extra complexity of type indices requires justification.
--- Fundamentally, this boils down to the fact that there is little
--- meaningful sense in which 'StorePath's rooted at different
--- directories are of the same type, i.e. there are few if any
--- non-trivial non-contrived functions or data types that could
--- equally well accept 'StorePath's from different stores. In current
--- practice, any real application dealing with Nix stores (including,
--- in particular, the Nix expression language) only operates over one
--- store root and only cares about 'StorePath's belonging to that
--- root. One could imagine a use case that cares about multiple store
--- roots at once (e.g. the normal \/nix\/store along with some private
--- store at \/root\/nix\/store to contain secrets), but in that case
--- distinguishing 'StorePath's that belong to one store or the other
--- is even /more/ critical: Most operations will only be correct over
--- one of the stores or another, and it would be an error to mix and
--- match (e.g. a 'StorePath' in one store could not legitimately refer
--- to one in another).
---
--- As of @5886bc5996537fbf00d1fcfbb29595b8ccc9743e@, the C++ Nix
--- codebase contains 30 separate places where we assert that a given
--- store dir is, in fact, in the store we care about; those run-time
--- assertions could be completely removed if we had stronger types
--- there. Moreover, there are dozens of other cases where input coming
--- from the user, from serializations, etc. is parsed and then
--- required to be in the appropriate store; this case is the
--- equivalent of an existentially quantified version of 'StorePath'
--- and, notably, requiring at runtime that the index matches the
--- ambient store directory we're working in. In every case where a
--- path is treated as a store path, there is exactly one legitimate
--- candidate for the store directory it belongs to.
---
--- It may be instructive to consider the example of "chroot stores".
--- Since Nix 2.0, it has been possible to have a store actually live
--- at one directory (say, $HOME\/nix\/store) with a different
--- effective store directory (say, \/nix\/store). Nix can build into
--- a chroot store by running the builds in a mount namespace where the
--- store is at the effective store directory, can download from a
--- binary cache containing paths for the effective store directory,
--- and can run programs in the store that expect to be living at the
--- effective store directory (via nix run). When viewed as store paths
--- (rather than random files in the filesystem), paths in a chroot
--- store have nothing in common with paths in a non-chroot store that
--- lives in the same directory, and a lot in common with paths in a
--- non-chroot store that lives in the effective store directory of the
--- store in question. Store paths in stores with the same effective
--- store directory share the same hashing scheme, can be copied
--- between each other, etc. Store paths in stores with different
--- effective store directories have no relationship to each other that
--- they don't have to arbitrary other files.
-type StoreDir = Symbol
-
--- | Smart constructor for 'StorePathName' that ensures the underlying
--- content invariant is met.
-makeStorePathName :: Text -> Maybe StorePathName
-makeStorePathName n = case matchTest storePathNameRegex n of
-  True  -> Just $ StorePathName n
-  False -> Nothing
-
--- | Regular expression to match valid store path names.
-storePathNameRegex :: Regex
-storePathNameRegex = makeRegex r
+makeStorePathName :: Text -> Either String StorePathName
+makeStorePathName n = case validStorePathName n of
+  True  -> Right $ StorePathName n
+  False -> Left $ reasonInvalid n
+
+reasonInvalid n | n == ""            = "Empty name"
+reasonInvalid n | (T.length n > 211) = "Path too long"
+reasonInvalid n | (T.head n == '.')  = "Leading dot"
+reasonInvalid n | otherwise          = "Invalid character"
+
+validStorePathName "" = False
+validStorePathName n  = (T.length n <= 211)
+                        && T.head n /= '.'
+                        && T.all validChar n
   where
-    r :: String
-    r = "[a-zA-Z0-9\\+\\-\\_\\?\\=][a-zA-Z0-9\\+\\-\\.\\_\\?\\=]*"
+  validChar c = any ($ c) $
+    [ isAsciiLower -- 'a'..'z'
+    , isAsciiUpper -- 'A'..'Z'
+    , isDigit
+    ] ++
+    map (==) "+-._?="
 
 -- | Copied from @RawFilePath@ in the @unix@ package, duplicated here
 -- to avoid the dependency.
 type RawFilePath = ByteString
 
 -- | Render a 'StorePath' as a 'RawFilePath'.
 storePathToRawFilePath
-  :: forall storeDir . (KnownStoreDir storeDir)
-  => StorePath storeDir
+  :: StorePath
   -> RawFilePath
-storePathToRawFilePath (StorePath {..}) = BS.concat
+storePathToRawFilePath StorePath {..} = BS.concat
     [ root
     , "/"
     , hashPart
     , "-"
     , name
     ]
   where
-    root = storeDirVal @storeDir
+    root = BC.pack storePathRoot
     hashPart = encodeUtf8 $ encodeBase32 storePathHash
     name = encodeUtf8 $ unStorePathName storePathName
 
--- | Get a value-level representation of a 'KnownStoreDir'
-storeDirVal :: forall storeDir . (KnownStoreDir storeDir)
-            => ByteString
-storeDirVal = BC.pack $ symbolVal @storeDir Proxy
+storePathToNarinfo StorePath {..} = BS.concat
+    [ encodeUtf8 $ encodeBase32 storePathHash
+    , ".narinfo"
+    ]
 
--- | A 'StoreDir' whose value is known at compile time.
---
--- A valid instance of 'KnownStoreDir' should represent a valid path,
--- i.e. all "characters" fit into bytes (as determined by the logic of
--- 'BC.pack') and there are no 0 "characters". Currently this is not
--- enforced, but it should be.
-type KnownStoreDir = KnownSymbol
+-- | Parse `StorePath` from `BC.ByteString`, checking
+-- that store directory matches `expectedRoot`.
+parsePath :: FilePath -> BC.ByteString -> Either String StorePath
+parsePath expectedRoot x =
+  let
+    (rootDir, fname) = splitFileName . BC.unpack $ x
+    (digestPart, namePart) = T.breakOn "-" $ T.pack fname
+    digest = decodeBase32 digestPart
+    name = makeStorePathName . T.drop 1 $ namePart
+    --rootDir' = dropTrailingPathSeparator rootDir
+    -- cannot use ^^ as it drops multiple slashes /a/b/// -> /a/b
+    rootDir' = init rootDir
+    storeDir = if expectedRoot == rootDir'
+      then Right rootDir'
+      else Left $ unwords $ [ "Root store dir mismatch, expected ", expectedRoot, "got", rootDir']
+  in
+    StorePath <$> digest <*> name <*> storeDir

hnix-store-core/src/System/Nix/ReadonlyStore.hs

@@ -6,36 +6,37 @@ module System.Nix.ReadonlyStore where
 
 import           Data.ByteString (ByteString)
 import qualified Data.ByteString as BS
+import qualified Data.Text as T
 import qualified Data.HashSet as HS
 import           Data.Text.Encoding
 import           System.Nix.Hash
 import           System.Nix.StorePath
 
-makeStorePath :: forall storeDir hashAlgo . (KnownStoreDir storeDir, NamedAlgo hashAlgo) => ByteString -> Digest hashAlgo -> StorePathName -> StorePath storeDir
-makeStorePath ty h nm = StorePath storeHash nm
+makeStorePath :: forall hashAlgo . (NamedAlgo hashAlgo) => FilePath -> ByteString -> Digest hashAlgo -> StorePathName -> StorePath
+makeStorePath fp ty h nm = StorePath storeHash nm fp
   where
     s = BS.intercalate ":"
       [ ty
       , encodeUtf8 $ algoName @hashAlgo
       , encodeUtf8 $ encodeBase16 h
-      , storeDirVal @storeDir
+      , encodeUtf8 $ T.pack fp
       , encodeUtf8 $ unStorePathName nm
       ]
     storeHash = hash s
 
-makeTextPath :: (KnownStoreDir storeDir) => StorePathName -> Digest 'SHA256 -> StorePathSet storeDir -> StorePath storeDir
-makeTextPath nm h refs = makeStorePath ty h nm
+makeTextPath :: FilePath -> StorePathName -> Digest 'SHA256 -> StorePathSet -> StorePath
+makeTextPath fp nm h refs = makeStorePath fp ty h nm
   where
     ty = BS.intercalate ":" ("text" : map storePathToRawFilePath (HS.toList refs))
 
-makeFixedOutputPath :: forall storeDir hashAlgo. (KnownStoreDir storeDir, ValidAlgo hashAlgo, NamedAlgo hashAlgo) => Bool -> Digest hashAlgo -> StorePathName -> StorePath storeDir
-makeFixedOutputPath recursive h nm =
-  makeStorePath ty h' nm
+makeFixedOutputPath :: (ValidAlgo hashAlgo, NamedAlgo hashAlgo) => FilePath -> Bool -> Digest hashAlgo -> StorePathName -> StorePath storeDir
+makeFixedOutputPath fp recursive h nm =
+  makeStorePath fp ty h' nm
   where
     (ty, h') =
       if recursive && algoName @hashAlgo == algoName @'SHA256
       then ("source", h)
       else ("output:out", hash ("fixed:out:" <> encodeUtf8 (encodeBase16 h) <> ":"))
 
-computeStorePathForText :: (KnownStoreDir storeDir) => StorePathName -> ByteString -> StorePathSet storeDir -> StorePath storeDir
-computeStorePathForText nm s refs = makeTextPath nm (hash s) refs
+computeStorePathForText :: FilePath -> StorePathName -> ByteString -> StorePathSet -> StorePath
+computeStorePathForText fp nm s refs = makeTextPath fp nm (hash s) refs

hnix-store-core/src/System/Nix/StorePath.hs

@@ -7,17 +7,15 @@ module System.Nix.StorePath
   , StorePathName
   , StorePathSet
   , StorePathHashAlgo
-  , StoreDir
   , ContentAddressableAddress(..)
   , NarHashMode(..)
   , -- * Manipulating 'StorePathName'
     makeStorePathName
   , unStorePathName
-  , storePathNameRegex
+  , validStorePathName
   , -- * Rendering out 'StorePath's
     storePathToRawFilePath
-  , storeDirVal
-  , KnownStoreDir
+  , parsePath
   ) where
 
 import System.Nix.Internal.StorePath

hnix-store-core/src/System/Nix/StorePathMetadata.hs

@@ -10,18 +10,18 @@ import Data.Time (UTCTime)
 import Data.Word (Word64)
 import System.Nix.Signature (NarSignature)
 
--- | Metadata about a 'StorePath' in @storeDir@.
-data StorePathMetadata storeDir = StorePathMetadata
+-- | Metadata about a 'StorePath'
+data StorePathMetadata = StorePathMetadata
   { -- | The path this metadata is about
-    path :: !(StorePath storeDir)
+    path :: !StorePath
   , -- | The path to the derivation file that built this path, if any
     -- and known.
-    deriverPath :: !(Maybe (StorePath storeDir))
+    deriverPath :: !(Maybe StorePath)
   , -- TODO should this be optional?
     -- | The hash of the nar serialization of the path.
     narHash :: !SomeNamedDigest
   , -- | The paths that this path directly references
-    references :: !(StorePathSet storeDir)
+    references :: !StorePathSet
   , -- | When was this path registered valid in the store?
     registrationTime :: !UTCTime
   , -- | The size of the nar serialization of the path, in bytes.