Skip to content

Commit 7782e5b

Browse files
Ericson2314Ericson2314
authored
Merge pull request #288 from obsidiansystems/content-address-method-cleanup
Content address method cleanup
2 parents dd69c4c + ce1d7da commit 7782e5b

File tree

15 files changed

+162
-117
lines changed

15 files changed

+162
-117
lines changed

‎hnix-store-core/hnix-store-core.cabal‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ library
6464
, System.Nix.Derivation
6565
, System.Nix.DerivedPath
6666
, System.Nix.Fingerprint
67+
, System.Nix.FileContentAddress
6768
, System.Nix.Hash
6869
, System.Nix.Hash.Truncation
6970
, System.Nix.OutputName

‎hnix-store-core/src/System/Nix/ContentAddress.hs‎

Lines changed: 17 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
{-# LANGUAGE OverloadedStrings #-}
22

33
module System.Nix.ContentAddress (
4-
ContentAddress
5-
, ContentAddressMethod
6-
, FileIngestionMethod
4+
ContentAddress (..)
5+
, ContentAddressMethod (..)
76
, contentAddressBuilder
87
, contentAddressParser
98
, buildContentAddress
@@ -18,19 +17,16 @@ import Data.Text (Text)
1817
import Data.Text.Lazy.Builder (Builder)
1918
import GHC.Generics (Generic)
2019
import System.Nix.Hash (HashAlgo)
21-
import System.Nix.Store.Types (FileIngestionMethod(..))
2220

2321
import qualified Data.Attoparsec.Text
2422
import qualified Data.Text.Lazy
2523
import qualified Data.Text.Lazy.Builder
2624
import qualified System.Nix.Hash
2725

2826
data ContentAddressMethod
29-
= FileIngestionMethod !FileIngestionMethod
30-
-- ^ The path was added to the store via makeFixedOutputPath or
31-
-- addToStore. It is addressed according to some hash algorithm
32-
-- applied to the nar serialization via some 'NarHashMode'.
33-
| TextIngestionMethod
27+
= ContentAddressMethod_Flat
28+
| ContentAddressMethod_NixArchive
29+
| ContentAddressMethod_Text
3430
-- ^ The path is a plain file added via makeTextPath or
3531
-- addTextToStore. It is addressed according to a sha256sum of the
3632
-- file contents.
@@ -59,19 +55,14 @@ buildContentAddress =
5955
. contentAddressBuilder
6056

6157
contentAddressBuilder :: ContentAddress -> Builder
62-
contentAddressBuilder (ContentAddress method digest) = case method of
63-
TextIngestionMethod ->
64-
"text:"
65-
<> System.Nix.Hash.algoDigestBuilder digest
66-
FileIngestionMethod r ->
67-
"fixed:"
68-
<> fileIngestionMethodBuilder r
69-
<> System.Nix.Hash.algoDigestBuilder digest
70-
71-
fileIngestionMethodBuilder :: FileIngestionMethod -> Builder
72-
fileIngestionMethodBuilder = \case
73-
FileIngestionMethod_Flat -> ""
74-
FileIngestionMethod_FileRecursive -> "r:"
58+
contentAddressBuilder (ContentAddress method digest) =
59+
(case method of
60+
ContentAddressMethod_Text -> "text"
61+
ContentAddressMethod_NixArchive -> "fixed:r"
62+
ContentAddressMethod_Flat -> "fixed"
63+
)
64+
<> ":"
65+
<> System.Nix.Hash.algoDigestBuilder digest
7566

7667
-- | Parse `ContentAddressableAddress` from `ByteString`
7768
parseContentAddress
@@ -83,17 +74,17 @@ parseContentAddress =
8374
contentAddressParser :: Parser ContentAddress
8475
contentAddressParser = do
8576
method <- parseContentAddressMethod
77+
_ <- ":"
8678
digest <- parseTypedDigest
8779
case digest of
8880
Left e -> fail e
8981
Right x -> return $ ContentAddress method x
9082

9183
parseContentAddressMethod :: Parser ContentAddressMethod
9284
parseContentAddressMethod =
93-
TextIngestionMethod <$ "text:"
94-
<|> FileIngestionMethod <$ "fixed:"
95-
<*> (FileIngestionMethod_FileRecursive <$ "r:"
96-
<|> pure FileIngestionMethod_Flat)
85+
(ContentAddressMethod_Text <$ "text")
86+
<|> (ContentAddressMethod_NixArchive <$ "fixed:r")
87+
<|> (ContentAddressMethod_Flat <$ "fixed")
9788

9889
parseTypedDigest :: Parser (Either String (DSum HashAlgo Digest))
9990
parseTypedDigest = System.Nix.Hash.mkNamedDigest <$> parseHashType <*> parseHash

‎hnix-store-core/src/System/Nix/FileContentAddress.hs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
module System.Nix.FileContentAddress
2+
( FileIngestionMethod(..)
3+
) where
4+
5+
import GHC.Generics (Generic)
6+
7+
data FileIngestionMethod
8+
= FileIngestionMethod_Flat
9+
| FileIngestionMethod_NixArchive
10+
deriving (Bounded, Eq, Generic, Enum, Ord, Show)

‎hnix-store-core/src/System/Nix/Store/Types.hs‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,11 @@
1+
-- | TODO rename module
12
module System.Nix.Store.Types
2-
( FileIngestionMethod(..)
3-
, PathFilter(..)
3+
( PathFilter(..)
44
, RepairMode(..)
55
) where
66

77
import GHC.Generics (Generic)
88

9-
-- | Add path recursively or not
10-
data FileIngestionMethod
11-
= FileIngestionMethod_Flat
12-
| FileIngestionMethod_FileRecursive
13-
deriving (Bounded, Eq, Generic, Enum, Ord, Show)
14-
159
-- | Path filtering function
1610
newtype PathFilter = PathFilter
1711
{ pathFilterFunction :: FilePath -> Bool

‎hnix-store-readonly/src/System/Nix/Store/ReadOnly.hs‎

Lines changed: 80 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1+
{-# LANGUAGE GADTs #-}
12
{-# LANGUAGE OverloadedStrings #-}
23

34
module System.Nix.Store.ReadOnly
4-
( makeStorePath
5-
, makeTextPath
5+
( References(..)
6+
, makeStorePath
67
, makeFixedOutputPath
7-
, computeStorePathForText
88
, computeStorePathForPath
99
) where
1010

@@ -15,8 +15,9 @@ import Data.Constraint.Extras (Has(has))
1515
import Data.Dependent.Sum (DSum((:=>)))
1616
import Data.HashSet (HashSet)
1717
import Data.Some (Some(Some))
18+
import System.Nix.ContentAddress (ContentAddressMethod (..))
1819
import System.Nix.Hash (BaseEncoding(Base16), HashAlgo(..))
19-
import System.Nix.Store.Types (FileIngestionMethod(..), PathFilter, RepairMode)
20+
import System.Nix.Store.Types (PathFilter, RepairMode)
2021
import System.Nix.StorePath (StoreDir, StorePath, StorePathName)
2122

2223
import qualified Crypto.Hash
@@ -30,6 +31,23 @@ import qualified System.Nix.Hash
3031
import qualified System.Nix.Nar
3132
import qualified System.Nix.StorePath
3233

34+
data References = References
35+
{ references_others :: HashSet StorePath
36+
, references_self :: Bool
37+
}
38+
39+
instance Semigroup References where
40+
a <> b = References
41+
{ references_others = references_others a <> references_others b
42+
, references_self = references_self a || references_self b
43+
}
44+
45+
instance Monoid References where
46+
mempty = References
47+
{ references_others = mempty
48+
, references_self = False
49+
}
50+
3351
makeStorePath
3452
:: StoreDir
3553
-> ByteString
@@ -49,68 +67,64 @@ makeStorePath storeDir ty (hashAlgo :=> (digest :: Digest a)) nm =
4967
, System.Nix.StorePath.unStorePathName nm
5068
]
5169

52-
makeTextPath
70+
makeType
5371
:: StoreDir
54-
-> StorePathName
55-
-> Digest SHA256
56-
-> HashSet StorePath
57-
-> StorePath
58-
makeTextPath storeDir nm h refs = makeStorePath storeDir ty (HashAlgo_SHA256 :=> h) nm
59-
where
60-
ty =
61-
Data.ByteString.intercalate
62-
":"
63-
$ "text"
64-
: Data.List.sort
65-
(System.Nix.StorePath.storePathToRawFilePath storeDir
66-
<$> Data.HashSet.toList refs)
72+
-> ByteString
73+
-> References
74+
-> ByteString
75+
makeType storeDir ty refs =
76+
Data.ByteString.intercalate ":" $ ty : (others ++ self)
77+
where
78+
others = Data.List.sort
79+
$ fmap (System.Nix.StorePath.storePathToRawFilePath storeDir)
80+
$ Data.HashSet.toList
81+
$ references_others refs
82+
self = ["self" | references_self refs]
6783

6884
makeFixedOutputPath
6985
:: StoreDir
70-
-> FileIngestionMethod
86+
-> ContentAddressMethod
7187
-> DSum HashAlgo Digest
88+
-> References
7289
-> StorePathName
7390
-> StorePath
74-
makeFixedOutputPath storeDir recursive algoDigest@(hashAlgo :=> digest) =
75-
if recursive == FileIngestionMethod_FileRecursive
76-
&& Some hashAlgo == Some HashAlgo_SHA256
77-
then makeStorePath storeDir "source" algoDigest
78-
else makeStorePath storeDir "output:out" (HashAlgo_SHA256 :=> h')
91+
makeFixedOutputPath storeDir method digest@(hashAlgo :=> h) refs =
92+
makeStorePath storeDir ty digest'
7993
where
80-
h' =
81-
Crypto.Hash.hash @ByteString @SHA256
82-
$ "fixed:out:"
83-
<> Data.Text.Encoding.encodeUtf8 (System.Nix.Hash.algoToText hashAlgo)
84-
<> (if recursive == FileIngestionMethod_FileRecursive then ":r:" else ":")
85-
<> Data.Text.Encoding.encodeUtf8 (System.Nix.Hash.encodeDigestWith Base16 digest)
86-
<> ":"
87-
88-
computeStorePathForText
89-
:: StoreDir
90-
-> StorePathName
91-
-> ByteString
92-
-> (HashSet StorePath -> StorePath)
93-
computeStorePathForText storeDir nm =
94-
makeTextPath storeDir nm
95-
. Crypto.Hash.hash
94+
(ty, digest') = case method of
95+
ContentAddressMethod_Text ->
96+
case hashAlgo of
97+
HashAlgo_SHA256
98+
| references_self refs == False -> (makeType storeDir "text" refs, digest)
99+
_ -> error "unsupported" -- TODO do better; maybe we'll just remove this restriction too?
100+
_ ->
101+
if method == ContentAddressMethod_NixArchive
102+
&& Some hashAlgo == Some HashAlgo_SHA256
103+
then (makeType storeDir "source" refs, digest)
104+
else let
105+
h' =
106+
Crypto.Hash.hash @ByteString @SHA256
107+
$ "fixed:out:"
108+
<> Data.Text.Encoding.encodeUtf8 (System.Nix.Hash.algoToText hashAlgo)
109+
<> (if method == ContentAddressMethod_NixArchive then ":r:" else ":")
110+
<> Data.Text.Encoding.encodeUtf8 (System.Nix.Hash.encodeDigestWith Base16 h)
111+
<> ":"
112+
in ("output:out", HashAlgo_SHA256 :=> h')
96113

97-
computeStorePathForPath
98-
:: StoreDir
99-
-> StorePathName -- ^ Name part of the newly created `StorePath`
100-
-> FilePath -- ^ Local `FilePath` to add
101-
-> FileIngestionMethod -- ^ Add target directory recursively
114+
digestPath
115+
:: FilePath -- ^ Local `FilePath` to add
116+
-> ContentAddressMethod -- ^ target directory method
102117
-> PathFilter -- ^ Path filter function
103118
-> RepairMode -- ^ Only used by local store backend
104-
-> IO StorePath
105-
computeStorePathForPath storeDir name pth recursive _pathFilter _repair = do
106-
selectedHash <-
107-
if recursive == FileIngestionMethod_FileRecursive
108-
then recursiveContentHash
109-
else flatContentHash
110-
pure $ makeFixedOutputPath storeDir recursive (HashAlgo_SHA256 :=> selectedHash) name
119+
-> IO (Digest SHA256)
120+
digestPath pth method _pathFilter _repair =
121+
case method of
122+
ContentAddressMethod_Flat -> flatContentHash
123+
ContentAddressMethod_NixArchive -> nixArchiveContentHash
124+
ContentAddressMethod_Text -> flatContentHash
111125
where
112-
recursiveContentHash :: IO (Digest SHA256)
113-
recursiveContentHash =
126+
nixArchiveContentHash :: IO (Digest SHA256)
127+
nixArchiveContentHash =
114128
Crypto.Hash.hashFinalize
115129
<$> execStateT streamNarUpdate (Crypto.Hash.hashInit @SHA256)
116130

@@ -127,3 +141,15 @@ computeStorePathForPath storeDir name pth recursive _pathFilter _repair = do
127141
<$> System.Nix.Nar.narReadFile
128142
System.Nix.Nar.narEffectsIO
129143
pth
144+
145+
computeStorePathForPath
146+
:: StoreDir
147+
-> StorePathName -- ^ Name part of the newly created `StorePath`
148+
-> FilePath -- ^ Local `FilePath` to add
149+
-> ContentAddressMethod -- ^ Add target directory methodly
150+
-> PathFilter -- ^ Path filter function
151+
-> RepairMode -- ^ Only used by local store backend
152+
-> IO StorePath
153+
computeStorePathForPath storeDir name pth method pathFilter repair = do
154+
selectedHash <- digestPath pth method pathFilter repair
155+
pure $ makeFixedOutputPath storeDir method (HashAlgo_SHA256 :=> selectedHash) mempty name

0 commit comments

Comments
 (0)