Merge pull request #255 from haskell-nix/srk/daemon

cereal remote, server side integration

Author: sorki

.github/workflows/ci.dhall.frozen

@@ -1,6 +1,6 @@
 let haskellCi =
       https://raw.githubusercontent.com/sorki/github-actions-dhall/main/haskell-ci.dhall
-        sha256:5d7058a7684fd5315467b562853bd1c4a43da691c09293d3715ee739dfa26e08
+        sha256:a39801f73d93c6e0f91942755ef8ae4c50947e9a9b180b6724957229470f7b8d
 
 let defSteps = haskellCi.defaultCabalSteps
 

.github/workflows/ci.yaml

@@ -59,7 +59,6 @@ jobs:
         - "macos-latest"
 name: Haskell CI
 'on':
-  pull_request: {}
   push: {}
   schedule:
   - cron: "4 20 10 * *"

cabal.project

@@ -20,4 +20,4 @@ package hnix-store-nar
   flags: +bounded_memory
 
 package hnix-store-remote
-  flags: +build-readme +io-testsuite
+  flags: +build-derivation +build-readme +io-testsuite

hnix-store-core/src/System/Nix/Base.hs

@@ -6,6 +6,7 @@ module System.Nix.Base
 
 import Data.ByteString (ByteString)
 import Data.Text (Text)
+import GHC.Generics (Generic)
 
 import qualified Data.Text.Encoding
 import qualified Data.ByteString.Base16
@@ -21,6 +22,7 @@ data BaseEncoding
   -- & NixBase seems be the most widely used in Nix.
   | Base16
   | Base64
+  deriving (Bounded, Eq, Enum, Generic, Ord, Show)
 
 -- | Encode @ByteString@ with @Base@ encoding, produce @Text@.
 encodeWith :: BaseEncoding -> ByteString -> Text

hnix-store-core/src/System/Nix/Build.hs

@@ -15,25 +15,28 @@ import GHC.Generics (Generic)
 
 -- keep the order of these Enums to match enums from reference implementations
 -- src/libstore/store-api.hh
-data BuildMode = Normal | Repair | Check
+data BuildMode
+  = BuildMode_Normal
+  | BuildMode_Repair
+  | BuildMode_Check
   deriving (Eq, Generic, Ord, Enum, Show)
 
 data BuildStatus =
-    Built
-  | Substituted
-  | AlreadyValid
-  | PermanentFailure
-  | InputRejected
-  | OutputRejected
-  | TransientFailure -- possibly transient
-  | CachedFailure    -- no longer used
-  | TimedOut
-  | MiscFailure
-  | DependencyFailed
-  | LogLimitExceeded
-  | NotDeterministic
-  | ResolvesToAlreadyValid
-  | NoSubstituters
+    BuildStatus_Built
+  | BuildStatus_Substituted
+  | BuildStatus_AlreadyValid
+  | BuildStatus_PermanentFailure
+  | BuildStatus_InputRejected
+  | BuildStatus_OutputRejected
+  | BuildStatus_TransientFailure -- possibly transient
+  | BuildStatus_CachedFailure    -- no longer used
+  | BuildStatus_TimedOut
+  | BuildStatus_MiscFailure
+  | BuildStatus_DependencyFailed
+  | BuildStatus_LogLimitExceeded
+  | BuildStatus_NotDeterministic
+  | BuildStatus_ResolvesToAlreadyValid
+  | BuildStatus_NoSubstituters
   deriving (Eq, Generic, Ord, Enum, Show)
 
 -- | Result of the build
@@ -55,4 +58,8 @@ data BuildResult = BuildResult
 
 buildSuccess :: BuildResult -> Bool
 buildSuccess BuildResult {..} =
-  status `elem` [Built, Substituted, AlreadyValid]
+  status `elem`
+    [ BuildStatus_Built
+    , BuildStatus_Substituted
+    , BuildStatus_AlreadyValid
+    ]

hnix-store-core/src/System/Nix/Hash.hs

@@ -167,7 +167,7 @@ decodeDigestWith b x =
     let
       toEither =
         maybeToRight
-          ("Cryptonite was not able to convert '(ByteString -> Digest a)' for: '" <> show bs <>"'.")
+          ("Crypton was not able to convert '(ByteString -> Digest a)' for: '" <> show bs <>"'.")
     (toEither . Crypto.Hash.digestFromByteString) bs
   where
     -- To not depend on @extra@

hnix-store-core/src/System/Nix/StorePath.hs

@@ -34,6 +34,7 @@ module System.Nix.StorePath
   , pathParser
     -- * Utilities for tests
   , unsafeMakeStorePath
+  , unsafeMakeStorePathHashPart
   ) where
 
 import Crypto.Hash (HashAlgorithm)
@@ -307,3 +308,11 @@ unsafeMakeStorePath
   -> StorePathName
   -> StorePath
 unsafeMakeStorePath = StorePath
+
+-- | Path hash parts rarely need to be constructed directly.
+-- Prefer @mkStorePathHashPart@
+-- Used by remote store in wire protocol
+unsafeMakeStorePathHashPart
+  :: ByteString
+  -> StorePathHashPart
+unsafeMakeStorePathHashPart = StorePathHashPart

hnix-store-remote/app/BuildDerivation.hs

@@ -0,0 +1,41 @@
+{-# LANGUAGE LambdaCase #-}
+module Main where
+
+import Data.Default.Class (Default(def))
+import Data.Text (Text)
+import System.Nix.Derivation (Derivation)
+import System.Nix.StorePath (StorePath)
+
+import qualified Data.Text
+import qualified Data.Text.IO
+import qualified Data.Attoparsec.Text
+import qualified System.Environment
+import qualified System.Nix.Build
+import qualified System.Nix.Derivation
+import qualified System.Nix.StorePath
+import qualified System.Nix.Store.Remote
+
+parseDerivation :: FilePath -> IO (Derivation StorePath Text)
+parseDerivation source = do
+  contents <- Data.Text.IO.readFile source
+  case Data.Attoparsec.Text.parseOnly
+    (System.Nix.Derivation.parseDerivation def) contents of
+      Left e -> error e
+      Right drv -> pure drv
+
+main :: IO ()
+main = System.Environment.getArgs >>= \case
+  [filename] -> do
+    case System.Nix.StorePath.parsePathFromText def (Data.Text.pack filename) of
+      Left e -> error $ show e
+      Right p -> do
+        d <- parseDerivation filename
+        out <-
+            System.Nix.Store.Remote.runStore
+          $ System.Nix.Store.Remote.buildDerivation
+              p
+              d
+              System.Nix.Build.BuildMode_Normal
+        print out
+  _ -> error "No input derivation file"
+

hnix-store-remote/hnix-store-remote.cabal

@@ -26,14 +26,17 @@ common commons
     , DeriveFoldable
     , DeriveTraversable
     , DeriveLift
+    , DerivingVia
     , FlexibleContexts
     , FlexibleInstances
+    , GADTs
     , RecordWildCards
     , ScopedTypeVariables
     , StandaloneDeriving
     , TypeApplications
     , TypeSynonymInstances
     , InstanceSigs
+    , KindSignatures
     , MultiParamTypeClasses
     , TupleSections
     , LambdaCase
@@ -53,6 +56,12 @@ flag io-testsuite
     Enable testsuite, which requires external
     binaries and Linux namespace support.
 
+flag build-derivation
+  default:
+    False
+  description:
+    Build build-derivation executable
+
 flag build-readme
   default:
     False
@@ -62,37 +71,68 @@ flag build-readme
 library
   import: commons
   exposed-modules:
-      System.Nix.Store.Remote
-    , System.Nix.Store.Remote.Binary
-    , System.Nix.Store.Remote.Serialize
-    , System.Nix.Store.Remote.Serialize.Prim
+      Data.Serializer
+    , Data.Serializer.Example
+    , System.Nix.Store.Remote
+    , System.Nix.Store.Remote.Arbitrary
     , System.Nix.Store.Remote.Logger
+    , System.Nix.Store.Remote.MonadStore
     , System.Nix.Store.Remote.Protocol
+    , System.Nix.Store.Remote.Serialize
+    , System.Nix.Store.Remote.Serialize.Prim
+    , System.Nix.Store.Remote.Serializer
+    , System.Nix.Store.Remote.Socket
     , System.Nix.Store.Remote.Types
-    , System.Nix.Store.Remote.Util
+    , System.Nix.Store.Remote.Types.Activity
+    , System.Nix.Store.Remote.Types.CheckMode
+    , System.Nix.Store.Remote.Types.GC
+    , System.Nix.Store.Remote.Types.Logger
+    , System.Nix.Store.Remote.Types.ProtoVersion
+    , System.Nix.Store.Remote.Types.StoreConfig
+    , System.Nix.Store.Remote.Types.SubstituteMode
+    , System.Nix.Store.Remote.Types.Verbosity
+    , System.Nix.Store.Remote.Types.WorkerOp
 
   build-depends:
       base >=4.12 && <5
     , hnix-store-core >= 0.8 && <0.9
     , hnix-store-nar >= 0.1
     , attoparsec
-    , binary
     , bytestring
     , cereal
     , containers
     , crypton
     , data-default-class
     , dependent-sum > 0.7 && < 1
+    , generic-arbitrary < 1.1
+    , hashable
     , text
     , time
+    , transformers
     , network
     , mtl
+    , QuickCheck
+    , quickcheck-instances
     , unordered-containers
-    , transformers
     , vector
   hs-source-dirs:      src
   ghc-options:         -Wall
 
+executable build-derivation
+  if !flag(build-derivation)
+    buildable: False
+  build-depends:
+      base >=4.12 && <5
+    , attoparsec
+    , hnix-store-core
+    , hnix-store-remote
+    , data-default-class
+    , text
+  default-language: Haskell2010
+  main-is: BuildDerivation.hs
+  hs-source-dirs:      app
+  ghc-options: -Wall -threaded -rtsopts "-with-rtsopts -N"
+
 executable remote-readme
   if !flag(build-readme)
     buildable: False
@@ -110,8 +150,10 @@ test-suite remote
   type:              exitcode-stdio-1.0
   main-is:           Driver.hs
   hs-source-dirs:    tests
-  ghc-options:       -Wall
+  ghc-options:       -Wall -threaded -rtsopts "-with-rtsopts -N"
   other-modules:
+      Data.SerializerSpec
+      NixSerializerSpec
       SerializeSpec
   build-tool-depends:
     hspec-discover:hspec-discover
@@ -121,6 +163,9 @@ test-suite remote
     , hnix-store-remote
     , hnix-store-tests
     , cereal
+    , crypton
+    , dependent-sum > 0.7 && < 1
+    , some > 1.0.5 && < 2
     , text
     , time
     , hspec
@@ -153,14 +198,14 @@ test-suite remote-io
     , containers
     , crypton
     , directory
-    , process
     , filepath
+    , hspec
     , hspec-expectations-lifted
-    , text
+    , linux-namespaces
+    , process
     , tasty
-    , hspec
     , tasty-hspec
-    , linux-namespaces
     , temporary
+    , text
     , unix
     , unordered-containers