Merge pull request #963 from IntersectMBO/mgalazyn/fix/restore-cardano-rpc

cardano-rpc | Use buf for code generation instead of cabal/protoc integration

Author: carbolymer

.gitattributes

@@ -0,0 +1,3 @@
+cardano-rpc/gen/** linguist-generated=true
+cardano-rpc/gen/** no-prettify
+

.github/workflows/haskell.yml

@@ -86,7 +86,12 @@ jobs:
 
     - name: "[Windows] Install grpc dependencies"
       if: runner.os == 'Windows'
-      run: /usr/bin/pacman --noconfirm -S mingw-w64-x86_64-snappy mingw-w64-x86_64-protobuf
+      run: |
+        /usr/bin/pacman --noconfirm -S mingw-w64-x86_64-snappy mingw-w64-x86_64-protobuf
+        cat <<EOF >> $GITHUB_ENV
+        LIBRARY_PATH=/mingw64/lib
+        CPATH=/mingw64/include
+        EOF
 
     - name: "[macOS] Install grpc dependencies"
       if: runner.os == 'macOS'
@@ -107,6 +112,34 @@ jobs:
         cabal-store: ${{ steps.setup-haskell.outputs.cabal-store }}
         cache-version: ${{ env.CABAL_CACHE_VERSION }}
 
+
+    - name: '[Linux] [cardano-rpc] Install buf'
+      if: runner.os == 'Linux'
+      run: |
+        curl -sSL "https://github.com/bufbuild/buf/releases/latest/download/buf-Linux-x86_64" -o "/usr/local/bin/buf"
+        chmod +x /usr/local/bin/buf
+
+    - name: '[Linux] [cardano-rpc] Install proto-lens-protoc'
+      if: runner.os == 'Linux'
+      run: |
+        cabal install proto-lens-protoc --installdir=$HOME/.local/bin
+
+    - name: '[Linux] [cardano-rpc] Generate protobuf code'
+      if: runner.os == 'Linux'
+      working-directory: cardano-rpc
+      run: buf generate proto
+
+    - name: '[Linux] [cardano-rpc] Check that generated files from proto definitions are up to date'
+      if: runner.os == 'Linux'
+      run: |
+        git add cardano-rpc/gen
+        if ! git diff --staged --quiet -- cardano-rpc/gen; then
+          echo "Generated files differ from repository files:"
+          git diff --staged --name-status -- cardano-rpc/gen
+          exit 1
+        fi
+        echo "All generated files match repository files"
+
     # Now we build.
     - name: Build all
       run: cabal build all --enable-tests

.github/workflows/hls.yml

@@ -119,6 +119,4 @@ jobs:
           cradle:
             cabal:
           EOF
-          # Remove the following line after `cardano-rpc` project is enabled in `cabal.project`
-          mv cardano-rpc ../
           haskell-language-server

.hlint.yaml

@@ -60,6 +60,9 @@
 # Allow snake_case naming in test modules (following Haskell test property convention)
 - ignore: {name: Use camelCase, within: [Test.Cardano.Api.**, Test.Golden.Cardano.Api.**]}
 
+# Ignore all files in cardano-rpc/gen (generated code)
+- ignore: {within: [Proto.Cardano, Proto.Utxorpc]}
+
 - ignore: {name: Eta reduce}
 - ignore: {name: Use + directly}
 - ignore: {name: Use fmap}

cabal.project

@@ -20,15 +20,9 @@ packages:
     cardano-api
     cardano-api-gen
     cardano-wasm
-    -- TODO fix potential issues with build-type: Custom and protoc and reenable
-    -- When uncommenting this line, remember to reenable in .github/workflows/hls.yml
-    -- by removing the line that moves it away before calling `haskell-language-server`.
-    -- cardano-rpc
+    cardano-rpc
 
-extra-packages: Cabal, process
-
-if impl(ghc < 9.8)
-  constraints: interpolatedstring-perl6:setup.time source
+extra-packages: Cabal
 
 -- It may slow down build plan preparation, but without it cabal has problems
 -- with solving constraints. Remove this when not needed anymore.

cardano-rpc/README.md

@@ -2,7 +2,7 @@
 
 ## What's this
 
-The `cardano-rpc` package provides client and server haskell modules for gRPC interface of `cardano-node`. 
+The `cardano-rpc` package provides client and server haskell modules for gRPC interface of `cardano-node`.
 It implements [UTxO RPC](https://utxorpc.org/introduction) protobuf communication protocol specification.
 
 ## Building
@@ -14,7 +14,24 @@ You need the following dependencies installed on your system:
 - [`snappy`](https://github.com/google/snappy) development files (`libsnappy-dev` in Ubuntu)
 - [`protobuf`](https://developers.google.com/protocol-buffers/) compiler (`protobuf-compiler` in Ubuntu)
 
-Then do:
+### Generating the Haskell code from proto definitions
+
+You need to install `buf` and `proto-lens-protoc`.
+1. Follow the `buf` installation guide at: https://buf.build/docs/cli/installation/
+1. To install Haskell protobuf code compiler:
+    ```bash
+    cabal install proto-lens-protoc
+    ```
+
+1. Generate Haskell code using:
+    ```bash
+    ( cd cardano-rpc/ ; buf generate proto )
+    ```
+    This will output the generated Haskell code into `cardano-rpc/gen` directory.
+
+### Building the haskell code
+
+To build the package use the following command:
 ```bash
 cabal build cardano-rpc
 ```

cardano-rpc/Setup.hs

@@ -0,0 +1,26 @@
+version: v1
+
+managed:
+  enabled: true
+
+plugins:
+  # node
+  # - plugin: es
+  #   out: gen/node/src
+  #   opt:
+  #     - target=ts
+  #
+  # - plugin: connect-es
+  #   out: gen/node/src
+  #   opt:
+  #     - target=ts
+  #
+  # - plugin: buf.build/bufbuild/protoschema-jsonschema
+  #   out: gen/jsonschema/schema
+
+  # haskell
+
+  - plugin: haskell-protolens
+    path: proto-lens-protoc
+    out: gen
+

cardano-rpc/buf.gen.yaml

@@ -17,26 +17,17 @@ license-files:
   LICENSE
   NOTICE
 
-build-type: Custom
+build-type: Simple
 extra-doc-files:
   CHANGELOG.md
   README.md
 
-extra-source-files: proto/**/*.proto
-
-custom-setup
-  setup-depends:
-    Cabal >=3.0 && <3.13,
-    base >=4.14 && <5,
-    proto-lens-setup >=0.4 && <0.5,
-
 common project-config
   default-language: Haskell2010
   default-extensions:
     ImportQualifiedPost
     OverloadedStrings
 
-  build-depends: base >=4.14 && <5
   ghc-options:
     -Wall
     -Wcompat
@@ -50,7 +41,6 @@ common project-config
 library
   import: project-config
   hs-source-dirs: src
-  build-tool-depends: proto-lens-protoc:proto-lens-protoc
   exposed-modules:
     Cardano.Rpc.Client
     Cardano.Rpc.Proto.Api.Node
@@ -65,29 +55,13 @@ library
     Cardano.Rpc.Server.Internal.UtxoRpc.Query
     Cardano.Rpc.Server.Internal.UtxoRpc.Submit
     Cardano.Rpc.Server.Internal.UtxoRpc.Type
-    Proto.Cardano.Rpc.Node
-    Proto.Cardano.Rpc.Node_Fields
-    Proto.Utxorpc.V1alpha.Cardano.Cardano
-    Proto.Utxorpc.V1alpha.Cardano.Cardano_Fields
-    Proto.Utxorpc.V1alpha.Query.Query
-    Proto.Utxorpc.V1alpha.Query.Query_Fields
-    Proto.Utxorpc.V1alpha.Submit.Submit
-    Proto.Utxorpc.V1alpha.Submit.Submit_Fields
 
   other-modules:
     Cardano.Rpc.Server.Internal.Orphans
     Paths_cardano_rpc
 
   autogen-modules:
     Paths_cardano_rpc
-    Proto.Cardano.Rpc.Node
-    Proto.Cardano.Rpc.Node_Fields
-    Proto.Utxorpc.V1alpha.Cardano.Cardano
-    Proto.Utxorpc.V1alpha.Cardano.Cardano_Fields
-    Proto.Utxorpc.V1alpha.Query.Query
-    Proto.Utxorpc.V1alpha.Query.Query_Fields
-    Proto.Utxorpc.V1alpha.Submit.Submit
-    Proto.Utxorpc.V1alpha.Submit.Submit_Fields
 
   build-depends:
     aeson,
@@ -98,6 +72,7 @@ library
     cardano-ledger-binary,
     cardano-ledger-conway,
     cardano-ledger-core,
+    cardano-rpc:gen,
     containers,
     contra-tracer,
     data-default,
@@ -107,24 +82,44 @@ library
     grpc-spec,
     proto-lens,
     proto-lens-protobuf-types,
-    proto-lens-runtime,
     rio,
     text,
 
+-- this should be replaced by utxorpc package from hackage
+-- ideally we should upstream whatever is implemented in Proto.Cardano.Rpc.Node
+-- into utxorpc
+library gen
+  import: project-config
+  hs-source-dirs: gen
+  exposed-modules:
+    Proto.Cardano.Rpc.Node
+    Proto.Cardano.Rpc.Node_Fields
+    Proto.Utxorpc.V1alpha.Cardano.Cardano
+    Proto.Utxorpc.V1alpha.Cardano.Cardano_Fields
+    Proto.Utxorpc.V1alpha.Query.Query
+    Proto.Utxorpc.V1alpha.Query.Query_Fields
+    Proto.Utxorpc.V1alpha.Submit.Submit
+    Proto.Utxorpc.V1alpha.Submit.Submit_Fields
+
+  build-depends:
+    proto-lens-protobuf-types,
+    proto-lens-runtime,
+
 test-suite cardano-rpc-test
   import: project-config
   hs-source-dirs: test/cardano-rpc-test
   main-is: cardano-rpc-test.hs
   type: exitcode-stdio-1.0
   build-depends:
+    base,
     cardano-api,
     cardano-api:gen,
     cardano-ledger-api,
     cardano-ledger-conway,
     cardano-ledger-core,
     cardano-rpc,
     containers,
-    hedgehog >=1.1,
+    hedgehog,
     rio,
     tasty,
     tasty-hedgehog,