Merge branch 'main' into remove-pre-commit-hooks

Author: stevenj

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -0,0 +1,42 @@
+# Description
+
+Please include a summary of the changes and the related issue.
+Please also include relevant motivation and context.
+List any dependencies that are required for this change.
+
+Fixes # (issue) _if applicable_
+
+## Type of change
+
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+## How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce.
+Please also list any relevant details for your test configuration
+
+- [ ] Test A
+- [ ] Test B
+
+**Test Configuration**: _if applicable_
+
+* Firmware version:
+* Hardware:
+* Toolchain:
+* SDK:
+
+## Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules

.github/workflows/nix.yml

@@ -26,6 +26,10 @@ concurrency:
   group: ${{ github.sha }}
   cancel-in-progress: true
 
+env:
+  ECR_REGISTRY: 432820653916.dkr.ecr.eu-central-1.amazonaws.com
+  S3_CACHE: s3://iog-catalyst-nix-cache?region=eu-central-1
+
 jobs:
   discover:
     outputs:
@@ -47,11 +51,13 @@ jobs:
     name: ${{ matrix.target.cell }} - ${{ matrix.target.name }}
     runs-on: ubuntu-latest
     steps:
-      - run: |
-          config="$HOME/.docker/config.json"
-          mkdir -p "${config%/*}"
-          jq -n --arg token "${{ secrets.DOCKER_AUTH_TOKEN }}" '{ "auths": { "registry.ci.iog.io": { auth: $token } } }' > "$config"
-          chmod 0600 "$config"
+      - name: Configure Registry
+        run: |
+          aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "${{ env.ECR_REGISTRY }}"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: eu-central-1
       - uses: divnix/std-action/run@main
         with:
           extra_nix_config: |
@@ -60,7 +66,7 @@ jobs:
           nix_key: ${{ secrets.NIX_SIGNING_KEY }}
           s3_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           s3_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          cache: s3://iog-catalyst-cache?region=eu-central-1
+          cache: ${{ env.S3_CACHE }}
   build-packages:
     if: always()
     needs:
@@ -81,7 +87,7 @@ jobs:
           nix_key: ${{ secrets.NIX_SIGNING_KEY }}
           s3_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           s3_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          cache: s3://iog-catalyst-cache?region=eu-central-1
+          cache: ${{ env.S3_CACHE }}
   build-devshells:
     if: always()
     needs:
@@ -102,4 +108,4 @@ jobs:
           nix_key: ${{ secrets.NIX_SIGNING_KEY }}
           s3_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           s3_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          cache: s3://iog-catalyst-cache?region=eu-central-1
+          cache: ${{ env.S3_CACHE }}

Cargo.lock

@@ -75,7 +75,7 @@
   nixConfig = {
     extra-substituters = [
       "https://cache.iog.io"
-      "https://iog-catalyst-cache.s3.eu-central-1.amazonaws.com"
+      "https://iog-catalyst-nix-cache.s3.eu-central-1.amazonaws.com"
     ];
     extra-trusted-public-keys = [
       "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ="

flake.nix

@@ -6,12 +6,14 @@
   inherit (inputs.cells.lib) constants;
   l = nixpkgs.lib // builtins;
 
-  mkOCI = name: namespace:
+  mkOCI = name: let
+    operable = cell.operables.${name};
+  in
     std.lib.ops.mkStandardOCI {
-      name = "${constants.registry}/${name}-${namespace}";
-      operable = cell.operables."${name}-${namespace}";
+      inherit operable;
+      name = "${constants.registry}/${name}";
       debug = true;
     };
-in
-  {}
-  // constants.mapToNamespaces {prefix = "jormungandr";} (mkOCI "jormungandr")
+in {
+  jormungandr = mkOCI "jormungandr";
+}

nix/jormungandr/containers.nix

@@ -7,70 +7,68 @@
   inherit (inputs.cells.lib) constants;
   l = nixpkgs.lib // builtins;
 
-  mkOperable = package: namespace: let
-    artifacts' = artifacts."artifacts-${namespace}";
-  in
-    std.lib.ops.mkOperable {
-      inherit package;
-      # TODO: Remove all the bitte stuff
-      runtimeScript = ''
-        ulimit -n 1024
+  package = cell.packages.jormungandr;
+in {
+  jormungandr = std.lib.ops.mkOperable {
+    inherit package;
+    # TODO: Remove all the bitte stuff
+    # TODO: update artifact path to correct mount path
+    runtimeScript = ''
+      ulimit -n 1024
 
-        nodeConfig="$NOMAD_TASK_DIR/node-config.json"
-        runConfig="$NOMAD_TASK_DIR/running.json"
-        runYaml="$NOMAD_TASK_DIR/running.yaml"
+      nodeConfig="$NOMAD_TASK_DIR/node-config.json"
+      runConfig="$NOMAD_TASK_DIR/running.json"
+      runYaml="$NOMAD_TASK_DIR/running.yaml"
 
-        chmod u+rwx -R "$NOMAD_TASK_DIR" || true
+      chmod u+rwx -R "$NOMAD_TASK_DIR" || true
 
-        function convert () {
-          chmod u+rwx -R "$NOMAD_TASK_DIR" || true
-          cp "$nodeConfig" "$runConfig"
-          remarshal --if json --of yaml "$runConfig" > "$runYaml"
-        }
+      function convert () {
+        chmod u+rwx -R "$NOMAD_TASK_DIR" || true
+        cp "$nodeConfig" "$runConfig"
+        remarshal --if json --of yaml "$runConfig" > "$runYaml"
+      }
 
-        if [ "$RESET" = "true" ]; then
-          echo "RESET is given, will start from scratch..."
-          rm -rf "$STORAGE_DIR"
-        elif [ -d "$STORAGE_DIR" ]; then
-          echo "$STORAGE_DIR found, not restoring from backup..."
-        else
-          echo "$STORAGE_DIR not found, restoring backup..."
+      if [ "$RESET" = "true" ]; then
+        echo "RESET is given, will start from scratch..."
+        rm -rf "$STORAGE_DIR"
+      elif [ -d "$STORAGE_DIR" ]; then
+        echo "$STORAGE_DIR found, not restoring from backup..."
+      else
+        echo "$STORAGE_DIR not found, restoring backup..."
 
-          restic restore latest \
-            --verbose=5 \
-            --no-lock \
-            --tag "$NAMESPACE" \
-            --target / \
-          || echo "couldn't restore backup, continue startup procedure..."
-        fi
+        restic restore latest \
+          --verbose=5 \
+          --no-lock \
+          --tag "$NAMESPACE" \
+          --target / \
+        || echo "couldn't restore backup, continue startup procedure..."
+      fi
 
-        set +x
-        echo "waiting for $REQUIRED_PEER_COUNT peers"
-        until [ "$(jq -e -r '.p2p.trusted_peers | length' < "$nodeConfig" || echo 0)" -ge "$REQUIRED_PEER_COUNT" ]; do
-          sleep 1
-        done
-        set -x
+      set +x
+      echo "waiting for $REQUIRED_PEER_COUNT peers"
+      until [ "$(jq -e -r '.p2p.trusted_peers | length' < "$nodeConfig" || echo 0)" -ge "$REQUIRED_PEER_COUNT" ]; do
+        sleep 1
+      done
+      set -x
 
-        convert
+      convert
 
-        if [ -n "$PRIVATE" ]; then
-          echo "Running with node with secrets..."
-          exec jormungandr \
-            --storage "$STORAGE_DIR" \
-            --config "$NOMAD_TASK_DIR/running.yaml" \
-            --genesis-block "${artifacts'}/block0.bin" \
-            --secret "$NOMAD_SECRETS_DIR/bft-secret.yaml" \
-            "$@" || true
-        else
-          echo "Running with follower node..."
-          exec jormungandr \
-            --storage "$STORAGE_DIR" \
-            --config "$NOMAD_TASK_DIR/running.yaml" \
-            --genesis-block "${artifacts'}/block0.bin" \
-            "$@" || true
-        fi
-      '';
-    };
-in
-  {}
-  // constants.mapToNamespaces {prefix = "jormungandr";} (mkOperable cell.packages.jormungandr)
+      if [ -n "$PRIVATE" ]; then
+        echo "Running with node with secrets..."
+        exec ${l.getExe package} \
+          --storage "$STORAGE_DIR" \
+          --config "$NOMAD_TASK_DIR/running.yaml" \
+          --genesis-block "artifacts/block0.bin" \
+          --secret "$NOMAD_SECRETS_DIR/bft-secret.yaml" \
+          "$@" || true
+      else
+        echo "Running with follower node..."
+        exec ${l.getExe package} \
+          --storage "$STORAGE_DIR" \
+          --config "$NOMAD_TASK_DIR/running.yaml" \
+          --genesis-block "artifacts/block0.bin" \
+          "$@" || true
+      fi
+    '';
+  };
+}

nix/jormungandr/operables.nix

@@ -22,7 +22,7 @@
 in rec {
   inherit envs events;
   namespaces = l.flatten (l.map (event: mkNamespaces event) events);
-  registry = "registry.ci.iog.io";
+  registry = "432820653916.dkr.ecr.eu-central-1.amazonaws.com";
 
   mapToNamespaces = {
     prefix ? "",

nix/lib/constants.nix

@@ -6,12 +6,14 @@
   inherit (inputs.cells.lib) constants;
   l = nixpkgs.lib // builtins;
 
-  mkOCI = name: namespace:
+  mkOCI = name: let
+    operable = cell.operables.${name};
+  in
     std.lib.ops.mkStandardOCI {
-      name = "${constants.registry}/${name}-${namespace}";
-      operable = cell.operables."${name}-${namespace}";
+      inherit operable;
+      name = "${constants.registry}/${name}";
       debug = true;
     };
-in
-  {}
-  // constants.mapToNamespaces {prefix = "vit-servicing-station-server";} (mkOCI "vit-servicing-station-server")
+in {
+  vit-servicing-station-server = mkOCI "vit-servicing-station-server";
+}

nix/vit-servicing-station/containers.nix

@@ -7,21 +7,24 @@
   inherit (inputs.cells.lib) constants;
   l = nixpkgs.lib // builtins;
 
-  mkVitOperable = package: namespace: let
-    artifacts' = artifacts."artifacts-${namespace}";
+  mkSimpleOperable = {
+    name,
+    runtimeInputs ? [],
+    args ? [],
+  }: let
+    package = cell.packages.${name};
   in
     std.lib.ops.mkOperable {
-      inherit package;
-      runtimeInputs = [
-        artifacts'
-      ];
+      inherit package runtimeInputs;
       runtimeScript = std.lib.ops.mkOperableScript {
-        inherit package;
-        args = {
-          "--in-settings-file" = "/local/station-config.json";
-        };
+        inherit args package;
       };
     };
-in
-  {}
-  // constants.mapToNamespaces {prefix = "vit-servicing-station-server";} (mkVitOperable cell.packages.vit-servicing-station-server)
+in {
+  vit-servicing-station-server = mkSimpleOperable {
+    name = "vit-servicing-station-server";
+    args = {
+      "--in-settings-file" = "/local/station-config.json";
+    };
+  };
+}

nix/vit-servicing-station/operables.nix

@@ -5,32 +5,47 @@ use chain_impl_mockchain::{
     header::BlockDate,
     transaction::{Input, Payload, Transaction},
 };
-use wallet::{EitherAccount, Settings};
+use std::{collections::HashMap, str::FromStr};
+use wallet::{AccountId, EitherAccount, Settings};
 
 use crate::Error;
 
 pub struct TxBuilder<P: Payload> {
     builder: wallet::TransactionBuilder<P>,
+    inputs: HashMap<AccountId, (Input, SpendingCounter)>,
 }
 
 impl<P: Payload> TxBuilder<P> {
     pub fn new(settings: Settings, valid_until: BlockDate, payload: P) -> Self {
         let builder = wallet::TransactionBuilder::new(settings, payload, valid_until);
-        Self { builder }
+        Self {
+            builder,
+            inputs: HashMap::new(),
+        }
     }
 
     pub fn build_tx(
         mut self,
-        account_bytes: &[u8],
+        account_id_hex: String,
         spending_counter: SpendingCounter,
     ) -> Result<Self, Error> {
+        let account_id = AccountId::from_str(&account_id_hex)
+            .map_err(|e| Error::wallet_transaction().with(e))?;
+
+        // It is needed to provide a 1 extra input as we are generating it later, but should take into account at this place.
+        let value = self.builder.estimate_fee_with(1, 0);
+        let input = Input::from_account_public_key(account_id.into(), value);
+        self.inputs.insert(account_id, (input, spending_counter));
+        Ok(self)
+    }
+
+    pub fn sign_tx(mut self, account_bytes: &[u8]) -> Result<Self, Error> {
         let account = EitherAccount::new_from_key(
             SecretKey::from_binary(account_bytes)
                 .map_err(|e| Error::wallet_transaction().with(e))?,
         );
-        // It is needed to provide a 1 extra input as we are generating it later, but should take into account at this place.
-        let value = self.builder.estimate_fee_with(1, 0);
-        let input = Input::from_account_public_key(account.account_id().into(), value);
+
+        let (input, spending_counter) = self.inputs.get(&account.account_id()).ok_or_else(|| Error::invalid_input("Cannot find corresponded input to the provided account, make sure that you have correctly execute build_tx function first"))?.clone();
         let witness_builder = account.witness_builder(spending_counter);
         self.builder.add_input(input, witness_builder);
         Ok(self)