050-user-memory-profile-ctrl.yaml

# SPDX-License-Identifier: Apache-2.0
name: "050: [USER] Memory Profile Ctrl"
on:
  workflow_dispatch:
    inputs:
      nlg-accounts: #'-R -c 32 -a 100000000 -T 1000 -n 100000 -S hot -p 50 -tt 1m
        required: true
        default: "1000"
        description: "Number of Accounts and NFT tokens"
        type: choice
        options:
          - "100"
          - "1000"
      nlg-time: #'-R -c 32 -a 100000000 -T 1000 -n 100000 -S hot -p 50 -tt 1m
        required: true
        default: "3"
        description: "Test execution time, in seconds, per each test (of 6 tests)"
        type: choice
        options:
          - "60"
          - "120"
          - "300"
          - "600"
          - "1200"
          - "3000"
          - "3600"
          - "7200"
          - "19800"
      memory-interval:
        required: true
        default: "200"
        type: string
        description: 'Interval between memory checks, secs'

      add-app-props:
        required: false
        default: ""
        type: string
        description: 'Add props to application.properties, e.g. "blockStream.streamMode=RECORDS". Newline is "\n"'
      add-settings:
        required: false
        default: ""
        type: string
        description: 'Add props to settings.txt, e.g. "state.saveStatePeriod, 300". Newline is "\n"'
      nlg-test:
        required: true
        default: "CryptoTransferLoadTest"
        type: string
        description: 'Test name'
      nlg-args:
        required: true
        default: "-c 1"
        type: string
        description: 'test arguments'
      nlg-tps:
        required: true
        default: "100"
        type: string
        description: 'Transactions Per Second'
      versions:
        required: true
        default: "export CONSENSUS_NODE_VERSION=v0.72.0-rc.4; export BLOCK_NODE_VERSION=v0.27.0; ONE_SHOT_WITH_BLOCK_NODE=false"
        type: string
        description: 'Versions of containers'
defaults:
  run:
    shell: bash

permissions:
  contents: write
  id-token: write
  actions: write
  statuses: write

env:
  GS_ROOT_DIR: gs://performance-engineering-reports/ephemeral/test_runs
  GS_ROOT_HTTPS: https://perf.analytics.eng.hashgraph.io/ephemeral/test_runs

jobs:
  run-single-node-tests:
    name: Single node tests
    runs-on: hl-cn-mem-profiler-lin
    timeout-minutes: 1440
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
        with:
          egress-policy: audit

      - name: Authenticate to Google Cloud
        id: google-auth
        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
        with:
          workload_identity_provider: "projects/716789254648/locations/global/workloadIdentityPools/perf-eng-reports-pool/providers/gh-provider"
          service_account: "gh-perf-report-writer@perf-engineering-reports.iam.gserviceaccount.com"

      - name: Setup Google Cloud SDK
        uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2

      - name: Checkout BlockNode scripts
        if: false
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: hiero-ledger/hiero-block-node
          token: ${{ secrets.GH_ACCESS_TOKEN }}
          ref: master
          path: "./hiero-block-node"

      - name: Checkout BlockNode scripts
        if: false
        run: |
          git clone https://${{ secrets.GH_ACCESS_TOKEN }}@github.com/hiero-ledger/hiero-block-node.git hiero-block-node

      - name: Setup Helm
        shell: bash
        run: |
          sudo ln -s /home/runner/_work/_tool/helm/4.1.1/x64/linux-amd64/helm /usr/bin/helm

      - name: Install Task
        uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0
        with:
          version: 3.39.2

      - name: Install KubeCtl
        uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0
        with:
          version: v1.32.2

      # Set up kind; needed for configuring the solo environment
      - name: Setup Kind
        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
        with:
          install_only: true
          node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
          version: v0.29.0
          kubectl_version: v1.32.2
          verbosity: 3
          wait: 120s

      - name: Setup Node
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: 20.18.0

      - name: Install Solo
        if: false
        run: |
          npm install -g "@hashgraph/solo@${{ inputs.solo-version || 'latest' }}"

          # verify the installation
          solo --version

      - name: Build Solo
        run: |
          set +x
          set +e
          curdir=`pwd`
          echo "${curdir}"
          ls
          curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
          export NVM_DIR="${HOME}/.nvm"
          [ -s "${NVM_DIR}/nvm.sh" ] && \. "${NVM_DIR}/nvm.sh"  # This loads nvm
          nvm install v24.14.0

          nvm use v24.14.0
          export PATH="${HOME}"/:"${PATH}"

          #solo script
          echo '#!/usr/bin/sh -f' > "${HOME}"/solo
          echo "node \"${curdir}\"/solo/dist/solo.js \"\$@\"" >> "${HOME}"/solo
          chmod a+rx "${HOME}"/solo
          sudo mv "${HOME}"/solo /usr/bin/solo

          git clone https://github.com/hiero-ledger/solo.git solo
          cd solo
          git checkout 942ec2d0f1738edead5bbc89ece6cd1516c3bae6
          task build:compile
          #npm run build

          which solo
          solo --version

          cd ..

      - name: Setup Java
        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
        with:
          distribution: temurin
          java-version: 25

      - name: Install tools
        run: |
          sudo apt-get update
          sudo apt-get install -y docker.io net-tools iputils-ping node-typescript snapd

          #YQ
          sudo add-apt-repository ppa:rmescandon/yq
          sudo apt update
          sudo apt install -y yq zip unzip
          sudo apt install -y x11-common x11-session-utils x11-utils x11-xserver-utils

      - name: Install Memory Analyzer
        run: |
          gcloud --no-user-output-enabled storage cp --recursive gs://performance-engineering-reports/permanent/MemoryAnalyzer-1.16.1.20250109-linux.gtk.x86_64.zip ${HOME}/MemoryAnalyzer-1.16.1.20250109-linux.gtk.x86_64.zip
          unzip -q -d ${HOME} ${HOME}/MemoryAnalyzer-1.16.1.20250109-linux.gtk.x86_64.zip
          ls ${HOME}/mat

      - name: Version update
        run: |
          /usr/bin/echo -e "${{ inputs.versions }}" > "${{ github.workspace }}"/solo/testenv.sh

      - name: Deploy K8S pod
        run: |
          ls -l
          #cd hiero-block-node/tools-and-tests/scripts/solo-e2e-test
          #cat .env
          #task up
          cd "${{ github.workspace }}"/solo
          . "${{ github.workspace }}"/solo/testenv.sh
          bash ./test/e2e/dual-cluster/setup-dual-e2e.sh
          
          wait_in_minutes=$(expr ${{ inputs.nlg-time }} / 60)
          echo SOLO_TEST_CLUSTER=solo-e2e >> .env
          echo ONE_SHOT_WITH_BLOCK_NODE=false >> .env
          echo MIRROR_NODE_PINGER_TPS=10 >> .env
          echo ONE_SHOT_METRICS_TEST_DURATION_IN_MINUTES=${wait_in_minutes} >> .env
          cat .env

      - name: Start Memory capturing
        run: |
          cd "${{ github.workspace }}"/solo
          cat <<'EOF' > "${{ github.workspace }}"/solo/heapInfoByPod.sh
          set +e
          set +x
          cd "${{ github.workspace }}"/solo
          count=0
          while [ true ]
          do
            kubectl get namespaces
            for i in `kubectl get namespaces | grep 'one-shot' | awk '{print $1}' | grep -v 'solo-setup'`
            do
              kubectl -n ${i} exec network-node1-0  -- \
              bash -c "ps -aef; jpid=\$(ps -aef | grep -w java | grep -v grep | awk '{print \$2}'); \
              if [ \"\$jpid\" != \"\" ]; then echo \"pod $i:\"; \
                jcmd \$jpid GC.heap_info; echo \"\"; \
                jmap -dump:live,format=b,file=/tmp/test.hprof \$jpid; fi"; \
              kubectl -n ${i} cp network-node1-0:/opt/hgcapp/services-hedera/HapiApp2.0/output/recording.jfr "${{ github.workspace }}"/solo/recording.jfr
              kubectl -n ${i} cp network-node1-0:/tmp/test.hprof "${{ github.workspace }}"/solo/$i_$count.hprof
              ls -l "${{ github.workspace }}"/solo/$i_$count.hprof
            done
            count=`expr $count + 1`
            sleep ${{ inputs.memory-interval }}
          done
          EOF
          nohup bash -c "sleep 300; sleep ${{ inputs.memory-interval }}; sh "${{ github.workspace }}"/solo/heapInfoByPod.sh" > "${{ github.workspace }}"/solo/heapInfoByPod.log 2>&1 &

      - name: Run NLG test
        run: |
          set +e
          set +x
          #cd hiero-block-node/tools-and-tests/scripts/solo-e2e-test
          #task load:up NLG_TEST_TYPE="${{ inputs.nlg-test }}" NLG_ARGS="${{ inputs.nlg-args }} -a ${{ inputs.nlg-accounts }} -tt ${{ inputs.nlg-time }}s" NLG_MAX_TPS="${{ inputs.nlg-tps }}"
          . "${{ github.workspace }}"/solo/testenv.sh
          cd "${{ github.workspace }}"/solo
          task test-e2e-performance

      - name: Stop NLG test
        if: false
        run: |
          cd hiero-block-node/tools-and-tests/scripts/solo-e2e-test
          task load:down

      - name: Report Kubernetes resources
        if: false
        run: |
          kubectl -n solo-network describe nodes

      - name: Create report dir
        shell: bash
        run: |
          mkdir "${{ github.workspace }}"/report

      - name: Java heap info
        if: ${{ !cancelled() }}
        run: |
          set +e
          set +x
          cd "${{ github.workspace }}"/solo
          cat heapInfoByPod.log

      - name: Analyze Java heaps
        if: ${{ !cancelled() }}
        run: |
          set +e
          set +x
          cd "${{ github.workspace }}"/solo
          #process hprof-s
          ls -1 *.hprof
          for i in `ls -1 *.hprof`
          do
              mkdir $i.dir
              cd $i.dir
              mv ../$i ./$i
              bash ${HOME}/mat/ParseHeapDump.sh ./$i org.eclipse.mat.api:overview
              mkdir -p "${{ github.workspace }}"/report/$i.report
              unzip -d "${{ github.workspace }}"/report/$i.report $(find . -name '*System_Overview.zip')
              cd "${{ github.workspace }}"/solo
          done

      - name: Publish logs
        if: ${{ !cancelled() }}
        run: |
          set +e
          set +x
          ls -l "${{ github.workspace }}"/solo
          cp -r $HOME/.solo "${{ github.workspace }}"/report/
          cp "${{ github.workspace }}"/solo/*.log  "${{ github.workspace }}"/report/
          cp "${{ github.workspace }}"/solo/*.jfr  "${{ github.workspace }}"/report/
          gcloud --no-user-output-enabled storage cp --recursive "${{ github.workspace }}"/report "${{ env.GS_ROOT_DIR }}/MemoryProfile_${{ github.run_number }}"/report
          echo Done: solo deploy logs  in "${{ env.GS_ROOT_HTTPS }}/MemoryProfile_${{ github.run_number }}"/report