050: [USER] Memory Profile Ctrl #53
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: Apache-2.0 | |
| name: "050: [USER] Memory Profile Ctrl" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| nlg-accounts: #'-R -c 32 -a 100000000 -T 1000 -n 100000 -S hot -p 50 -tt 1m | |
| required: true | |
| default: "1000" | |
| description: "Number of Accounts and NFT tokens" | |
| type: choice | |
| options: | |
| - "100" | |
| - "1000" | |
| nlg-time: #'-R -c 32 -a 100000000 -T 1000 -n 100000 -S hot -p 50 -tt 1m | |
| required: true | |
| default: "3" | |
| description: "Test execution time, in seconds, per each test (of 6 tests)" | |
| type: choice | |
| options: | |
| - "60" | |
| - "120" | |
| - "300" | |
| - "600" | |
| - "1200" | |
| - "3000" | |
| - "3600" | |
| - "7200" | |
| - "19800" | |
| memory-interval: | |
| required: true | |
| default: "200" | |
| type: string | |
| description: 'Interval between memory checks, secs' | |
| add-app-props: | |
| required: false | |
| default: "" | |
| type: string | |
| description: 'Add props to application.properties, e.g. "blockStream.streamMode=RECORDS". Newline is "\n"' | |
| add-settings: | |
| required: false | |
| default: "" | |
| type: string | |
| description: 'Add props to settings.txt, e.g. "state.saveStatePeriod, 300". Newline is "\n"' | |
| nlg-test: | |
| required: true | |
| default: "CryptoTransferLoadTest" | |
| type: string | |
| description: 'Test name' | |
| nlg-args: | |
| required: true | |
| default: "-c 1" | |
| type: string | |
| description: 'test arguments' | |
| nlg-tps: | |
| required: true | |
| default: "100" | |
| type: string | |
| description: 'Transactions Per Second' | |
| versions: | |
| required: true | |
| default: "export CONSENSUS_NODE_VERSION=v0.72.0-rc.4; export BLOCK_NODE_VERSION=v0.27.0; ONE_SHOT_WITH_BLOCK_NODE=false" | |
| type: string | |
| description: 'Versions of containers' | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| contents: write | |
| id-token: write | |
| actions: write | |
| statuses: write | |
| env: | |
| GS_ROOT_DIR: gs://performance-engineering-reports/ephemeral/test_runs | |
| GS_ROOT_HTTPS: https://perf.analytics.eng.hashgraph.io/ephemeral/test_runs | |
| jobs: | |
| run-single-node-tests: | |
| name: Single node tests | |
| runs-on: hl-cn-mem-profiler-lin | |
| timeout-minutes: 1440 | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 | |
| with: | |
| egress-policy: audit | |
| - name: Authenticate to Google Cloud | |
| id: google-auth | |
| uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 | |
| with: | |
| workload_identity_provider: "projects/716789254648/locations/global/workloadIdentityPools/perf-eng-reports-pool/providers/gh-provider" | |
| service_account: "gh-perf-report-writer@perf-engineering-reports.iam.gserviceaccount.com" | |
| - name: Setup Google Cloud SDK | |
| uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2 | |
| - name: Checkout BlockNode scripts | |
| if: false | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: hiero-ledger/hiero-block-node | |
| token: ${{ secrets.GH_ACCESS_TOKEN }} | |
| ref: master | |
| path: "./hiero-block-node" | |
| - name: Checkout BlockNode scripts | |
| if: false | |
| run: | | |
| git clone https://${{ secrets.GH_ACCESS_TOKEN }}@github.com/hiero-ledger/hiero-block-node.git hiero-block-node | |
| - name: Setup Helm | |
| shell: bash | |
| run: | | |
| sudo ln -s /home/runner/_work/_tool/helm/4.1.1/x64/linux-amd64/helm /usr/bin/helm | |
| - name: Install Task | |
| uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 | |
| with: | |
| version: 3.39.2 | |
| - name: Install KubeCtl | |
| uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 | |
| with: | |
| version: v1.32.2 | |
| # Set up kind; needed for configuring the solo environment | |
| - name: Setup Kind | |
| uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0 | |
| with: | |
| install_only: true | |
| node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30 | |
| version: v0.29.0 | |
| kubectl_version: v1.32.2 | |
| verbosity: 3 | |
| wait: 120s | |
| - name: Setup Node | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: 20.18.0 | |
| - name: Install Solo | |
| if: false | |
| run: | | |
| npm install -g "@hashgraph/solo@${{ inputs.solo-version || 'latest' }}" | |
| # verify the installation | |
| solo --version | |
| - name: Build Solo | |
| run: | | |
| set +x | |
| set +e | |
| curdir=`pwd` | |
| echo "${curdir}" | |
| ls | |
| curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash | |
| export NVM_DIR="${HOME}/.nvm" | |
| [ -s "${NVM_DIR}/nvm.sh" ] && \. "${NVM_DIR}/nvm.sh" # This loads nvm | |
| nvm install v24.14.0 | |
| nvm use v24.14.0 | |
| export PATH="${HOME}"/:"${PATH}" | |
| #solo script | |
| echo '#!/usr/bin/sh -f' > "${HOME}"/solo | |
| echo "node \"${curdir}\"/solo/dist/solo.js \"\$@\"" >> "${HOME}"/solo | |
| chmod a+rx "${HOME}"/solo | |
| sudo mv "${HOME}"/solo /usr/bin/solo | |
| git clone https://github.com/hiero-ledger/solo.git solo | |
| cd solo | |
| git checkout 942ec2d0f1738edead5bbc89ece6cd1516c3bae6 | |
| task build:compile | |
| #npm run build | |
| which solo | |
| solo --version | |
| cd .. | |
| - name: Setup Java | |
| uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 | |
| with: | |
| distribution: temurin | |
| java-version: 25 | |
| - name: Install tools | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y docker.io net-tools iputils-ping node-typescript snapd | |
| #YQ | |
| sudo add-apt-repository ppa:rmescandon/yq | |
| sudo apt update | |
| sudo apt install -y yq zip unzip | |
| sudo apt install -y x11-common x11-session-utils x11-utils x11-xserver-utils | |
| - name: Install Memory Analyzer | |
| run: | | |
| gcloud --no-user-output-enabled storage cp --recursive gs://performance-engineering-reports/permanent/MemoryAnalyzer-1.16.1.20250109-linux.gtk.x86_64.zip ${HOME}/MemoryAnalyzer-1.16.1.20250109-linux.gtk.x86_64.zip | |
| unzip -q -d ${HOME} ${HOME}/MemoryAnalyzer-1.16.1.20250109-linux.gtk.x86_64.zip | |
| ls ${HOME}/mat | |
| - name: Version update | |
| run: | | |
| /usr/bin/echo -e "${{ inputs.versions }}" > "${{ github.workspace }}"/solo/testenv.sh | |
| - name: Deploy K8S pod | |
| run: | | |
| ls -l | |
| #cd hiero-block-node/tools-and-tests/scripts/solo-e2e-test | |
| #cat .env | |
| #task up | |
| cd "${{ github.workspace }}"/solo | |
| . "${{ github.workspace }}"/solo/testenv.sh | |
| bash ./test/e2e/dual-cluster/setup-dual-e2e.sh | |
| wait_in_minutes=$(expr ${{ inputs.nlg-time }} / 60) | |
| echo SOLO_TEST_CLUSTER=solo-e2e >> .env | |
| echo ONE_SHOT_WITH_BLOCK_NODE=false >> .env | |
| echo MIRROR_NODE_PINGER_TPS=10 >> .env | |
| echo ONE_SHOT_METRICS_TEST_DURATION_IN_MINUTES=${wait_in_minutes} >> .env | |
| cat .env | |
| - name: Start Memory capturing | |
| run: | | |
| cd "${{ github.workspace }}"/solo | |
| cat <<'EOF' > "${{ github.workspace }}"/solo/heapInfoByPod.sh | |
| set +e | |
| set +x | |
| cd "${{ github.workspace }}"/solo | |
| count=0 | |
| while [ true ] | |
| do | |
| kubectl get namespaces | |
| for i in `kubectl get namespaces | grep 'one-shot' | awk '{print $1}' | grep -v 'solo-setup'` | |
| do | |
| kubectl -n ${i} exec network-node1-0 -- \ | |
| bash -c "ps -aef; jpid=\$(ps -aef | grep -w java | grep -v grep | awk '{print \$2}'); \ | |
| if [ \"\$jpid\" != \"\" ]; then echo \"pod $i:\"; \ | |
| jcmd \$jpid GC.heap_info; echo \"\"; \ | |
| jmap -dump:live,format=b,file=/tmp/test.hprof \$jpid; fi"; \ | |
| kubectl -n ${i} cp network-node1-0:/opt/hgcapp/services-hedera/HapiApp2.0/output/recording.jfr "${{ github.workspace }}"/solo/recording.jfr | |
| kubectl -n ${i} cp network-node1-0:/tmp/test.hprof "${{ github.workspace }}"/solo/$i_$count.hprof | |
| ls -l "${{ github.workspace }}"/solo/$i_$count.hprof | |
| done | |
| count=`expr $count + 1` | |
| sleep ${{ inputs.memory-interval }} | |
| done | |
| EOF | |
| nohup bash -c "sleep 300; sleep ${{ inputs.memory-interval }}; sh "${{ github.workspace }}"/solo/heapInfoByPod.sh" > "${{ github.workspace }}"/solo/heapInfoByPod.log 2>&1 & | |
| - name: Run NLG test | |
| run: | | |
| set +e | |
| set +x | |
| #cd hiero-block-node/tools-and-tests/scripts/solo-e2e-test | |
| #task load:up NLG_TEST_TYPE="${{ inputs.nlg-test }}" NLG_ARGS="${{ inputs.nlg-args }} -a ${{ inputs.nlg-accounts }} -tt ${{ inputs.nlg-time }}s" NLG_MAX_TPS="${{ inputs.nlg-tps }}" | |
| . "${{ github.workspace }}"/solo/testenv.sh | |
| cd "${{ github.workspace }}"/solo | |
| task test-e2e-performance | |
| - name: Stop NLG test | |
| if: false | |
| run: | | |
| cd hiero-block-node/tools-and-tests/scripts/solo-e2e-test | |
| task load:down | |
| - name: Report Kubernetes resources | |
| if: false | |
| run: | | |
| kubectl -n solo-network describe nodes | |
| - name: Create report dir | |
| shell: bash | |
| run: | | |
| mkdir "${{ github.workspace }}"/report | |
| - name: Java heap info | |
| if: ${{ !cancelled() }} | |
| run: | | |
| set +e | |
| set +x | |
| cd "${{ github.workspace }}"/solo | |
| cat heapInfoByPod.log | |
| - name: Analyze Java heaps | |
| if: ${{ !cancelled() }} | |
| run: | | |
| set +e | |
| set +x | |
| cd "${{ github.workspace }}"/solo | |
| #process hprof-s | |
| ls -1 *.hprof | |
| for i in `ls -1 *.hprof` | |
| do | |
| mkdir $i.dir | |
| cd $i.dir | |
| mv ../$i ./$i | |
| bash ${HOME}/mat/ParseHeapDump.sh ./$i org.eclipse.mat.api:overview | |
| mkdir -p "${{ github.workspace }}"/report/$i.report | |
| unzip -d "${{ github.workspace }}"/report/$i.report $(find . -name '*System_Overview.zip') | |
| cd "${{ github.workspace }}"/solo | |
| done | |
| - name: Publish logs | |
| if: ${{ !cancelled() }} | |
| run: | | |
| set +e | |
| set +x | |
| ls -l "${{ github.workspace }}"/solo | |
| cp -r $HOME/.solo "${{ github.workspace }}"/report/ | |
| cp "${{ github.workspace }}"/solo/*.log "${{ github.workspace }}"/report/ | |
| cp "${{ github.workspace }}"/solo/*.jfr "${{ github.workspace }}"/report/ | |
| gcloud --no-user-output-enabled storage cp --recursive "${{ github.workspace }}"/report "${{ env.GS_ROOT_DIR }}/MemoryProfile_${{ github.run_number }}"/report | |
| echo Done: solo deploy logs in "${{ env.GS_ROOT_HTTPS }}/MemoryProfile_${{ github.run_number }}"/report |