Network fault tolerance #2658
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code_Coverage | |
| permissions: {} | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| concurrency: | |
| # New commit on branch cancels a running workflow on the same branch | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| BuildAndTest-Coverage: | |
| strategy: | |
| matrix: | |
| go: ["1.25"] | |
| job_name: ["linux-fuse3", "linux-fuse2"] | |
| include: | |
| - job_name: linux-fuse3 | |
| os: ubuntu-latest | |
| containerName: "test-cnt-ubn-fuse3" | |
| fuselib: libfuse3-dev | |
| - job_name: linux-fuse2 | |
| os: ubuntu-latest | |
| containerName: "test-cnt-ubn-fuse2" | |
| fuselib: libfuse-dev | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| MOUNT_DIR: "/home/runner/blob_mnt" | |
| TEMP_DIR: "/home/runner/cloudfuse_tmp" | |
| BLOCK_TEMP_DIR: "/home/runner/cloudfuse_block_tmp" | |
| WORK_DIR: "." | |
| cloudfuse_ADLS_CFG: "./cloudfuse.adls.yaml" | |
| cloudfuse_CFG: "./cloudfuse.yaml" | |
| cloudfuse_STREAM_CFG: "./cloudfuse_stream.yaml" | |
| cloudfuse_BLOCK_CFG: "./cloudfuse_block.yaml" | |
| zig: 0.15.1 | |
| GH_API_TOKEN: ${{ secrets.GH_API_TOKEN }} | |
| # Skip for dependabot updates | |
| if: ${{ github.triggering_actor != 'dependabot[bot]' }} | |
| timeout-minutes: 60 | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Install Go | |
| uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 | |
| with: | |
| go-version: ${{ matrix.go }} | |
| check-latest: true | |
| - run: go version | |
| - name: Install libfuse | |
| env: | |
| FUSELIB: ${{ matrix.fuselib }} | |
| run: |- | |
| sudo apt-get install fuse3 ${FUSELIB} -y | |
| - name: Cache Zig | |
| if: runner.os == 'Linux' | |
| uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 | |
| with: | |
| path: ~/.local/bin | |
| key: zig-${{ env.zig }}-${{ runner.arch }}-${{ runner.os }} | |
| - name: Install Zig | |
| env: | |
| RUNNER_ARCH: ${{ runner.arch }} | |
| ENV_ZIG: ${{ env.zig }} | |
| run: | | |
| if [[ "${RUNNER_ARCH}" == "X64" ]]; then | |
| ZIG_ARCH="x86_64" | |
| elif [[ "${RUNNER_ARCH}" == "ARM64" ]]; then | |
| ZIG_ARCH="aarch64" | |
| else | |
| echo "Unsupported architecture: ${RUNNER_ARCH}" | |
| exit 1 | |
| fi | |
| ZIG_DIR="zig-${ZIG_ARCH}-linux-${ENV_ZIG}" | |
| ZIG_TARBALL="${ZIG_DIR}.tar.xz" | |
| ZIG_DIR_PATH="$HOME/.local/bin/${ZIG_DIR}" | |
| # If cached Zig already exists, add it to PATH and skip download | |
| if [ -d "${ZIG_DIR_PATH}" ]; then | |
| echo "Found cached Zig at ${ZIG_DIR_PATH}" | |
| echo "${ZIG_DIR_PATH}" >> $GITHUB_PATH | |
| exit 0 | |
| fi | |
| # Otherwise download and extract Zig, then add to PATH | |
| curl -L https://ziglang.org/download/${ENV_ZIG}/${ZIG_TARBALL} -o zig.tar.xz | |
| mkdir -p $HOME/.local/bin | |
| tar -xf zig.tar.xz -C $HOME/.local/bin | |
| echo "${ZIG_DIR_PATH}" >> $GITHUB_PATH | |
| rm zig.tar.xz | |
| - name: Create Directory Structure | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| run: |- | |
| sudo mkdir -p ${MOUNT_DIR} | |
| sudo chown -R `whoami` ${MOUNT_DIR} | |
| chmod 777 ${MOUNT_DIR} | |
| sudo mkdir -p ${TEMP_DIR} | |
| sudo chown -R `whoami` ${TEMP_DIR} | |
| chmod 777 ${TEMP_DIR} | |
| echo "--- Debug: After Create Directory Structure ---" | |
| echo "MOUNT_DIR is: ${MOUNT_DIR}" | |
| echo "Listing MOUNT_DIR:" | |
| ls -ld ${MOUNT_DIR} || echo "MOUNT_DIR not found after creation step" | |
| echo "TEMP_DIR is: ${TEMP_DIR}" | |
| echo "Listing TEMP_DIR:" | |
| ls -ld ${TEMP_DIR} || echo "TEMP_DIR not found after creation step" | |
| echo "---------------------------------------------" | |
| - name: Build | |
| env: | |
| JOB_NAME: ${{ matrix.job_name }} | |
| run: | | |
| if [ "${JOB_NAME}" == "linux-fuse2" ]; then | |
| ./build.sh fuse2 | |
| else | |
| ./build.sh | |
| fi | |
| - name: Create Azure Configuration File | |
| env: | |
| AZTEST_BLOCK_ACC_NAME: ${{ secrets.AZTEST_BLOCK_ACC_NAME }} | |
| AZTEST_ADLS_ACC_NAME: ${{ secrets.AZTEST_ADLS_ACC_NAME }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| AZTEST_BLOCK_KEY: ${{ secrets.AZTEST_BLOCK_KEY }} | |
| AZTEST_ADLS_KEY: ${{ secrets.AZTEST_ADLS_KEY }} | |
| AZTEST_BLOCK_SAS: ${{ secrets.AZTEST_BLOCK_SAS }} | |
| AZTEST_BLOCK_CONT_SAS_UBN_18: ${{ secrets.AZTEST_BLOCK_CONT_SAS_UBN_18 }} | |
| AZTEST_BLOCK_CONT_SAS_UBN_20: ${{ secrets.AZTEST_BLOCK_CONT_SAS_UBN_20 }} | |
| AZTEST_ADLS_SAS: ${{ secrets.AZTEST_ADLS_SAS }} | |
| AZTEST_APP_ID: ${{ secrets.AZTEST_APP_ID }} | |
| AZTEST_RES_ID: ${{ secrets.AZTEST_RES_ID }} | |
| AZTEST_OBJ_ID: ${{ secrets.AZTEST_OBJ_ID }} | |
| AZTEST_CLIENT: ${{ secrets.AZTEST_CLIENT }} | |
| AZTEST_TENANT: ${{ secrets.AZTEST_TENANT }} | |
| AZTEST_SECRET: ${{ secrets.AZTEST_SECRET }} | |
| run: |- | |
| cnfFile=$HOME/azuretest.json | |
| echo $cnfFile | |
| touch $cnfFile | |
| echo "{" > $cnfFile | |
| echo "\"block-acct\"": "\"${AZTEST_BLOCK_ACC_NAME}\"", >> $cnfFile | |
| echo "\"adls-acct\"": "\"${AZTEST_ADLS_ACC_NAME}\"", >> $cnfFile | |
| echo "\"block-cont\"": "\"${CONTAINER_NAME}\"", >> $cnfFile | |
| echo "\"adls-cont\"": "\"${CONTAINER_NAME}\"", >> $cnfFile | |
| echo "\"block-key\"": "\"${AZTEST_BLOCK_KEY}\"", >> $cnfFile | |
| echo "\"adls-key\"": "\"${AZTEST_ADLS_KEY}\"", >> $cnfFile | |
| echo "\"block-sas\"": "\"${AZTEST_BLOCK_SAS}\"", >> $cnfFile | |
| echo "\"block-cont-sas-ubn-18\"": "\"${AZTEST_BLOCK_CONT_SAS_UBN_18}\"", >> $cnfFile | |
| echo "\"block-cont-sas-ubn-20\"": "\"${AZTEST_BLOCK_CONT_SAS_UBN_20}\"", >> $cnfFile | |
| echo "\"adls-sas\"": "\"${AZTEST_ADLS_SAS}\"", >> $cnfFile | |
| echo "\"msi-appid\"": "\"${AZTEST_APP_ID}\"", >> $cnfFile | |
| echo "\"msi-resid\"": "\"${AZTEST_RES_ID}\"", >> $cnfFile | |
| echo "\"msi-objid\"": "\"${AZTEST_OBJ_ID}\"", >> $cnfFile | |
| echo "\"spn-client\"": "\"${AZTEST_CLIENT}\"", >> $cnfFile | |
| echo "\"spn-tenant\"": "\"${AZTEST_TENANT}\"", >> $cnfFile | |
| echo "\"spn-secret\"": "\"${AZTEST_SECRET}\"", >> $cnfFile | |
| echo "\"skip-msi\"": "true", >> $cnfFile | |
| echo "\"skip-azcli\"": "true", >> $cnfFile | |
| echo "\"proxy-address\"": "\"\"" >> $cnfFile | |
| echo "}" >> $cnfFile | |
| - name: Create S3 Configuration File | |
| env: | |
| S3TEST_BUCKET_NAME: ${{ secrets.S3TEST_BUCKET_NAME }} | |
| S3TEST_ACCESS_KEY: ${{ secrets.S3TEST_ACCESS_KEY }} | |
| S3TEST_SECRET_KEY: ${{ secrets.S3TEST_SECRET_KEY }} | |
| S3TEST_ENDPOINT: ${{ secrets.S3TEST_ENDPOINT }} | |
| S3TEST_REGION: ${{ secrets.S3TEST_REGION }} | |
| run: |- | |
| cnfFile=$HOME/s3test.json | |
| echo $cnfFile | |
| touch $cnfFile | |
| echo "{" > $cnfFile | |
| echo "\"bucket-name\"": "\"${S3TEST_BUCKET_NAME}\"", >> $cnfFile | |
| echo "\"access-key\"": "\"${S3TEST_ACCESS_KEY}\"", >> $cnfFile | |
| echo "\"secret-key\"": "\"${S3TEST_SECRET_KEY}\"", >> $cnfFile | |
| echo "\"endpoint\"": "\"${S3TEST_ENDPOINT}\"", >> $cnfFile | |
| echo "\"region\"": "\"${S3TEST_REGION}\"" >> $cnfFile | |
| echo "}" >> $cnfFile | |
| - name: UT Code Coverage | |
| env: | |
| JOB_NAME: ${{ matrix.job_name }} | |
| run: | | |
| if [ "${JOB_NAME}" == "linux-fuse2" ]; then | |
| go test -v -timeout=1h ./... --tags=unittest,storagetest -covermode count -coverprofile ./cloudfuse_ut.cov | |
| else | |
| go test -v -timeout=1h ./... --tags=unittest,storagetest,fuse3 -covermode count -coverprofile ./cloudfuse_ut.cov | |
| fi | |
| - name: Build coverage binary | |
| env: | |
| JOB_NAME: ${{ matrix.job_name }} | |
| run: | | |
| if [ "${JOB_NAME}" == "linux-fuse2" ]; then | |
| go test -coverpkg="./..." -covermode=count -c -o cloudfuse.test | |
| else | |
| go test -coverpkg="./..." -tags=fuse3 -covermode=count -c -o cloudfuse.test | |
| fi | |
| - name: Create Config File - Block Blob | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_gentest1.cov gen-test-config --config-file=azure_key.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG}" | |
| - name: Block Blob Coverage | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_block.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} | |
| cd - | |
| sudo fusermount -u ${MOUNT_DIR} | |
| UNMOUNT_TIMEOUT=30 | |
| SECONDS_WAITED_UNMOUNT=0 | |
| while [ $SECONDS_WAITED_UNMOUNT -lt $UNMOUNT_TIMEOUT ]; do | |
| if ! mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_UNMOUNT=$((SECONDS_WAITED_UNMOUNT + POLL_INTERVAL)) | |
| done | |
| - name: Block Blob Coverage with profilers | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| cp ${cloudfuse_CFG} /tmp/configBlockProfilerTemp.yaml | |
| echo "dynamic-profile: true" >> /tmp/configBlockProfilerTemp.yaml | |
| echo "cpu-profile: /tmp/cpuBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml | |
| echo "mem-profile: /tmp/memBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml | |
| echo "health_monitor:" >> /tmp/configBlockProfilerTemp.yaml | |
| echo " enable-monitoring: true" >> /tmp/configBlockProfilerTemp.yaml | |
| echo " monitor-disable-list:" >> /tmp/configBlockProfilerTemp.yaml | |
| echo " - cloudfuse_stats" >> /tmp/configBlockProfilerTemp.yaml | |
| cat /tmp/configBlockProfilerTemp.yaml | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_block_profiler.cov mount ${MOUNT_DIR} --config-file=/tmp/configBlockProfilerTemp.yaml --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} | |
| cd - | |
| sudo fusermount -u ${MOUNT_DIR} | |
| UNMOUNT_TIMEOUT=30 | |
| SECONDS_WAITED_UNMOUNT=0 | |
| while [ $SECONDS_WAITED_UNMOUNT -lt $UNMOUNT_TIMEOUT ]; do | |
| if ! mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_UNMOUNT=$((SECONDS_WAITED_UNMOUNT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - Block Blob - Block Cache | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| BLOCK_TEMP_DIR: ${{ env.BLOCK_TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| containerName: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_gentest1.cov gen-test-config --config-file=azure_key_block_cache.yaml --container-name=${containerName} --temp-path=${BLOCK_TEMP_DIR} --output-file=${cloudfuse_CFG}" | |
| - name: Block Blob Coverage - Block Cache | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| BLOCK_TEMP_DIR: ${{ env.BLOCK_TEMP_DIR }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${BLOCK_TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_block_block_cache.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${BLOCK_TEMP_DIR} | |
| cd - | |
| sudo fusermount -u ${MOUNT_DIR} -z | |
| UNMOUNT_TIMEOUT=30 | |
| SECONDS_WAITED_UNMOUNT=0 | |
| while [ $SECONDS_WAITED_UNMOUNT -lt $UNMOUNT_TIMEOUT ]; do | |
| if ! mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_UNMOUNT=$((SECONDS_WAITED_UNMOUNT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - ADLS | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.AZTEST_ADLS_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.AZTEST_ADLS_KEY }}" | |
| ACCOUNT_TYPE: adls | |
| ACCOUNT_ENDPOINT: https://${{ secrets.AZTEST_ADLS_ACC_NAME }}.dfs.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_ADLS_CFG: ${{ env.cloudfuse_ADLS_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test gen-test-config --config-file=azure_key.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_ADLS_CFG}" | |
| - name: ADLS Coverage | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_ADLS_CFG: ${{ env.cloudfuse_ADLS_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_adls.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_ADLS_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -adls=true -tmp-path=${TEMP_DIR} | |
| cd - | |
| sudo fusermount -u ${MOUNT_DIR} | |
| UNMOUNT_TIMEOUT=30 | |
| SECONDS_WAITED_UNMOUNT=0 | |
| while [ $SECONDS_WAITED_UNMOUNT -lt $UNMOUNT_TIMEOUT ]; do | |
| if ! mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_UNMOUNT=$((SECONDS_WAITED_UNMOUNT + POLL_INTERVAL)) | |
| done | |
| - name: ADLS Coverage with profilers | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_ADLS_CFG: ${{ env.cloudfuse_ADLS_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| cp ${cloudfuse_ADLS_CFG} /tmp/configAdlsProfilerTemp.yaml | |
| echo "dynamic-profile: true" >> /tmp/configAdlsProfilerTemp.yaml | |
| echo "cpu-profile: /tmp/cpuAdlsProfTmp" >> /tmp/configAdlsProfilerTemp.yaml | |
| echo "mem-profile: /tmp/memAdlsProfTmp" >> /tmp/configAdlsProfilerTemp.yaml | |
| echo "health_monitor:" >> /tmp/configAdlsProfilerTemp.yaml | |
| echo " enable-monitoring: true" >> /tmp/configAdlsProfilerTemp.yaml | |
| echo " monitor-disable-list:" >> /tmp/configAdlsProfilerTemp.yaml | |
| echo " - cloudfuse_stats" >> /tmp/configAdlsProfilerTemp.yaml | |
| cat /tmp/configAdlsProfilerTemp.yaml | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_adls_profiler.cov mount ${MOUNT_DIR} --config-file=/tmp/configAdlsProfilerTemp.yaml --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} | |
| cd - | |
| sudo fusermount -u ${MOUNT_DIR} | |
| UNMOUNT_TIMEOUT=30 | |
| SECONDS_WAITED_UNMOUNT=0 | |
| while [ $SECONDS_WAITED_UNMOUNT -lt $UNMOUNT_TIMEOUT ]; do | |
| if ! mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_UNMOUNT=$((SECONDS_WAITED_UNMOUNT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - S3 | |
| env: | |
| S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" | |
| S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" | |
| S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" | |
| S3_REGION: "${{ secrets.S3TEST_REGION }}" | |
| S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| RANDOM_NUMBERS=$(head /dev/urandom | tr -dc '0-9' | head -c 8) | |
| COMBINED_STRING="test${RANDOM_NUMBERS}" | |
| echo "S3_SUBDIRECTORY=${COMBINED_STRING}" >> $GITHUB_ENV | |
| echo $COMBINED_STRING | |
| export S3_SUBDIRECTORY=${COMBINED_STRING} | |
| ./cloudfuse.test gen-test-config --config-file=s3_key.yaml --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG} | |
| - name: S3 Coverage | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_s3.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} | |
| cd - | |
| sudo fusermount -u ${MOUNT_DIR} | |
| UNMOUNT_TIMEOUT=30 | |
| SECONDS_WAITED_UNMOUNT=0 | |
| while [ $SECONDS_WAITED_UNMOUNT -lt $UNMOUNT_TIMEOUT ]; do | |
| if ! mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_UNMOUNT=$((SECONDS_WAITED_UNMOUNT + POLL_INTERVAL)) | |
| done | |
| - name: S3 Coverage with profilers | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| cp ${cloudfuse_CFG} /tmp/configBlockProfilerTemp.yaml | |
| echo "dynamic-profile: true" >> /tmp/configBlockProfilerTemp.yaml | |
| echo "cpu-profile: /tmp/cpuBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml | |
| echo "mem-profile: /tmp/memBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml | |
| echo "health_monitor:" >> /tmp/configBlockProfilerTemp.yaml | |
| echo " enable-monitoring: true" >> /tmp/configBlockProfilerTemp.yaml | |
| echo " monitor-disable-list:" >> /tmp/configBlockProfilerTemp.yaml | |
| echo " - cloudfuse_stats" >> /tmp/configBlockProfilerTemp.yaml | |
| cat /tmp/configBlockProfilerTemp.yaml | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_s3_profiler.cov mount ${MOUNT_DIR} --config-file=/tmp/configBlockProfilerTemp.yaml --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} | |
| cd - | |
| sudo fusermount -u ${MOUNT_DIR} | |
| UNMOUNT_TIMEOUT=30 | |
| SECONDS_WAITED_UNMOUNT=0 | |
| while [ $SECONDS_WAITED_UNMOUNT -lt $UNMOUNT_TIMEOUT ]; do | |
| if ! mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_UNMOUNT=$((SECONDS_WAITED_UNMOUNT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - S3 - Block Cache | |
| env: | |
| S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" | |
| S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" | |
| S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" | |
| S3_REGION: "${{ secrets.S3TEST_REGION }}" | |
| S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| BLOCK_TEMP_DIR: ${{ env.BLOCK_TEMP_DIR }} | |
| run: "./cloudfuse.test gen-test-config --config-file=s3_key_block_cache.yaml --temp-path=${BLOCK_TEMP_DIR} --output-file=${cloudfuse_CFG}" | |
| - name: S3 Coverage - Block Cache | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| BLOCK_TEMP_DIR: ${{ env.BLOCK_TEMP_DIR }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${BLOCK_TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_s3_block_cache.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${BLOCK_TEMP_DIR} | |
| cd - | |
| sudo fusermount -u ${MOUNT_DIR} -z | |
| UNMOUNT_TIMEOUT=30 | |
| SECONDS_WAITED_UNMOUNT=0 | |
| while [ $SECONDS_WAITED_UNMOUNT -lt $UNMOUNT_TIMEOUT ]; do | |
| if ! mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_UNMOUNT=$((SECONDS_WAITED_UNMOUNT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - S3 with Size Tracker | |
| env: | |
| S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" | |
| S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" | |
| S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" | |
| S3_REGION: "${{ secrets.S3TEST_REGION }}" | |
| S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| RANDOM_NUMBERS=$(head /dev/urandom | tr -dc '0-9' | head -c 8) | |
| COMBINED_STRING="test${RANDOM_NUMBERS}" | |
| echo "S3_SUBDIRECTORY=${COMBINED_STRING}" >> $GITHUB_ENV | |
| echo $COMBINED_STRING | |
| export S3_SUBDIRECTORY=${COMBINED_STRING} | |
| ./cloudfuse.test gen-test-config --config-file=s3_key_size_tracker.yaml --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG} | |
| - name: S3 Coverage with Size Tracker | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_s3.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} -size-tracker=true | |
| cd - | |
| sudo fusermount -u ${MOUNT_DIR} | |
| UNMOUNT_TIMEOUT=30 | |
| SECONDS_WAITED_UNMOUNT=0 | |
| while [ $SECONDS_WAITED_UNMOUNT -lt $UNMOUNT_TIMEOUT ]; do | |
| if ! mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_UNMOUNT=$((SECONDS_WAITED_UNMOUNT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - Stream | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_STREAM_CFG: ${{ env.cloudfuse_STREAM_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_gentest3.cov gen-test-config --config-file=azure_stream.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_STREAM_CFG}" | |
| - name: Block Blob Stream Preparation | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_STREAM_CFG: ${{ env.cloudfuse_STREAM_CFG }} | |
| run: "rm -rf ${MOUNT_DIR}/*\nrm -rf ${TEMP_DIR}/*\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_stream_prep.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_STREAM_CFG} --foreground=true &\nsleep 10\nps -aux | grep cloudfuse\nfor i in {10,50,100}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > ${WORK_DIR}/myfile_{}'\nfor i in {10,50,100}; do echo $i; done | parallel --will-cite -j 5 'cp ${WORK_DIR}/myfile_{} ${MOUNT_DIR}/'\nsudo fusermount -u ${MOUNT_DIR} \nsleep 5" | |
| - name: Block Blob Stream Coverage | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_STREAM_CFG: ${{ env.cloudfuse_STREAM_CFG }} | |
| run: "rm -rf ${MOUNT_DIR}/*\nrm -rf ${TEMP_DIR}/*\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_stream.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_STREAM_CFG} --foreground=true &\nsleep 10\nps -aux | grep cloudfuse\nrm -rf ${MOUNT_DIR}/*\ncd test/e2e_tests\ngo test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR}\ncd -\nsudo fusermount -u ${MOUNT_DIR} \nsleep 5" | |
| - name: Create Config File - Block Blob | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test gen-test-config --config-file=azure_key.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG}" | |
| - name: "CLI : Component Generation" | |
| env: | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| run: |- | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/generate_cmd.cov generate test_component | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| - name: "CLI : Mount List" | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/list_empty_cmd.cov mount list | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| - name: "CLI : Mount all and List" | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: "./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_cmd.cov mount all ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --log-level=log_debug \nif [ $? -ne 0 ]; then\n exit 1\nfi\n\nsleep 20 \n\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/list_cmd_all.cov mount list\nif [ $? -ne 0 ]; then\n exit 1\nfi\n./cloudfuse.test unmount all" | |
| - name: "CLI : Unmount options" | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| shell: bash {0} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_cmd_all.cov mount all ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --log-level=log_debug | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| sleep 20 | |
| cd $(MOUNT_DIR)/$(containerName) | |
| ./cloudfuse.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_cmd_cont_fail.cov unmount $(MOUNT_DIR)/$(containerName) | |
| cd - | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/umnt_cmd_cont.cov unmount ${MOUNT_DIR}/${CONTAINER_NAME} | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/umnt_wild_cmd.cov unmount \"testmut*\" | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/umnt_wild_cmd.cov unmount \"abcdef*\" | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/umnt_negative_cmd.cov unmount abcdef | |
| if [ $? -ne 1 ]; then | |
| exit 1 | |
| fi | |
| for i in {1..5}; do ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/umnt_all_cmd$i.cov unmount all; done | |
| - name: "CLI : Invalid mount path" | |
| timeout-minutes: 2 | |
| shell: bash {0} | |
| env: | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_neg.cov mount all /abc --config-file=${cloudfuse_CFG} --log-level=log_debug | |
| if [ $? -eq 0 ]; then | |
| exit 1 | |
| fi | |
| - name: "CLI : Mount all with secure config" | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| timeout-minutes: 2 | |
| run: "./cloudfuse.test unmount all\ncp ${cloudfuse_CFG} /tmp/configMountall.yaml\necho \"mountall:\" >> /tmp/configMountall.yaml\necho \" container-allowlist:\" >> /tmp/configMountall.yaml\necho \" - abcd\" >> /tmp/configMountall.yaml\ncat /tmp/configMountall.yaml\n\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/secure_encrypt_all.cov secure encrypt --config-file=/tmp/configMountall.yaml --output-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 \nif [ $? -ne 0 ]; then\n exit 1\nfi\n\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_all_cmd_secure.cov mount all ${MOUNT_DIR} --config-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 --log-level=log_debug --foreground=true &\nif [ $? -ne 0 ]; then\n exit 1\nfi\n\nsleep 5\n./cloudfuse.test unmount all" | |
| - name: "CLI : Mount all with secure config 2" | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| timeout-minutes: 2 | |
| run: "./cloudfuse.test unmount all\ncp ${cloudfuse_CFG} /tmp/configMountall.yaml\necho \"mountall:\" >> /tmp/configMountall.yaml\necho \" container-denylist:\" >> /tmp/configMountall.yaml\necho \" - abcd\" >> /tmp/configMountall.yaml\ncat /tmp/configMountall.yaml\n\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/secure_encrypt_all2.cov secure encrypt --config-file=/tmp/configMountall.yaml --output-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 \nif [ $? -ne 0 ]; then\n exit 1\nfi\n\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_all_cmd_secure2.cov mount all ${MOUNT_DIR} --config-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 --log-level=log_debug --foreground=true &\nif [ $? -ne 0 ]; then\n exit 1\nfi\n\nsleep 5\n./cloudfuse.test unmount all" | |
| - name: "CLI : Remount test" | |
| timeout-minutes: 2 | |
| shell: bash {0} | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test unmount all | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_foreg.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --log-level=log_debug --foreground=true & | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| sleep 5 | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_remount.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --log-level=log_debug | |
| if [ $? -eq 0 ]; then | |
| exit 1 | |
| fi | |
| sleep 5 | |
| ./cloudfuse.test unmount all | |
| - name: "CLI : doc generation" | |
| timeout-minutes: 2 | |
| shell: bash {0} | |
| env: | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/doc1.cov doc | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/doc2.cov doc --output-location /notexists | |
| touch ~/a.txt | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/doc2.cov doc --output-location ~/a.txt | |
| rm -rf ~/a.txt | |
| - name: "CLI : version" | |
| timeout-minutes: 2 | |
| shell: bash {0} | |
| env: | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/version1.cov --version | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/version2.cov version | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/version2.cov version --check | |
| - name: "CLI : Config change simulator" | |
| shell: bash {0} | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| set +x | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test unmount all | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_foreg_2.cov mount all ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --log-level=log_debug --foreground=true & | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| sleep 5 | |
| echo -e "\n\nlogging:\n level:log_debug\n type:base" >> ${cloudfuse_CFG} | |
| sed -i 's/timeout:.*/timeout: 300/' ${cloudfuse_CFG} | |
| sleep 5 | |
| ./cloudfuse.test unmount all | |
| - name: "CLI : Secure Config" | |
| shell: bash {0} | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "set +x\nrm -rf ${MOUNT_DIR}/*\nrm -rf ${TEMP_DIR}/*\n./cloudfuse.test unmount all\n./cloudfuse.test gen-test-config --config-file=azure_key.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG}\n\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/secure_encrypt.cov secure encrypt --config-file=${cloudfuse_CFG} --output-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 \nif [ $? -ne 0 ]; then\n exit 1\nfi\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_secure.cov mount ${MOUNT_DIR} --config-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 &\nsleep 10\nps -aux | grep cloudfuse\nrm -rf ${MOUNT_DIR}/*\ncd test/e2e_tests\ngo test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -adls=false -tmp-path=${TEMP_DIR}\ncd -\n\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/secure_set.cov secure set --config-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 --key=logging.level --value=log_debug\n./cloudfuse.test unmount all\nsleep 5" | |
| - name: "CLI : Health monitor stop pid" | |
| shell: bash {0} | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test unmount all | |
| sudo kill -9 `pidof cloudfuse.test` | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| mkdir -p ${MOUNT_DIR}/hmon_test | |
| sudo ln -s `pwd`/cfusemon /usr/local/bin/cfusemon | |
| ls -l /usr/local/bin/cfusemon* | |
| cat /tmp/configAdlsProfilerTemp.yaml | |
| ./cloudfuse.test mount ${MOUNT_DIR}/hmon_test --config-file=/tmp/configAdlsProfilerTemp.yaml | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| ps -aux | grep cfusemon | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cfusemon_stop_pid.cov health-monitor stop --pid=`pidof cloudfuse` | |
| ./cloudfuse.test unmount all | |
| - name: "CLI : Health monitor stop all" | |
| shell: bash {0} | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test unmount all | |
| sudo kill -9 `pidof cloudfuse.test` | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| mkdir -p ${MOUNT_DIR}/hmon_test | |
| cat /tmp/configAdlsProfilerTemp.yaml | |
| ./cloudfuse.test mount ${MOUNT_DIR}/hmon_test --config-file=/tmp/configAdlsProfilerTemp.yaml | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| ps -aux | grep cfusemon | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cfusemon_stop_all.cov health-monitor stop all | |
| ./cloudfuse.test unmount all | |
| - name: Install & Start Proxy | |
| run: |- | |
| sudo apt-get update --fix-missing -y | |
| sudo apt remove mitmproxy -y | |
| sudo apt-get install python3-setuptools -y | |
| sudo apt install python3-pip -y | |
| sudo pip3 install mitmproxy --ignore-installed blinker | |
| mitmdump -w proxy_logs.txt & | |
| - name: Configure Proxy | |
| run: |- | |
| sudo mkdir /usr/local/share/ca-certificates/extra | |
| sudo cp ~/.mitmproxy/mitmproxy-ca-cert.cer /usr/local/share/ca-certificates/extra/mitmproxy-ca-cert.crt | |
| sudo update-ca-certificates | |
| export https_proxy="127.0.0.1:8080" | |
| export http_proxy="127.0.0.1:8080" | |
| - name: Create Config File - Block Blob Proxy | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test gen-test-config --config-file=azure_key_proxy.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG}" | |
| - name: Block Blob Coverage Proxy | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: "rm -rf ${MOUNT_DIR}/*\nrm -rf ${TEMP_DIR}/*\n./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_block_proxy.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true &\nsleep 10\nps -aux | grep cloudfuse\nrm -rf ${MOUNT_DIR}/*\ncd test/e2e_tests\ngo test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR}\ncd -\nsudo fusermount -u ${MOUNT_DIR} \nsleep 5" | |
| - name: Create Config File - ADLS Proxy | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.AZTEST_ADLS_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.AZTEST_ADLS_KEY }}" | |
| ACCOUNT_TYPE: adls | |
| ACCOUNT_ENDPOINT: https://${{ secrets.AZTEST_ADLS_ACC_NAME }}.dfs.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_ADLS_CFG: ${{ env.cloudfuse_ADLS_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test gen-test-config --config-file=azure_key_proxy.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_ADLS_CFG}" | |
| - name: ADLS Coverage Proxy | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_ADLS_CFG: ${{ env.cloudfuse_ADLS_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_adls_proxy.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_ADLS_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if mount | grep -q "${MOUNT_DIR}"; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| ps -aux | grep cloudfuse | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -adls=true -tmp-path=${TEMP_DIR} | |
| cd - | |
| ./cloudfuse.test unmount all | |
| sleep 5 | |
| - name: Save coverage report | |
| env: | |
| TEMP: ${{ runner.temp }} | |
| run: | | |
| echo 'mode: count' > ./cloudfuse_coverage_raw.rpt | |
| tail -q -n +2 ./*.cov >> ./cloudfuse_coverage_raw.rpt | |
| cat ./cloudfuse_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/custom" | grep -v "component/azstorage/azauthcli.go" | grep -v "exported/exported.go" | grep -v "component/block_cache/stream.go" | grep -v "component/azstorage/azAuthWorkloadIdentity.go" | grep -v "component/azstorage/azauthWorkloadIdentity.go" | grep -v "component/azstorage/policies.go" > ./cloudfuse_coverage.rpt | |
| go tool cover -func cloudfuse_coverage.rpt > ./cloudfuse_func_cover.rpt | |
| go tool cover -html=./cloudfuse_coverage.rpt -o ./cloudfuse_coverage.html | |
| go tool cover -html=./cloudfuse_ut.cov -o ./cloudfuse_ut.html | |
| cp ./*.rpt ${TEMP}/ | |
| cp ./*.cov ${TEMP}/ | |
| cp ./*.html ${TEMP}/ | |
| - name: Overall coverage check | |
| run: |- | |
| chmod 777 ./test/scripts/coveragecheck.sh | |
| ./test/scripts/coveragecheck.sh | |
| - name: File level coverage check | |
| run: "./test/scripts/coveragecheck.sh file" | |
| BuildAndTest-Coverage-Windows: | |
| strategy: | |
| matrix: | |
| go: ["1.25"] | |
| job_name: ["windows"] | |
| include: | |
| - job_name: windows | |
| os: windows-latest | |
| containerName: "test-cnt-win" | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| MOUNT_DIR: "C:\\Users\\runneradmin\\blob_mnt" | |
| TEMP_DIR: "C:\\Users\\runneradmin\\cloudfuse_tmp" | |
| BLOCK_TEMP_DIR: "C:\\Users\\runneradmin\\cloudfuse_block_tmp" | |
| WORK_DIR: "." | |
| cloudfuse_ADLS_CFG: "./cloudfuse.adls.yaml" | |
| cloudfuse_ADLS_CFG_HMON: "./cloudfuse.adls.hmon.yaml" | |
| cloudfuse_CFG: "./cloudfuse.yaml" | |
| cloudfuse_STREAM_CFG: "./cloudfuse_stream.yaml" | |
| GH_API_TOKEN: ${{ secrets.GH_API_TOKEN }} | |
| # Skip for dependabot updates | |
| if: ${{ github.triggering_actor != 'dependabot[bot]' }} | |
| timeout-minutes: 60 | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Install Go | |
| uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 | |
| with: | |
| go-version: ${{ matrix.go }} | |
| check-latest: true | |
| - run: go version | |
| # From rclone github repo under MIT license | |
| # https://github.com/rclone/rclone/blob/master/.github/workflows/build.yml | |
| - name: Install Libraries on Windows | |
| shell: powershell | |
| run: | | |
| $ProgressPreference = 'SilentlyContinue' | |
| choco install -y winfsp zip | |
| echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| # Copy mingw32-make.exe to make.exe so the same command line | |
| # can be used on Windows as on macOS and Linux | |
| $path = (get-command mingw32-make.exe).Path | |
| Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe') | |
| - name: Create Directory Structure | |
| shell: bash | |
| env: | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| run: |- | |
| mkdir -p ${TEMP_DIR} | |
| chown -R `whoami` ${TEMP_DIR} | |
| chmod 777 ${TEMP_DIR} | |
| - name: Build | |
| run: | | |
| ./build.sh | |
| - name: Create Azure Configuration File on Windows | |
| env: | |
| AZTEST_BLOCK_ACC_NAME: ${{ secrets.AZTEST_BLOCK_ACC_NAME }} | |
| AZTEST_ADLS_ACC_NAME: ${{ secrets.AZTEST_ADLS_ACC_NAME }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| AZTEST_BLOCK_KEY: ${{ secrets.AZTEST_BLOCK_KEY }} | |
| AZTEST_ADLS_KEY: ${{ secrets.AZTEST_ADLS_KEY }} | |
| AZTEST_BLOCK_SAS: ${{ secrets.AZTEST_BLOCK_SAS }} | |
| AZTEST_BLOCK_CONT_SAS_UBN_18: ${{ secrets.AZTEST_BLOCK_CONT_SAS_UBN_18 }} | |
| AZTEST_BLOCK_CONT_SAS_UBN_20: ${{ secrets.AZTEST_BLOCK_CONT_SAS_UBN_20 }} | |
| AZTEST_ADLS_SAS: ${{ secrets.AZTEST_ADLS_SAS }} | |
| AZTEST_APP_ID: ${{ secrets.AZTEST_APP_ID }} | |
| AZTEST_RES_ID: ${{ secrets.AZTEST_RES_ID }} | |
| AZTEST_OBJ_ID: ${{ secrets.AZTEST_OBJ_ID }} | |
| AZTEST_CLIENT: ${{ secrets.AZTEST_CLIENT }} | |
| AZTEST_TENANT: ${{ secrets.AZTEST_TENANT }} | |
| AZTEST_SECRET: ${{ secrets.AZTEST_SECRET }} | |
| run: |- | |
| $cnfFile="~\azuretest.json" | |
| Write-Host $cnfFile | |
| $content = @" | |
| { | |
| "block-acct": "$env:AZTEST_BLOCK_ACC_NAME", | |
| "adls-acct": "$env:AZTEST_ADLS_ACC_NAME", | |
| "block-cont": "$env:CONTAINER_NAME", | |
| "adls-cont": "$env:CONTAINER_NAME", | |
| "block-key": "$env:AZTEST_BLOCK_KEY", | |
| "adls-key": "$env:AZTEST_ADLS_KEY", | |
| "block-sas": "$env:AZTEST_BLOCK_SAS", | |
| "block-cont-sas-ubn-18": "$env:AZTEST_BLOCK_CONT_SAS_UBN_18", | |
| "block-cont-sas-ubn-20": "$env:AZTEST_BLOCK_CONT_SAS_UBN_20", | |
| "adls-sas": "$env:AZTEST_ADLS_SAS", | |
| "msi-appid": "$env:AZTEST_APP_ID", | |
| "msi-resid": "$env:AZTEST_RES_ID", | |
| "msi-objid": "$env:AZTEST_OBJ_ID", | |
| "spn-client": "$env:AZTEST_CLIENT", | |
| "spn-tenant": "$env:AZTEST_TENANT", | |
| "spn-secret": "$env:AZTEST_SECRET", | |
| "skip-msi": true, | |
| "skip-azcli": true, | |
| "proxy-address": "" | |
| } | |
| "@ | |
| $content | Out-File -FilePath $cnfFile -Encoding Ascii | |
| shell: powershell | |
| - name: Create S3 Configuration File on Windows | |
| env: # Added | |
| S3TEST_BUCKET_NAME: ${{ secrets.S3TEST_BUCKET_NAME }} | |
| S3TEST_ACCESS_KEY: ${{ secrets.S3TEST_ACCESS_KEY }} | |
| S3TEST_SECRET_KEY: ${{ secrets.S3TEST_SECRET_KEY }} | |
| S3TEST_ENDPOINT: ${{ secrets.S3TEST_ENDPOINT }} | |
| S3TEST_REGION: ${{ secrets.S3TEST_REGION }} | |
| run: |- | |
| $cnfFile="~\s3test.json" | |
| Write-Host $cnfFile | |
| $content = @" | |
| { | |
| "bucket-name": "$env:S3TEST_BUCKET_NAME", | |
| "access-key": "$env:S3TEST_ACCESS_KEY", | |
| "secret-key": "$env:S3TEST_SECRET_KEY", | |
| "endpoint": "$env:S3TEST_ENDPOINT", | |
| "region": "$env:S3TEST_REGION" | |
| } | |
| "@ | |
| $content | Out-File -FilePath $cnfFile -Encoding Ascii | |
| shell: powershell | |
| - name: UT Code Coverage | |
| run: go test -v -timeout=1h ./... --tags=unittest,storagetest -covermode count -coverprofile ./cloudfuse_ut.cov | |
| - name: Build coverage binary | |
| run: go test -coverpkg="./..." -covermode=count -c -o cloudfuse.test | |
| - name: Create Config File - Block Blob | |
| shell: bash | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_gentest1.cov gen-test-config --config-file=azure_key.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG}" | |
| - name: Block Blob Coverage | |
| shell: bash | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_block.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} -quick-test=true -stream-direct-test=true | |
| cd - | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $CFL_PID > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: Block Blob Coverage with profilers | |
| shell: bash | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| cp ${cloudfuse_CFG} /tmp/configBlockProfilerTemp.yaml | |
| echo "dynamic-profile: true" >> /tmp/configBlockProfilerTemp.yaml | |
| echo "cpu-profile: /tmp/cpuBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml | |
| echo "mem-profile: /tmp/memBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml | |
| echo "health_monitor:" >> /tmp/configBlockProfilerTemp.yaml | |
| echo " enable-monitoring: true" >> /tmp/configBlockProfilerTemp.yaml | |
| echo " monitor-disable-list:" >> /tmp/configBlockProfilerTemp.yaml | |
| echo " - cloudfuse_stats" >> /tmp/configBlockProfilerTemp.yaml | |
| cat /tmp/configBlockProfilerTemp.yaml | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_block_profiler.cov mount ${MOUNT_DIR} --config-file=/tmp/configBlockProfilerTemp.yaml --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} -quick-test=true -stream-direct-test=true | |
| cd - | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $CFL_PID > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - Block Blob - Block Cache | |
| shell: bash | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| containerName: ${{ matrix.containerName }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| BLOCK_TEMP_DIR: ${{ env.BLOCK_TEMP_DIR }} | |
| run: "./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_gentest1.cov gen-test-config --config-file=azure_key_block_cache.yaml --container-name=${containerName} --temp-path=${BLOCK_TEMP_DIR} --output-file=${cloudfuse_CFG}" | |
| - name: Block Blob Coverage - Block Cache | |
| shell: bash | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| BLOCK_TEMP_DIR: ${{ env.BLOCK_TEMP_DIR }} | |
| run: |- | |
| rm -rf ${BLOCK_TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_block_block_cache.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| CFL_PID=$! | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${BLOCK_TEMP_DIR} -quick-test=true -stream-direct-test=true | |
| cd - | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $CFL_PID > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - ADLS | |
| shell: bash | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.AZTEST_ADLS_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.AZTEST_ADLS_KEY }}" | |
| ACCOUNT_TYPE: adls | |
| ACCOUNT_ENDPOINT: https://${{ secrets.AZTEST_ADLS_ACC_NAME }}.dfs.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_ADLS_CFG: ${{ env.cloudfuse_ADLS_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test gen-test-config --config-file=azure_key.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_ADLS_CFG}" | |
| - name: ADLS Coverage | |
| shell: bash | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_ADLS_CFG: ${{ env.cloudfuse_ADLS_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_adls.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_ADLS_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} -adls=true -quick-test=true -stream-direct-test=true | |
| cd - | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $CFL_PID > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: ADLS Coverage with profilers | |
| shell: bash | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_ADLS_CFG: ${{ env.cloudfuse_ADLS_CFG }} | |
| cloudfuse_ADLS_CFG_HMON: ${{ env.cloudfuse_ADLS_CFG_HMON }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| cp ${cloudfuse_ADLS_CFG} ${cloudfuse_ADLS_CFG_HMON} | |
| echo "dynamic-profile: true" >> ${cloudfuse_ADLS_CFG_HMON} | |
| echo "cpu-profile: $HOME/hmontemp/cpuAdlsProfTmp" >> ${cloudfuse_ADLS_CFG_HMON} | |
| echo "mem-profile: $HOME/hmontemp/memAdlsProfTmp" >> ${cloudfuse_ADLS_CFG_HMON} | |
| echo "health_monitor:" >> ${cloudfuse_ADLS_CFG_HMON} | |
| echo " enable-monitoring: true" >> ${cloudfuse_ADLS_CFG_HMON} | |
| echo " monitor-disable-list:" >> ${cloudfuse_ADLS_CFG_HMON} | |
| echo " - cloudfuse_stats" >> ${cloudfuse_ADLS_CFG_HMON} | |
| cat ${cloudfuse_ADLS_CFG_HMON} | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_adls_profiler.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_ADLS_CFG_HMON} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| rm -rf ${MOUNT_DIR}/* | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} -quick-test=true -stream-direct-test=true | |
| cd - | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $CFL_PID > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - S3 | |
| shell: bash | |
| env: | |
| S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" | |
| S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" | |
| S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" | |
| S3_REGION: "${{ secrets.S3TEST_REGION }}" | |
| S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| RANDOM_NUMBERS="" | |
| for i in {1..8}; do | |
| RANDOM_NUMBERS+=$(( RANDOM % 10 )) | |
| done | |
| COMBINED_STRING="test${RANDOM_NUMBERS}" | |
| echo "S3_SUBDIRECTORY=${COMBINED_STRING}" >> $GITHUB_ENV | |
| echo $COMBINED_STRING | |
| export S3_SUBDIRECTORY=${COMBINED_STRING} | |
| ./cloudfuse.test gen-test-config --config-file=s3_key.yaml --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG} | |
| - name: S3 Coverage | |
| shell: bash | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_s3.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} -quick-test=true -stream-direct-test=true | |
| cd - | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $CFL_PID > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: S3 Coverage with profilers | |
| shell: bash | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| cp ${cloudfuse_CFG} configBlockProfilerTemp.yaml | |
| echo "dynamic-profile: true" >> configBlockProfilerTemp.yaml | |
| echo "cpu-profile: /tmp/cpuBlockProfTmp" >> configBlockProfilerTemp.yaml | |
| echo "mem-profile: /tmp/memBlockProfTmp" >> configBlockProfilerTemp.yaml | |
| echo "health_monitor:" >> configBlockProfilerTemp.yaml | |
| echo " enable-monitoring: true" >> configBlockProfilerTemp.yaml | |
| echo " monitor-disable-list:" >> configBlockProfilerTemp.yaml | |
| echo " - cloudfuse_stats" >> configBlockProfilerTemp.yaml | |
| cat /tmp/configBlockProfilerTemp.yaml | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_s3_profiler.cov mount ${MOUNT_DIR} --config-file=/tmp/configBlockProfilerTemp.yaml --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} -quick-test=true -stream-direct-test=true | |
| cd - | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $CFL_PID > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - S3 - Block Cache | |
| shell: bash | |
| env: | |
| S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" | |
| S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" | |
| S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" | |
| S3_REGION: "${{ secrets.S3TEST_REGION }}" | |
| S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| BLOCK_TEMP_DIR: ${{ env.BLOCK_TEMP_DIR }} | |
| run: "./cloudfuse.test gen-test-config --config-file=s3_key_block_cache.yaml --temp-path=${BLOCK_TEMP_DIR} --output-file=${cloudfuse_CFG}" | |
| - name: S3 Coverage - Block Cache | |
| shell: bash | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| BLOCK_TEMP_DIR: ${{ env.BLOCK_TEMP_DIR }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${BLOCK_TEMP_DIR}/* | |
| rm -rf ${BLOCK_TEMP_DIR} | |
| mkdir -p ${BLOCK_TEMP_DIR} | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_s3_block_cache.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| CFL_PID=$! | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${BLOCK_TEMP_DIR} -quick-test=true -stream-direct-test=true | |
| cd - | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $CFL_PID > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - S3 with Size Tracker | |
| shell: bash | |
| env: | |
| S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" | |
| S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" | |
| S3_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" | |
| S3_REGION: "${{ secrets.S3TEST_REGION }}" | |
| S3_SECRET_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| RANDOM_NUMBERS="" | |
| for i in {1..8}; do | |
| RANDOM_NUMBERS+=$(( RANDOM % 10 )) | |
| done | |
| COMBINED_STRING="test${RANDOM_NUMBERS}" | |
| echo "S3_SUBDIRECTORY=${COMBINED_STRING}" >> $GITHUB_ENV | |
| echo $COMBINED_STRING | |
| export S3_SUBDIRECTORY=${COMBINED_STRING} | |
| ./cloudfuse.test gen-test-config --config-file=s3_key_size_tracker.yaml --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG} | |
| - name: S3 Coverage with Size Tracker | |
| shell: bash | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_s3.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| cd test/e2e_tests | |
| go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} -quick-test=true -stream-direct-test=true -size-tracker=true | |
| cd - | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $CFL_PID > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: Create Config File - Stream | |
| shell: bash | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_STREAM_CFG: ${{ env.cloudfuse_STREAM_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_gentest3.cov gen-test-config --config-file=azure_stream.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_STREAM_CFG}" | |
| # Need to fix these to run on Windows | |
| # - name: Block Blob Stream Preparation | |
| # shell: bash | |
| # run: |- | |
| # rm -rf ${MOUNT_DIR}/* | |
| # rm -rf ${TEMP_DIR}/* | |
| # ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_stream_prep.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_STREAM_CFG} --foreground=true & | |
| # sleep 10 | |
| # pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| # for i in {10,50,100}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > ${WORK_DIR}/myfile_{}' | |
| # for i in {10,50,100}; do echo $i; done | parallel --will-cite -j 5 'cp ${WORK_DIR}/myfile_{} ${MOUNT_DIR}/' | |
| # kill $pid | |
| # sleep 5" | |
| # - name: Block Blob Stream Coverage | |
| # shell: bash | |
| # run: |- | |
| # rm -rf ${MOUNT_DIR}/* | |
| # rm -rf ${TEMP_DIR}/* | |
| # ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cloudfuse_stream.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_STREAM_CFG} --foreground=true & | |
| # sleep 10 | |
| # pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| # rm -rf ${MOUNT_DIR}/* | |
| # cd test/e2e_tests | |
| # go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -tmp-path=${TEMP_DIR} -quick-test=true -stream-direct-test=true | |
| # cd - | |
| # kill $pid | |
| # sleep 5 | |
| - name: Create Config File - Block Blob | |
| shell: bash | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: "./cloudfuse.test gen-test-config --config-file=azure_key.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG}" | |
| # Following tests don't work on Windows | |
| # - name: 'CLI : Component Generation' | |
| # shell: bash | |
| # run: |- | |
| # ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/generate_cmd.cov generate test_component | |
| # if [ $? -ne 0 ]; then | |
| # exit 1 | |
| # fi | |
| # - name: 'CLI : Mount List' | |
| # shell: bash | |
| # run: |- | |
| # rm -rf ${MOUNT_DIR}/* | |
| # rm -rf ${TEMP_DIR}/* | |
| # ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/list_empty_cmd.cov mount list | |
| # if [ $? -ne 0 ]; then | |
| # exit 1 | |
| # fi | |
| - name: "CLI : Invalid mount path" | |
| timeout-minutes: 2 | |
| shell: bash {0} | |
| env: | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| mkdir -p /abc | |
| set +x | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_neg.cov mount all /abc --config-file=${cloudfuse_CFG} --log-level=log_debug --foreground=true | |
| if [ $? -eq 0 ]; then | |
| exit 1 | |
| fi | |
| - name: "CLI : Remount test" | |
| timeout-minutes: 2 | |
| shell: bash {0} | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_foreg.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --log-level=log_debug --foreground=true & | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| sleep 5 | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_remount.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --log-level=log_debug --foreground=true | |
| if [ $? -eq 0 ]; then | |
| exit 1 | |
| fi | |
| sleep 5 | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| kill $pid | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $pid > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: "CLI : doc generation" | |
| timeout-minutes: 2 | |
| shell: bash {0} | |
| env: | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/doc1.cov doc | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/doc2.cov doc --output-location /notexists | |
| touch ~/a.txt | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/doc2.cov doc --output-location ~/a.txt | |
| rm -rf ~/a.txt | |
| - name: "CLI : version" | |
| timeout-minutes: 2 | |
| shell: bash {0} | |
| env: | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| run: |- | |
| set +x | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/version1.cov --version | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/version2.cov version | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/version2.cov version --check | |
| - name: "CLI : Config change simulator" | |
| shell: bash {0} | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| run: |- | |
| set +x | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_foreg_2.cov mount ${MOUNT_DIR} --config-file=${cloudfuse_CFG} --log-level=log_debug --foreground=true & | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| sleep 5 | |
| echo -e "\n\nlogging:\n level:log_debug\n type:base" >> ${cloudfuse_CFG} | |
| sed -i 's/timeout:.*/timeout: 300/' ${cloudfuse_CFG} | |
| sleep 5 | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| kill $pid | |
| sleep 5 | |
| - name: "CLI : Secure Config" | |
| shell: bash {0} | |
| env: | |
| NIGHTLY_STO_ACC_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| NIGHTLY_STO_ACC_KEY: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| ACCOUNT_TYPE: block | |
| ACCOUNT_ENDPOINT: https://${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}.blob.core.windows.net | |
| VERBOSE_LOG: false | |
| USE_HTTP: false | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| WORK_DIR: ${{ env.WORK_DIR }} | |
| cloudfuse_CFG: ${{ env.cloudfuse_CFG }} | |
| CONTAINER_NAME: ${{ matrix.containerName }} | |
| run: |- | |
| set +x | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| ./cloudfuse.test unmount all | |
| ./cloudfuse.test gen-test-config --config-file=azure_key.yaml --container-name=${CONTAINER_NAME} --temp-path=${TEMP_DIR} --output-file=${cloudfuse_CFG} | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/secure_encrypt.cov secure encrypt --config-file=${cloudfuse_CFG} --output-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 | |
| if [ $? -ne 0 ]; then | |
| exit 1 | |
| fi | |
| # ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/mount_secure.cov mount ${MOUNT_DIR} --config-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 --foreground=true & | |
| # sleep 10 | |
| # pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| # rm -rf ${MOUNT_DIR}/* | |
| # cd test/e2e_tests | |
| # go test -v -timeout=7200s ./... -args -mnt-path=${MOUNT_DIR} -adls=false -tmp-path=${TEMP_DIR} | |
| # cd - | |
| # ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/secure_set.cov secure set --config-file=${WORK_DIR}/cloudfuse.azsec --passphrase=12312312312312312312312312312312 --key=logging.level --value=log_debug | |
| # kill $pid | |
| # sleep 5 | |
| - name: "CLI : Health monitor stop pid" | |
| shell: bash {0} | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_ADLS_CFG_HMON: ${{ env.cloudfuse_ADLS_CFG_HMON }} | |
| run: |- | |
| set +x | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| cat ${cloudfuse_ADLS_CFG_HMON} | |
| ./cloudfuse.test mount ${MOUNT_DIR} --config-file=${cloudfuse_ADLS_CFG_HMON} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| win_pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f5` | |
| pid_mon=`ps -a | grep cfusemon | tr -s ' ' | cut -d ' ' -f2` | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cfusemon_stop_pid.cov health-monitor stop --pid=$win_pid | |
| kill $pid | |
| kill $pid_mon | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $pid > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: "CLI : Health monitor stop all" | |
| shell: bash {0} | |
| env: | |
| MOUNT_DIR: ${{ env.MOUNT_DIR }} | |
| TEMP_DIR: ${{ env.TEMP_DIR }} | |
| cloudfuse_ADLS_CFG_HMON: ${{ env.cloudfuse_ADLS_CFG_HMON }} | |
| run: |- | |
| set +x | |
| rm -rf ${MOUNT_DIR}/* | |
| rm -rf ${TEMP_DIR}/* | |
| cat ${cloudfuse_ADLS_CFG_HMON} | |
| ./cloudfuse.test mount ${MOUNT_DIR} --config-file=${cloudfuse_ADLS_CFG_HMON} --foreground=true & | |
| READY_TIMEOUT=20 # Seconds to wait for mount to be ready | |
| POLL_INTERVAL=1 # Seconds between checks | |
| SECONDS_WAITED=0 | |
| while [ $SECONDS_WAITED -lt $READY_TIMEOUT ]; do | |
| if ls "${MOUNT_DIR}" >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED=$((SECONDS_WAITED + POLL_INTERVAL)) | |
| done | |
| pid=`ps -a | grep cloudfuse | tr -s ' ' | cut -d ' ' -f2` | |
| pid_mon=`ps -a | grep cfusemon | tr -s ' ' | cut -d ' ' -f2` | |
| ./cloudfuse.test -test.v -test.coverprofile=${WORK_DIR}/cfusemon_stop_all.cov health-monitor stop all | |
| kill $pid | |
| kill $pid_mon | |
| PROCESS_EXIT_TIMEOUT=10 # Total timeout for graceful process exit | |
| SECONDS_WAITED_EXIT=0 | |
| while [ $SECONDS_WAITED_EXIT -lt $PROCESS_EXIT_TIMEOUT ]; do | |
| if ! ps -p $pid > /dev/null; then | |
| break | |
| fi | |
| sleep $POLL_INTERVAL | |
| SECONDS_WAITED_EXIT=$((SECONDS_WAITED_EXIT + POLL_INTERVAL)) | |
| done | |
| - name: Save coverage report | |
| shell: bash | |
| run: | | |
| echo 'mode: count' > ./cloudfuse_coverage_raw.rpt | |
| tail -q -n +2 ./*.cov >> ./cloudfuse_coverage_raw.rpt | |
| cat ./cloudfuse_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/custom" | grep -v "component/azstorage/azauthcli.go" | grep -v "exported/exported.go" | grep -v "component/block_cache/stream.go" | grep -v "component/azstorage/azauthWorkloadIdentity.go" | grep -v "component/azstorage/policies.go" > ./cloudfuse_coverage.rpt | |
| go tool cover -func cloudfuse_coverage.rpt > ./cloudfuse_func_cover.rpt | |
| go tool cover -html=./cloudfuse_coverage.rpt -o ./cloudfuse_coverage.html | |
| go tool cover -html=./cloudfuse_ut.cov -o ./cloudfuse_ut.html | |
| - name: Overall coverage check | |
| shell: bash | |
| run: |- | |
| chmod 777 ./test/scripts/coveragecheck.sh | |
| ./test/scripts/coveragecheck.sh | |
| - name: File level coverage check | |
| shell: bash | |
| run: "./test/scripts/coveragecheck.sh file" | |
| AccountCleanup-Cleanup: | |
| needs: | |
| - BuildAndTest-Coverage | |
| - BuildAndTest-Coverage-Windows | |
| strategy: | |
| matrix: | |
| go: ["1.25"] | |
| job_name: ["linux"] | |
| include: | |
| - job_name: linux | |
| os: ubuntu-latest | |
| containerName: "test-cnt-ubn" | |
| fuselib: libfuse3-dev | |
| fuselib2: fuse3 | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Install Go | |
| uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 | |
| with: | |
| go-version: ${{ matrix.go }} | |
| check-latest: true | |
| - run: go version | |
| - name: Install dependency | |
| env: | |
| FUSE_LIB: ${{ matrix.fuselib }} | |
| FUSE_LIB2: ${{ matrix.fuselib2 }} | |
| run: |- | |
| sudo apt-get install ${FUSE_LIB} ${FUSE_LIB2} | |
| - name: Cleanup Blob Storage | |
| run: go test -timeout 120m -v test/accoutcleanup/accountcleanup_test.go | |
| env: | |
| STORAGE_ACCOUNT_NAME: "${{ secrets.NIGHTLY_STO_BLOB_ACC_NAME }}" | |
| STORAGE_ACCOUNT_Key: "${{ secrets.NIGHTLY_STO_BLOB_ACC_KEY }}" | |
| - name: Cleanup Datalake Storage | |
| run: go test -timeout 120m -v test/accoutcleanup/accountcleanup_test.go | |
| env: | |
| STORAGE_ACCOUNT_NAME: "${{ secrets.AZTEST_ADLS_ACC_NAME }}" | |
| STORAGE_ACCOUNT_Key: "${{ secrets.AZTEST_ADLS_KEY }}" | |
| - name: Cleanup S3 Storage | |
| run: go test -timeout 120m -v test/s3cleanup/s3cleanup_test.go | |
| env: | |
| AWS_ACCESS_KEY_ID: "${{ secrets.S3TEST_ACCESS_KEY }}" | |
| AWS_SECRET_ACCESS_KEY: "${{ secrets.S3TEST_SECRET_KEY }}" | |
| AWS_REGION: "${{ secrets.S3TEST_REGION }}" | |
| S3_BUCKET_NAME: "${{ secrets.S3TEST_BUCKET_NAME }}" | |
| S3_ENDPOINT: "${{ secrets.S3TEST_ENDPOINT }}" |