diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9556d6ccc4..062208b0e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,251 +125,6 @@ jobs: wget https://repo.manticoresearch.com/repository/ci/boost_1_75_0.tgz tar -xf boost_1_75_0.tgz - build_linux_debug: - if: (needs.changes.outputs.source == 'true') && (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - needs: [changes, check_branch] - name: Linux debug build - uses: ./.github/workflows/build_template.yml - with: - CTEST_CONFIGURATION_TYPE: "Debug" - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - artifact_name: debug_build - cache_key: build_linux_debug_x86_64 - cmake_command: | - export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake - ctest -VV -S misc/ctest/gltest.cmake --no-compress-output - - test_linux_debug: - if: (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - name: Linux debug mode tests - needs: [build_linux_debug, check_branch] - uses: ./.github/workflows/test_template.yml - with: - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - build_artifact_name: debug_build - artifact_name: debug_test_results - results_name: "Linux debug test results" - timeout: 10 - - build_linux_release: - if: (needs.changes.outputs.source == 'true') && (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - needs: [changes, check_branch] - name: Linux release build - uses: ./.github/workflows/build_template.yml - with: - artifact_name: release_build - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - cache_key: build_linux_release_x86_64 - cmake_command: | - export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake - ctest -VV -S misc/ctest/gltest.cmake --no-compress-output - - test_linux_release: - if: (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - name: Linux release mode tests - needs: [build_linux_release, check_branch] - uses: ./.github/workflows/test_template.yml - with: - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - build_artifact_name: release_build - artifact_name: release_test_results - results_name: "Linux release test results" - timeout: 10 - - - pack_jammy_cache_check: - needs: changes - name: Jammy x86_64 package [check cache] - runs-on: ubuntu-22.04 - outputs: - cache-key: ${{ steps.generate-key.outputs.cache-key }} - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Generate cache key - id: generate-key - run: | - if [ "${{ github.event_name }}" == "pull_request" ]; then - branch=${{ github.head_ref }} - else - branch=$(git rev-parse --abbrev-ref HEAD) - fi - cache_key="jammy-build-${branch}" - echo "cache-key=${cache_key}" >> $GITHUB_OUTPUT - - - name: Set up cache - id: cache - uses: actions/cache@v4 - with: - path: build - key: ${{ steps.generate-key.outputs.cache-key }} - - - name: Override cache hit - id: override-cache-hit - run: | - if [ "${{ needs.changes.outputs.source }}" == "true" ]; then - echo "cache-hit=false" >> $GITHUB_OUTPUT - fi - - - name: Upload build artifacts from cache - if: ${{ steps.override-cache-hit.outputs.cache-hit == 'true' }} - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: build_jammy_RelWithDebInfo_x86_64 - path: "build/manticore*deb" - - pack_jammy: - needs: [pack_jammy_cache_check, changes, check_branch] - if: | - needs.pack_jammy_cache_check.outputs.cache-hit != 'true' || - needs.changes.outputs.source == 'true' - name: Jammy x86_64 package [build] - uses: ./.github/workflows/build_template.yml - with: - DISTR: jammy - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - arch: x86_64 - cmake_command: | - mkdir build - cd build - cmake -DPACK=1 .. - export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake - cmake --build . --target package - cache_key: pack_jammy_x86_64 - artifact_list: "build/manticore*deb" - - pack_jammy_cache_update: - if: (needs.pack_jammy_cache_check.outputs.cache-hit != 'true') && (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - name: Jammy x86_64 package [update cache] - needs: [pack_jammy, pack_jammy_cache_check] - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository # We have to checkout to access .github/workflows/ in further steps - uses: actions/checkout@v3 - - - name: Download built x86_64 Ubuntu Jammy packages - uses: manticoresoftware/download_artifact_with_retries@v3 - continue-on-error: true - with: - name: build_jammy_RelWithDebInfo_x86_64 - path: . - - - name: Save cache - uses: actions/cache@v4 - with: - path: build - key: ${{ needs.pack_jammy_cache_check.outputs.cache-key }} - - test_kit_docker_build: - name: Test Kit docker image - needs: [pack_jammy, pack_jammy_cache_check] - if: always() && needs.pack_jammy.result != 'failure' - runs-on: ubuntu-22.04 - outputs: - out-build: ${{ steps.build.outputs.build_image }} - steps: - - name: Checkout repository # We have to checkout to access .github/workflows/ in further steps - uses: actions/checkout@v3 - - name: Download built x86_64 Ubuntu Jammy packages - uses: manticoresoftware/download_artifact_with_retries@v3 - with: - name: build_jammy_RelWithDebInfo_x86_64 - path: . - # Uncomment this shortcut for debug to save time by not preparing the packages in the pack_jammy job - # - run: | - # wget http://dev2.manticoresearch.com/build_jammy_RelWithDebInfo_x86_64.zip - # unzip build_jammy_RelWithDebInfo_x86_64.zip - # tar -xvf artifact.tar - - name: Calculate short commit hash - id: sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Building docker - id: build - run: | - BUILD_COMMIT=${{ steps.sha.outputs.sha_short }} /bin/bash dist/test_kit_docker_build.sh - echo "build_image=ghcr.io/$REPO_OWNER/manticoresearch:test-kit-${{ steps.sha.outputs.sha_short }}" >> $GITHUB_OUTPUT - - name: Upload docker image artifact - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: manticore_test_kit.img - path: manticore_test_kit.img - - clt: - if: always() && needs.test_kit_docker_build.result != 'failure' - name: CLT - needs: test_kit_docker_build - uses: ./.github/workflows/clt_tests.yml - secrets: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }} - JINA_API_KEY: ${{ secrets.JINA_API_KEY }} - with: - docker_image: test-kit:img - artifact_name: manticore_test_kit.img - repository: ${{ github.repository }} - ref: ${{ github.sha }} - - test_kit_docker_push: - if: always() && needs.changes.outputs.source == 'true' - needs: - - changes - - clt - name: Push Test Kit docker image to repo - runs-on: ubuntu-22.04 - env: - GHCR_USER: ${{ vars.GHCR_USER }} - GHCR_PASSWORD: ${{ secrets.GHCR_PASSWORD }} - REPO_OWNER: ${{ github.repository_owner }} - steps: - - name: Checkout repository # We have to checkout to access .github/workflows/ in further steps - uses: actions/checkout@v3 - - name: Download artifact - uses: manticoresoftware/download_artifact_with_retries@main - with: - name: manticore_test_kit.img - path: . - - name: Calculate short commit hash - id: sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Pushing docker image to repo - id: test-kit-push - run: | - TEST_RESULT=${{ needs.clt.result }} BUILD_COMMIT=${{ steps.sha.outputs.sha_short }} /bin/bash dist/test_kit_docker_push.sh - - build_aarch64: - if: (needs.changes.outputs.source == 'true') && (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - needs: [changes, check_branch] - name: Linux aarch64 build - uses: ./.github/workflows/build_template.yml - # Use -VV instead of -V below for more verbose output - with: - arch: aarch64 - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - cmake_command: | - mkdir build - cd build - export CMAKE_TOOLCHAIN_FILE=$(pwd)/../dist/build_dockers/cross/linux.cmake - ctest -V -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output - cache_key: build_jammy_aarch64 - - build_freebsd: - if: (needs.changes.outputs.source == 'true') && (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - needs: [changes, check_branch] - name: FreeBSD x86_64 build - uses: ./.github/workflows/build_template.yml - with: - DISTR: freebsd13 - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - boost_url_key: none - cmake_command: | - mkdir build - cd build - export CMAKE_TOOLCHAIN_FILE=$(pwd)/../dist/build_dockers/cross/freebsd.cmake - ctest -VV -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output - cache_key: build_freebsd_x86_64 - build_windows: if: (needs.changes.outputs.source == 'true') && (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true needs: [changes, check_branch] @@ -394,16 +149,10 @@ jobs: strategy: fail-fast: false matrix: - name: [1_400, 401_650, 651_and_on] + name: [651_and_on] include: - - name: 1_400 - start: 1 - end: 400 - - name: 401_650 - start: 401 - end: 650 - name: 651_and_on - start: 651 + start: 1 end: 999999 with: CTEST_START: ${{ matrix.start }} @@ -420,18 +169,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Download test report artifacts 1_400 - uses: manticoresoftware/download_artifact_with_retries@v3 - continue-on-error: true - with: - name: windows_test_1_400 - path: . - - name: Download test report artifacts 401_650 - uses: manticoresoftware/download_artifact_with_retries@v3 - continue-on-error: true - with: - name: windows_test_401_650 - path: . - name: Download test report artifacts 651_and_on uses: manticoresoftware/download_artifact_with_retries@v3 continue-on-error: true @@ -455,10 +192,9 @@ jobs: files: build/xml_*/junit_tests.xml comment_mode: failures - name: Per-test results - # IMPORTANT: The value of 3 below should correspond to the test shard count, needs..strategy.job-total doesn't work run: | for file in build/status*; do echo -n "$file: "; cat "$file"; done - grep -o "success" build/status* | wc -l | awk '{if ($1==3) exit 0; else {print "Found only "$1" successful runs out of 3"; exit 1}}' + grep -o "success" build/status* | wc -l | awk '{if ($1==1) exit 0; else {print "Found only "$1" successful runs out of 1"; exit 1}}' shell: bash - name: Upload combined artifacts if: always() diff --git a/.github/workflows/win_test_template.yml b/.github/workflows/win_test_template.yml index 48a7de0134..8892f936fb 100644 --- a/.github/workflows/win_test_template.yml +++ b/.github/workflows/win_test_template.yml @@ -32,7 +32,7 @@ jobs: CTEST_START: ${{ inputs.CTEST_START }} CTEST_END: ${{ inputs.CTEST_END }} # The following is useful to test a specific test, just uncomment it, no need to disable CTEST_START/END - # CTEST_REGEX: test_234 + CTEST_REGEX: (test_|rt_)(411|296) NO_BUILD: 1 COLUMNAR_LOCATOR: ${{ inputs.COLUMNAR_LOCATOR }} steps: @@ -79,9 +79,63 @@ jobs: # Add -VV to ctest to display extra debug info run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 continue-on-error: true + - name: 🚀 Test 2 + id: test2 + # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 + # Add -VV to ctest to display extra debug info + run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 + continue-on-error: true + - name: 🚀 Test 3 + id: test3 + # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 + # Add -VV to ctest to display extra debug info + run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 + continue-on-error: true + - name: 🚀 Test 4 + id: test4 + # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 + # Add -VV to ctest to display extra debug info + run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 + continue-on-error: true + - name: 🚀 Test 5 + id: test5 + # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 + # Add -VV to ctest to display extra debug info + run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 + continue-on-error: true + - name: 🚀 Test 6 + id: test6 + # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 + # Add -VV to ctest to display extra debug info + run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 + continue-on-error: true + - name: 🚀 Test 7 + id: test7 + # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 + # Add -VV to ctest to display extra debug info + run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 + continue-on-error: true + - name: 🚀 Test 8 + id: test8 + # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 + # Add -VV to ctest to display extra debug info + run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 + continue-on-error: true + - name: 🚀 Test 9 + id: test9 + # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 + # Add -VV to ctest to display extra debug info + run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 + continue-on-error: true + - name: 🚀 Test 10 + id: test10 + # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 + # Add -VV to ctest to display extra debug info + run: ctest -VV -S misc/ctest/gltest.cmake --no-compress-output --timeout 600 + continue-on-error: true - name: Remember status if: always() - run: echo "${{ steps.test.outcome }}" > build/status_${{ inputs.CTEST_START }}_${{ inputs.CTEST_END }} + run: echo "${{ steps.test10.outcome }}" > build/status_${{ inputs.CTEST_START }}_${{ inputs.CTEST_END }} continue-on-error: true - name: Prepare test results if: always() diff --git a/misc/ctest/gltest.cmake b/misc/ctest/gltest.cmake index a1c001d62a..4c2045d305 100755 --- a/misc/ctest/gltest.cmake +++ b/misc/ctest/gltest.cmake @@ -20,7 +20,7 @@ set ( CTEST_RESOURCE "$ENV{CTEST_RESOURCE}" ) set_property ( GLOBAL PROPERTY Label P$ENV{CI_PIPELINE_ID} J$ENV{CI_JOB_ID} ) # how may times try the test before it is considered failed -set (RETRIES 5) +set (RETRIES 1) include ( ProcessorCount ) ProcessorCount ( N ) diff --git a/src/threadutils.h b/src/threadutils.h index 5ceb977f2b..973f602606 100644 --- a/src/threadutils.h +++ b/src/threadutils.h @@ -205,7 +205,15 @@ class OperationsQueue_c }; /// stack of a thread (that is NOT stack of the coroutine!) -static const DWORD STACK_SIZE = 128 * 1024; +/// On Windows, use larger stack size to handle deep query parsing recursion +/// (bison parser with many OR clauses can exhaust smaller stacks) +/// With 1519 OR clauses, bison's internal stack requires significant depth +/// Note: Increasing too much can cause address space issues on 32-bit or fragmentation +#if _WIN32 +static const DWORD STACK_SIZE = 64 * 1024; // 3MB for Windows - needed for very long OR chains (1500+ clauses) +#else +static const DWORD STACK_SIZE = 128 * 1024; // 128KB default for Linux/Unix +#endif /// get the pointer to my thread's stack const void * TopOfStack ();