man-group
diff --git a/‎.github/actions/enable_logging/action.yml‎
Lines changed: 13 additions & 0 deletions b/‎.github/actions/enable_logging/action.yml‎
Lines changed: 13 additions & 0 deletions
diff --git a/‎.github/actions/set_persistent_storage_env_vars/action.yml‎
Lines changed: 3 additions & 1 deletion b/‎.github/actions/set_persistent_storage_env_vars/action.yml‎
Lines changed: 3 additions & 1 deletion
diff --git a/‎.github/workflows/build.yml‎
Lines changed: 42 additions & 10 deletions b/‎.github/workflows/build.yml‎
Lines changed: 42 additions & 10 deletions
diff --git a/‎.github/workflows/build_steps.yml‎
Lines changed: 14 additions & 0 deletions b/‎.github/workflows/build_steps.yml‎
Lines changed: 14 additions & 0 deletions
diff --git a/‎.github/workflows/build_with_conda.yml‎
Lines changed: 72 additions & 2 deletions b/‎.github/workflows/build_with_conda.yml‎
Lines changed: 72 additions & 2 deletions
@@ -0,0 +1,13 @@
+name: 'Enable ArcticDB Debug Logging'
+description: 'Set the necessary variables for finer logging on all storage types'
+runs:
+  using: "composite"
+  steps:
+  # add a step to run locally define python script
+    - name: Enable ArcticDB Debug Logging
+      shell: bash
+      run: |
+        # Enable all debug logs
+        echo "ARCTICDB_all_loglevel=debug" >> $GITHUB_ENV
+        echo "ARCTICDB_AWS_LogLevel_int=6" >> $GITHUB_ENV
+        echo "AZURE_LOG_LEVEL=1" >> $GITHUB_ENV
@@ -23,7 +23,9 @@ runs:
 
       run: |
         # Common
-        echo "ARCTICDB_PERSISTENT_STORAGE_TESTS=1" >> $GITHUB_ENV
+        if [[ "${{inputs.persistent_storage}}" != "no" ]]; then 
+          echo "ARCTICDB_PERSISTENT_STORAGE_TESTS=1" >> $GITHUB_ENV
+        fi
         echo "ARCTICDB_PERSISTENT_STORAGE_UNIQUE_ID=${{ github.ref_name }}_${{ github.run_id }}" >> $GITHUB_ENV
         echo "ARCTICDB_PERSISTENT_STORAGE_STRATEGY_BRANCH=${{ inputs.strategy_branch }}" >> $GITHUB_ENV
         # This is the top level path for all test, this is where to write data that should be shared between jobs (e.g. seed job)
 
@@ -19,6 +19,7 @@ on:
           - 'no'
           - 'AWS_S3'
           - 'GCPXML'
+          - 'AZURE'
         default: 'no'
 
       pypi_publish:
@@ -34,20 +35,25 @@ on:
         description: Tag of the ArcticDB development image to use for the Linux C++ tests build
         type: string
         default: arcticdb-dev-clang:latest
-      pytest_args:
-        description: Rewrite what tests will run or do your own pytest line if string starts with pytest ... (Example -- pytest -n auto -v --count=50 -x python/tests/compat)
-        type: string
-        default: ""
-      version_cache_full_test:
-        description: 'Run tests with both version cache 0 and 2000000000'
+      macos_enabled:
+        description: Enable macOS tests
         required: false
         default: false
         type: boolean
-      macos_enabled:
-        description: Enable macOS tests
+      version_cache_full_test:
+        description: 'Run tests with both version cache 0 and 2000000000'
         required: false
         default: false
         type: boolean
+      flags:
+        type: string
+        description: 'Enable debug logging or debug by setting respective value to 1'
+        required: true
+        default: "DEBUG_LOGGING=0;DEBUG=0"                  
+      pytest_args:
+        description: Rewrite what tests will run or do your own pytest line if string starts with pytest ... (Example -- pytest -n auto -v --count=50 -x python/tests/compat)
+        type: string
+        default: ""
       run_full_matrix_of_persistent_tests:
         description: Run full matrix of persistent tests
         required: false
@@ -66,6 +72,8 @@ jobs:
       INPUT_STORAGE: ${{ inputs.persistent_storage }}
     outputs:
       storage: ${{ steps.set-storage.outputs.storage }}
+      DEBUG_LOGGING_ENABLED: ${{ steps.parse-flags.outputs.DEBUG_LOGGING_ENABLED }}
+      DEBUG_ENABLED: ${{ steps.parse-flags.outputs.DEBUG_ENABLED }}      
     steps:
       - name: Determine Storage Type Based On Schedule
         id: set-storage
@@ -78,6 +86,8 @@ jobs:
             _storage='AWS_S3'
           elif [[ "$day" == "Mon" || "$day" == "Wed" ]]; then
             _storage='GCPXML'
+          elif [[ "$day" == "Fri" ]]; then
+            _storage='AZURE'
           else
             echo "UNSPECIFIED RESULT for this day (assumed LMDB)"
             _storage='no'
@@ -99,6 +109,21 @@ jobs:
           echo "Final storage type for workflow execution: $_storage"
           echo "storage=$_storage" >> $GITHUB_OUTPUT
 
+      - name: Parse flags
+        id: parse-flags
+        run: |
+          declare -A FLAGS
+          IFS=';' read -ra PAIRS <<< "${{ inputs.flags }}"
+          for pair in "${PAIRS[@]}"; do
+            key="${pair%%=*}"
+            value="${pair##*=}"
+            # Trim leading/trailing whitespace
+            value="$(echo "$value" | xargs)"
+            echo "$key=$value"
+            # Format of flags will be as defined below
+            echo "${key}_ENABLED=$value" >> $GITHUB_OUTPUT
+          done
+
   cibw_docker_image:
     uses: ./.github/workflows/cibw_docker_image.yml
     permissions: {packages: write}
@@ -307,6 +332,9 @@ jobs:
       pytest_xdist_mode: ${{matrix.pytest_xdist_mode}}
       pytest_args: ${{inputs.pytest_args || ''}}
       version_cache_full_test: ${{inputs.version_cache_full_test || false}}
+      DEBUG_ENABLED: ${{ needs.storage_type.outputs.DEBUG_ENABLED }}            
+      DEBUG_LOGGING_ENABLED: ${{ needs.storage_type.outputs.DEBUG_LOGGING_ENABLED }}            
+
 
   cpp-test-windows:
     needs: [common_config]
@@ -355,6 +383,8 @@ jobs:
       pytest_xdist_mode: "-n logical --dist worksteal"
       pytest_args: ${{inputs.pytest_args || ''}}
       version_cache_full_test: ${{github.event_name == 'schedule' || inputs.version_cache_full_test || false}}
+      DEBUG_ENABLED: ${{ needs.storage_type.outputs.DEBUG_ENABLED }}            
+      DEBUG_LOGGING_ENABLED: ${{ needs.storage_type.outputs.DEBUG_LOGGING_ENABLED }}            
 
   persistent_storage_verify_linux:
     needs: [common_config, build-python-wheels-linux, build-python-wheels-windows]
@@ -418,7 +448,7 @@ jobs:
       job_type: cleanup
 
   build-python-wheels-macos:
-    needs: [common_config]
+    needs: [common_config, storage_type]
     if: ${{ inputs.macos_enabled }}
     strategy:
       fail-fast: false
@@ -439,9 +469,11 @@ jobs:
       cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
       matrix: ${{toJson(matrix.matrix_override)}}
       python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
-      persistent_storage: ${{ inputs.persistent_storage }}
+      persistent_storage: ${{ needs.storage_type.outputs.storage }}
       pytest_xdist_mode: ${{matrix.pytest_xdist_mode}}
       pytest_args: ${{inputs.pytest_args}}
+      DEBUG_ENABLED: ${{ needs.storage_type.outputs.DEBUG_ENABLED }}            
+      DEBUG_LOGGING_ENABLED: ${{ needs.storage_type.outputs.DEBUG_LOGGING_ENABLED }}            
 
   can_merge:
     needs: [cpp-test-linux, cpp-test-windows, cpp-test-macos, build-python-wheels-linux, build-python-wheels-windows, build-python-wheels-macos, persistent_storage_verify_linux, persistent_storage_verify_windows]
 
@@ -14,6 +14,8 @@ on:
       pytest_args:       {default: "", type: string, description: a way to rewrite the pytest args to change what tests are being run}
       version_cache_full_test: {default: false, type: boolean, description: if true - tests will run with both version cache 0 and 2000000000 otherwise only 2000000000}
       run_full_matrix_of_persistent_tests: {default: false, type: boolean, description: if true - persistent tests will run for all python versions else only for 12 }
+      DEBUG_LOGGING_ENABLED: {default: "0", type: string, required: false, description: 'Enable debug logging "1" (disabled "0")'}
+      DEBUG_ENABLED: {default: "0", type: string, required: false, description: 'Enable debug logging "1" (disabled "0")'}
 jobs:
   compile:
     strategy:
@@ -419,8 +421,19 @@ jobs:
           aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}"
           aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}"
 
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ inputs.DEBUG_ENABLED == '1' }}  
+        
+      - name: Set ArcticDB Debug Logging
+        if: ${{ inputs.DEBUG_LOGGING_ENABLED == '1' }}        
+        uses: ./.github/actions/enable_logging
+
       - name: Run test
         run: |
+          # find ssl directory where cacerts are (for Azure)
+          openssl version -d 
+          # list file descriptors and other limits of the runner
           ulimit -a
           export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0
           if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == pytest* ]]; then
@@ -438,6 +451,7 @@ jobs:
           HYPOTHESIS_PROFILE: ci_${{matrix.os}}
           PYTEST_XDIST_MODE: ${{inputs.pytest_xdist_mode}}
           ARCTICDB_PYTEST_ARGS: ${{inputs.pytest_args}}
+          STORAGE_TYPE: ${{ env.real_tests_storage_type }}
           # Testing with 0(no version cache) and -1(will use default timeout)
           VERSION_MAP_RELOAD_INTERVAL: ${{matrix.version_cache_timeout == 'NoCache' && 0 || -1}}
 
 
@@ -22,8 +22,32 @@ on:
         type: boolean
         required: true
         default: true        
+      persistent_storage:
+        description: "Run against what persistent storage type? (no is LMDB/default)"
+        type: choice
+        options:
+          - 'no'
+          - 'AWS_S3'
+          - 'GCPXML'
+          - 'AZURE'
+        default: 'no'
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with debugging enabled'
+        required: false
+        default: false                  
+      run_enable_logging:
+        description: 'Enabled debug logging'
+        type: boolean
+        required: false
+        default: false        
+      run_commandline:
+        description: 'Run custom commandline before tests, Like: export ARCTICDB_STORAGE_AZURE=1; ....'
+        type: string
+        required: false
+        default: ""    
       run_custom_pytest_command:
-        description: 'Run custom pytest command (curdir is project root). Or pass additional arguments to default command'
+        description: '*Run custom pytest command, instead of standard(Note: curdir is project root), or pass additional arguments to default command'
         type: string
         required: false
         default: ""        
@@ -107,11 +131,34 @@ jobs:
       - name: Check no arcticdb file depend on tests package
         shell: bash -l {0}
         run: |
-          build_tooling/checks.sh          
+          build_tooling/checks.sh       
+          
+      - name: Set persistent storage variables
+        # Should be executed for all persistent storages but not for LMDB
+        if: ${{ env.real_tests_storage_type != 'no' }}
+        uses: ./.github/actions/set_persistent_storage_env_vars
+        with:
+          aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}"
+          aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}"
+          gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}"
+          gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}"
+          azure_container: "githubblob" # DEFAULT BUCKET FOR AZURE
+          azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}"
+          persistent_storage: ${{ inputs.persistent_storage }}
+
+      - name: Set ArcticDB Debug Logging
+        if: ${{ inputs.run_enable_logging }}        
+        uses: ./.github/actions/enable_logging
+
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ inputs.debug_enabled }}        
 
       - name: Test with pytest
         shell: bash -l {0}
         run: |
+          echo "Run commandline: $COMMANDLINE"
+          eval "$COMMANDLINE"
           export ARCTICDB_RAND_SEED=$RANDOM
           export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0
           if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == *pytest* ]]; then
@@ -125,6 +172,7 @@ jobs:
           fi
         env:
           ARCTICDB_USING_CONDA: 1
+          COMMANDLINE: ${{ inputs.run_commandline }}
           # Use the Mongo created in the service container above to test against
           CI_MONGO_HOST: mongodb
           ARCTICDB_PYTEST_ARGS: ${{ inputs.run_custom_pytest_command }}
@@ -191,9 +239,30 @@ jobs:
         run: |
           npm install -g azurite
 
+      - name: Set persistent storage variables
+        uses: ./.github/actions/set_persistent_storage_env_vars
+        with:
+          aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}"
+          aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}"
+          gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}"
+          gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}"
+          azure_container: "githubblob" # DEFAULT BUCKET FOR AZURE
+          azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}"
+          persistent_storage: ${{ inputs.persistent_storage }}
+
+      - name: Set ArcticDB Debug Logging
+        if: ${{ inputs.run_enable_logging }}        
+        uses: ./.github/actions/enable_logging
+
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ inputs.debug_enabled }}        
+
       - name: Test with pytest
         shell: bash -l {0}
         run: |
+          echo "Run commandline: $COMMANDLINE"
+          eval "$COMMANDLINE"
           export ARCTICDB_RAND_SEED=$RANDOM
           export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0
           if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == pytest* ]]; then
@@ -208,5 +277,6 @@ jobs:
           fi
         env:
           ARCTICDB_USING_CONDA: 1
+          COMMANDLINE: ${{ inputs.run_commandline }}
           ARCTICDB_PYTEST_ARGS: ${{ inputs.run_custom_pytest_command }}