diff --git a/.github/workflows/pg-extension-build.yaml b/.github/workflows/pg-extension-build.yaml index f509faaa23..466b8e0a7d 100644 --- a/.github/workflows/pg-extension-build.yaml +++ b/.github/workflows/pg-extension-build.yaml @@ -94,7 +94,10 @@ jobs: shell: bash run: |- bash scripts/build_scripts/manage_cache.sh download postgres - git clone https://github.com/microsoft/vcpkg.git && cd vcpkg && git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836 && ./bootstrap-vcpkg.sh + git clone https://github.com/microsoft/vcpkg.git + cd vcpkg + git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836 + ./bootstrap-vcpkg.sh - name: install python dependencies shell: bash @@ -110,15 +113,7 @@ jobs: python3 scripts/build_pg_ext.py prod --deeplake-static --pg-versions "${PG_VERSION}" echo -e "${GREEN}Done.${DEFAULT}" - - name: install clang-tidy - shell: bash - run: |- - echo -e "${YELLOW}Installing clang-tidy...${DEFAULT}" - yum install -y clang-tools-extra - clang-tidy --version - echo -e "${GREEN}Clang-tidy installed.${DEFAULT}" - - - name: run clang-tidy + - name: clang-tidy shell: bash continue-on-error: true run: |- diff --git a/scripts/build_scripts/manage_cache.sh b/scripts/build_scripts/manage_cache.sh index 19dfdd5cdb..bcd284d2d5 100755 --- a/scripts/build_scripts/manage_cache.sh +++ b/scripts/build_scripts/manage_cache.sh @@ -4,49 +4,49 @@ VCPKG_CACHE_PATH="${VCPKG_CACHE_PATH:-/vcpkg-cache}" CACHE_BUCKET="${CACHE_BUCKET:-activeloop-platform-tests}" if [ "${CREDENTIALS_PATH}" ] && [ -f "${CREDENTIALS_PATH}" ]; then - set -a - # shellcheck disable=SC1090 - source "${CREDENTIALS_PATH}" - set +a + set -a + # shellcheck disable=SC1090 + source "${CREDENTIALS_PATH}" + set +a fi -case "$(arch)" in +case "$(uname -m)" in "x86_64" | "amd64") - ARCH_NAME="x86_64" - ;; + ARCH_NAME="x86_64" + ;; "aarch64" | "arm64") - ARCH_NAME="aarch64" - ;; + ARCH_NAME="aarch64" + ;; *) - echo "Unsupported architecture: $(arch)" - exit 1 - ;; + echo "Unsupported architecture: $(arch)" + exit 1 + ;; esac CACHE_PREFIX="vcpkg-cache/indra/${ARCH_NAME}" if [ "${AWS_ENDPOINT_URL}" ]; then - __cmd__="s5cmd --endpoint-url ${AWS_ENDPOINT_URL}" + __cmd__="s5cmd --endpoint-url ${AWS_ENDPOINT_URL}" else - __cmd__="s5cmd" + __cmd__="s5cmd" fi case "$1" in "download") - mkdir -p "${VCPKG_CACHE_PATH}" - $__cmd__ cp "s3://${CACHE_BUCKET}/${CACHE_PREFIX}/${2}/*" "${VCPKG_CACHE_PATH}/" || { - echo -e "\n\n\n[WARN] Failed to download cache: $2\n\n\n" - exit 0 - } - ;; + mkdir -p "${VCPKG_CACHE_PATH}" + $__cmd__ cp "s3://${CACHE_BUCKET}/${CACHE_PREFIX}/${2}/*" "${VCPKG_CACHE_PATH}/" || { + echo -e "\n\n\n[WARN] Failed to download cache: $2\n\n\n" + exit 0 + } + ;; "upload") - $__cmd__ cp "${VCPKG_CACHE_PATH}/" "s3://${CACHE_BUCKET}/${CACHE_PREFIX}/${2}/" || { - echo -e "\n\n\n[WARN] Failed to upload cache: $2\n\n\n" - exit 0 - } - ;; + $__cmd__ cp "${VCPKG_CACHE_PATH}/" "s3://${CACHE_BUCKET}/${CACHE_PREFIX}/${2}/" || { + echo -e "\n\n\n[WARN] Failed to upload cache: $2\n\n\n" + exit 0 + } + ;; *) - echo "Unsupported command: $1" - exit 1 - ;; + echo "Unsupported command: $1" + exit 1 + ;; esac diff --git a/scripts/run_clang_tidy.sh b/scripts/run_clang_tidy.sh index 74fd878c11..dd59d313de 100755 --- a/scripts/run_clang_tidy.sh +++ b/scripts/run_clang_tidy.sh @@ -1,98 +1,92 @@ -#!/bin/bash +#!/usr/bin/env bash + set -e # Script to run clang-tidy on pg_deeplake source files # Usage: ./scripts/run_clang_tidy.sh [build_dir] -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -BUILD_DIR="${1:-${PROJECT_ROOT}/builds/deeplake-pg-dev}" - -# Convert BUILD_DIR to absolute path if it's relative -if [[ "$BUILD_DIR" != /* ]]; then - BUILD_DIR="${PROJECT_ROOT}/${BUILD_DIR}" -fi +trap "rm -rf ${TEMP_DIR}" EXIT -echo "Running clang-tidy on cpp/deeplake_pg..." -echo "Build directory: ${BUILD_DIR}" +function run_tidy() { + local output file_warnings file_errors + output=$(clang-tidy --header-filter='.*deeplake_pg.*' -p "$BUILD_DIR" "$1" 2>&1 || true) + file_warnings=$(echo "$output" | grep -c "deeplake_pg/.*warning:" || true) + file_errors=$(echo "$output" | grep -c "deeplake_pg/.*error:" || true) + echo "$file|$file_warnings|$file_errors" >"${TEMP_DIR}/${1}.count" + if [ "$file_errors" -gt 0 ]; then + echo "$output" | grep "deeplake_pg/.*error:" >"${TEMP_DIR}/${1}.output" + elif [ "$file_warnings" -gt 0 ]; then + echo "$output" | grep "deeplake_pg/.*warning:" >"${TEMP_DIR}/${1}.output" + fi +} + +function log() { + local level ts + level="$1" + ts="$(date --utc -Iseconds)" + shift + printf "[%s] - [%s] - \"%s\"\n" "${level^^}" "${ts}" "$*" +} + +SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +PROJECT_ROOT="$(cd $SCRIPT_DIR && realpath ..)" +BUILD_DIR="$(realpath "${1:-${PROJECT_ROOT}/builds/deeplake-pg-dev}")" + +log info "running clang-tidy on cpp/deeplake_pg..." +log info "build directory: ${BUILD_DIR}" if [ ! -f "${BUILD_DIR}/compile_commands.json" ]; then - echo "Error: compile_commands.json not found in ${BUILD_DIR}" - echo "Please build the project first to generate compile_commands.json" - exit 1 + log error "compile_commands.json not found in ${BUILD_DIR}, please build the project first to generate compile_commands.json" + exit 1 fi cd "${PROJECT_ROOT}/cpp/deeplake_pg" - -# Create temp directory for parallel output TEMP_DIR=$(mktemp -d) -trap "rm -rf ${TEMP_DIR}" EXIT -echo "Running clang-tidy in parallel..." +log info "running clang-tidy in parallel..." -# Run clang-tidy for each file in parallel +worker_count="$(nproc)" for file in *.cpp; do - ( - OUTPUT=$(clang-tidy --header-filter='.*deeplake_pg.*' -p "$BUILD_DIR" "$file" 2>&1 || true) - - # Count warnings in this file (only in deeplake_pg directory) - FILE_WARNINGS=$(echo "$OUTPUT" | grep -c "deeplake_pg/.*warning:" || true) - # Count errors in this file - FILE_ERRORS=$(echo "$OUTPUT" | grep -c "deeplake_pg/.*error:" || true) - - # Save results to temp file - echo "$file|$FILE_WARNINGS|$FILE_ERRORS" > "${TEMP_DIR}/${file}.count" - - if [ "$FILE_ERRORS" -gt 0 ]; then - echo "$OUTPUT" | grep "deeplake_pg/.*error:" > "${TEMP_DIR}/${file}.output" - elif [ "$FILE_WARNINGS" -gt 0 ]; then - echo "$OUTPUT" | grep "deeplake_pg/.*warning:" > "${TEMP_DIR}/${file}.output" - fi - ) & + run_tidy $file & + while [ "$(jobs | wc -l)" -ge "$worker_count" ]; do + sleep 0.1 + done done - -# Wait for all parallel jobs to complete wait -echo "" -echo "Processing results..." +log info "processing results..." WARNINGS=0 ERRORS=0 -# Process results in order for file in *.cpp; do - if [ -f "${TEMP_DIR}/${file}.count" ]; then - IFS='|' read -r fname FILE_WARNINGS FILE_ERRORS < "${TEMP_DIR}/${file}.count" - - if [ "$FILE_ERRORS" -gt 0 ]; then - echo "❌ $file - has $FILE_ERRORS errors" - cat "${TEMP_DIR}/${file}.output" - ERRORS=$((ERRORS + FILE_ERRORS)) - elif [ "$FILE_WARNINGS" -gt 0 ]; then - echo "⚠ $file - has $FILE_WARNINGS warnings" - cat "${TEMP_DIR}/${file}.output" - WARNINGS=$((WARNINGS + FILE_WARNINGS)) - else - echo "✓ $file - no issues" - fi + if [ -f "${TEMP_DIR}/${file}.count" ]; then + IFS='|' read -r fname FILE_WARNINGS FILE_ERRORS <"${TEMP_DIR}/${file}.count" + + if [ "$FILE_ERRORS" -gt 0 ]; then + log error "❌ $file - has $FILE_ERRORS errors" + cat "${TEMP_DIR}/${file}.output" + ERRORS=$((ERRORS + FILE_ERRORS)) + elif [ "$FILE_WARNINGS" -gt 0 ]; then + log warn "⚠ $file - has $FILE_WARNINGS warnings" + cat "${TEMP_DIR}/${file}.output" + WARNINGS=$((WARNINGS + FILE_WARNINGS)) + else + log info "✓ $file - no issues" fi + fi done -echo "" -echo "====================" -echo "Clang-Tidy Summary" -echo "====================" -echo "Total warnings: $WARNINGS" -echo "Total errors: $ERRORS" +log info "clang-tidy summary: warnings=$WARNINGS errors=$ERRORS" if [ $ERRORS -gt 0 ]; then - echo "❌ Clang-tidy found $ERRORS errors!" - exit 1 + log error "❌ Clang-tidy found $ERRORS errors!" + exit 1 elif [ $WARNINGS -gt 0 ]; then - echo "⚠ Clang-tidy found $WARNINGS warnings (non-blocking)" - exit 0 + log warn "⚠ Clang-tidy found $WARNINGS warnings (non-blocking)" + exit 0 else - echo "✅ No issues found!" - exit 0 + log info "✅ No issues found!" + exit 0 fi + diff --git a/vcpkg b/vcpkg new file mode 160000 index 0000000000..6f29f12e82 --- /dev/null +++ b/vcpkg @@ -0,0 +1 @@ +Subproject commit 6f29f12e82a8293156836ad81cc9bf5af41fe836