Skip to content

Commit c762fea

Browse files
khustup2khustup2
authored
Merge pull request #3108 from activeloopai/sre_tidy_changes
sre tidy changes
2 parents 5917a0c + 3db4003 commit c762fea

File tree

4 files changed

+95
-105
lines changed

4 files changed

+95
-105
lines changed

‎.github/workflows/pg-extension-build.yaml‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,10 @@ jobs:
9494
shell: bash
9595
run: |-
9696
bash scripts/build_scripts/manage_cache.sh download postgres
97-
git clone https://github.com/microsoft/vcpkg.git && cd vcpkg && git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836 && ./bootstrap-vcpkg.sh
97+
git clone https://github.com/microsoft/vcpkg.git
98+
cd vcpkg
99+
git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836
100+
./bootstrap-vcpkg.sh
98101
99102
- name: install python dependencies
100103
shell: bash
@@ -110,15 +113,7 @@ jobs:
110113
python3 scripts/build_pg_ext.py prod --deeplake-static --pg-versions "${PG_VERSION}"
111114
echo -e "${GREEN}Done.${DEFAULT}"
112115
113-
- name: install clang-tidy
114-
shell: bash
115-
run: |-
116-
echo -e "${YELLOW}Installing clang-tidy...${DEFAULT}"
117-
yum install -y clang-tools-extra
118-
clang-tidy --version
119-
echo -e "${GREEN}Clang-tidy installed.${DEFAULT}"
120-
121-
- name: run clang-tidy
116+
- name: clang-tidy
122117
shell: bash
123118
continue-on-error: true
124119
run: |-

‎scripts/build_scripts/manage_cache.sh‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,49 +4,49 @@ VCPKG_CACHE_PATH="${VCPKG_CACHE_PATH:-/vcpkg-cache}"
44
CACHE_BUCKET="${CACHE_BUCKET:-activeloop-platform-tests}"
55

66
if [ "${CREDENTIALS_PATH}" ] && [ -f "${CREDENTIALS_PATH}" ]; then
7-
set -a
8-
# shellcheck disable=SC1090
9-
source "${CREDENTIALS_PATH}"
10-
set +a
7+
set -a
8+
# shellcheck disable=SC1090
9+
source "${CREDENTIALS_PATH}"
10+
set +a
1111
fi
1212

13-
case "$(arch)" in
13+
case "$(uname -m)" in
1414
"x86_64" | "amd64")
15-
ARCH_NAME="x86_64"
16-
;;
15+
ARCH_NAME="x86_64"
16+
;;
1717
"aarch64" | "arm64")
18-
ARCH_NAME="aarch64"
19-
;;
18+
ARCH_NAME="aarch64"
19+
;;
2020
*)
21-
echo "Unsupported architecture: $(arch)"
22-
exit 1
23-
;;
21+
echo "Unsupported architecture: $(arch)"
22+
exit 1
23+
;;
2424
esac
2525

2626
CACHE_PREFIX="vcpkg-cache/indra/${ARCH_NAME}"
2727

2828
if [ "${AWS_ENDPOINT_URL}" ]; then
29-
__cmd__="s5cmd --endpoint-url ${AWS_ENDPOINT_URL}"
29+
__cmd__="s5cmd --endpoint-url ${AWS_ENDPOINT_URL}"
3030
else
31-
__cmd__="s5cmd"
31+
__cmd__="s5cmd"
3232
fi
3333

3434
case "$1" in
3535
"download")
36-
mkdir -p "${VCPKG_CACHE_PATH}"
37-
$__cmd__ cp "s3://${CACHE_BUCKET}/${CACHE_PREFIX}/${2}/*" "${VCPKG_CACHE_PATH}/" || {
38-
echo -e "\n\n\n[WARN] Failed to download cache: $2\n\n\n"
39-
exit 0
40-
}
41-
;;
36+
mkdir -p "${VCPKG_CACHE_PATH}"
37+
$__cmd__ cp "s3://${CACHE_BUCKET}/${CACHE_PREFIX}/${2}/*" "${VCPKG_CACHE_PATH}/" || {
38+
echo -e "\n\n\n[WARN] Failed to download cache: $2\n\n\n"
39+
exit 0
40+
}
41+
;;
4242
"upload")
43-
$__cmd__ cp "${VCPKG_CACHE_PATH}/" "s3://${CACHE_BUCKET}/${CACHE_PREFIX}/${2}/" || {
44-
echo -e "\n\n\n[WARN] Failed to upload cache: $2\n\n\n"
45-
exit 0
46-
}
47-
;;
43+
$__cmd__ cp "${VCPKG_CACHE_PATH}/" "s3://${CACHE_BUCKET}/${CACHE_PREFIX}/${2}/" || {
44+
echo -e "\n\n\n[WARN] Failed to upload cache: $2\n\n\n"
45+
exit 0
46+
}
47+
;;
4848
*)
49-
echo "Unsupported command: $1"
50-
exit 1
51-
;;
49+
echo "Unsupported command: $1"
50+
exit 1
51+
;;
5252
esac

‎scripts/run_clang_tidy.sh‎

Lines changed: 61 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,98 +1,92 @@
1-
#!/bin/bash
1+
#!/usr/bin/env bash
2+
23
set -e
34

45
# Script to run clang-tidy on pg_deeplake source files
56
# Usage: ./scripts/run_clang_tidy.sh [build_dir]
67

7-
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
8-
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
9-
BUILD_DIR="${1:-${PROJECT_ROOT}/builds/deeplake-pg-dev}"
10-
11-
# Convert BUILD_DIR to absolute path if it's relative
12-
if [[ "$BUILD_DIR" != /* ]]; then
13-
BUILD_DIR="${PROJECT_ROOT}/${BUILD_DIR}"
14-
fi
8+
trap "rm -rf ${TEMP_DIR}" EXIT
159

16-
echo "Running clang-tidy on cpp/deeplake_pg..."
17-
echo "Build directory: ${BUILD_DIR}"
10+
function run_tidy() {
11+
local output file_warnings file_errors
12+
output=$(clang-tidy --header-filter='.*deeplake_pg.*' -p "$BUILD_DIR" "$1" 2>&1 || true)
13+
file_warnings=$(echo "$output" | grep -c "deeplake_pg/.*warning:" || true)
14+
file_errors=$(echo "$output" | grep -c "deeplake_pg/.*error:" || true)
15+
echo "$file|$file_warnings|$file_errors" >"${TEMP_DIR}/${1}.count"
16+
if [ "$file_errors" -gt 0 ]; then
17+
echo "$output" | grep "deeplake_pg/.*error:" >"${TEMP_DIR}/${1}.output"
18+
elif [ "$file_warnings" -gt 0 ]; then
19+
echo "$output" | grep "deeplake_pg/.*warning:" >"${TEMP_DIR}/${1}.output"
20+
fi
21+
}
22+
23+
function log() {
24+
local level ts
25+
level="$1"
26+
ts="$(date --utc -Iseconds)"
27+
shift
28+
printf "[%s] - [%s] - \"%s\"\n" "${level^^}" "${ts}" "$*"
29+
}
30+
31+
SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
32+
PROJECT_ROOT="$(cd $SCRIPT_DIR && realpath ..)"
33+
BUILD_DIR="$(realpath "${1:-${PROJECT_ROOT}/builds/deeplake-pg-dev}")"
34+
35+
log info "running clang-tidy on cpp/deeplake_pg..."
36+
log info "build directory: ${BUILD_DIR}"
1837

1938
if [ ! -f "${BUILD_DIR}/compile_commands.json" ]; then
20-
echo "Error: compile_commands.json not found in ${BUILD_DIR}"
21-
echo "Please build the project first to generate compile_commands.json"
22-
exit 1
39+
log error "compile_commands.json not found in ${BUILD_DIR}, please build the project first to generate compile_commands.json"
40+
exit 1
2341
fi
2442

2543
cd "${PROJECT_ROOT}/cpp/deeplake_pg"
26-
27-
# Create temp directory for parallel output
2844
TEMP_DIR=$(mktemp -d)
29-
trap "rm -rf ${TEMP_DIR}" EXIT
3045

31-
echo "Running clang-tidy in parallel..."
46+
log info "running clang-tidy in parallel..."
3247

33-
# Run clang-tidy for each file in parallel
48+
worker_count="$(nproc)"
3449
for file in *.cpp; do
35-
(
36-
OUTPUT=$(clang-tidy --header-filter='.*deeplake_pg.*' -p "$BUILD_DIR" "$file" 2>&1 || true)
37-
38-
# Count warnings in this file (only in deeplake_pg directory)
39-
FILE_WARNINGS=$(echo "$OUTPUT" | grep -c "deeplake_pg/.*warning:" || true)
40-
# Count errors in this file
41-
FILE_ERRORS=$(echo "$OUTPUT" | grep -c "deeplake_pg/.*error:" || true)
42-
43-
# Save results to temp file
44-
echo "$file|$FILE_WARNINGS|$FILE_ERRORS" > "${TEMP_DIR}/${file}.count"
45-
46-
if [ "$FILE_ERRORS" -gt 0 ]; then
47-
echo "$OUTPUT" | grep "deeplake_pg/.*error:" > "${TEMP_DIR}/${file}.output"
48-
elif [ "$FILE_WARNINGS" -gt 0 ]; then
49-
echo "$OUTPUT" | grep "deeplake_pg/.*warning:" > "${TEMP_DIR}/${file}.output"
50-
fi
51-
) &
50+
run_tidy $file &
51+
while [ "$(jobs | wc -l)" -ge "$worker_count" ]; do
52+
sleep 0.1
53+
done
5254
done
53-
54-
# Wait for all parallel jobs to complete
5555
wait
5656

57-
echo ""
58-
echo "Processing results..."
57+
log info "processing results..."
5958

6059
WARNINGS=0
6160
ERRORS=0
6261

63-
# Process results in order
6462
for file in *.cpp; do
65-
if [ -f "${TEMP_DIR}/${file}.count" ]; then
66-
IFS='|' read -r fname FILE_WARNINGS FILE_ERRORS < "${TEMP_DIR}/${file}.count"
67-
68-
if [ "$FILE_ERRORS" -gt 0 ]; then
69-
echo "❌ $file - has $FILE_ERRORS errors"
70-
cat "${TEMP_DIR}/${file}.output"
71-
ERRORS=$((ERRORS + FILE_ERRORS))
72-
elif [ "$FILE_WARNINGS" -gt 0 ]; then
73-
echo "⚠ $file - has $FILE_WARNINGS warnings"
74-
cat "${TEMP_DIR}/${file}.output"
75-
WARNINGS=$((WARNINGS + FILE_WARNINGS))
76-
else
77-
echo "✓ $file - no issues"
78-
fi
63+
if [ -f "${TEMP_DIR}/${file}.count" ]; then
64+
IFS='|' read -r fname FILE_WARNINGS FILE_ERRORS <"${TEMP_DIR}/${file}.count"
65+
66+
if [ "$FILE_ERRORS" -gt 0 ]; then
67+
log error "❌ $file - has $FILE_ERRORS errors"
68+
cat "${TEMP_DIR}/${file}.output"
69+
ERRORS=$((ERRORS + FILE_ERRORS))
70+
elif [ "$FILE_WARNINGS" -gt 0 ]; then
71+
log warn "⚠ $file - has $FILE_WARNINGS warnings"
72+
cat "${TEMP_DIR}/${file}.output"
73+
WARNINGS=$((WARNINGS + FILE_WARNINGS))
74+
else
75+
log info "✓ $file - no issues"
7976
fi
77+
fi
8078
done
8179

82-
echo ""
83-
echo "===================="
84-
echo "Clang-Tidy Summary"
85-
echo "===================="
86-
echo "Total warnings: $WARNINGS"
87-
echo "Total errors: $ERRORS"
80+
log info "clang-tidy summary: warnings=$WARNINGS errors=$ERRORS"
8881

8982
if [ $ERRORS -gt 0 ]; then
90-
echo "❌ Clang-tidy found $ERRORS errors!"
91-
exit 1
83+
log error "❌ Clang-tidy found $ERRORS errors!"
84+
exit 1
9285
elif [ $WARNINGS -gt 0 ]; then
93-
echo "⚠ Clang-tidy found $WARNINGS warnings (non-blocking)"
94-
exit 0
86+
log warn "⚠ Clang-tidy found $WARNINGS warnings (non-blocking)"
87+
exit 0
9588
else
96-
echo "✅ No issues found!"
97-
exit 0
89+
log info "✅ No issues found!"
90+
exit 0
9891
fi
92+

‎vcpkg‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Subproject commit 6f29f12e82a8293156836ad81cc9bf5af41fe836

0 commit comments

Comments
 (0)