Skip to content

Commit a3ad46d

Browse files
BohuTANGBohuTANG
authored
chore: add settings/login_history log fetch (#18586)
* chore: add settings/login_history log fetch * add upload log shell
1 parent 5cb3fc5 commit a3ad46d

File tree

3 files changed

+188
-16
lines changed

3 files changed

+188
-16
lines changed

‎scripts/selfhost/fetch_log.sh‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -433,56 +433,72 @@ main() {
433433

434434
# Initialize workspace
435435
echo ""
436-
echo "[0/6] Initializing workspace..."
436+
echo "[0/8] Initializing workspace..."
437437
execute_query "DROP STAGE IF EXISTS a5c7667401c0c728c2ef9703bdaea66d9ae2d906;"
438438
execute_query "CREATE STAGE a5c7667401c0c728c2ef9703bdaea66d9ae2d906;"
439439
echo " ✓ Workspace initialized"
440440

441441
local total=0
442442
local success=0
443443

444-
# Stage 1: Columns
444+
# Stage 1: System Settings
445+
process_stage "Fetch System Settings" \
446+
"COPY INTO @a5c7667401c0c728c2ef9703bdaea66d9ae2d906 FROM system.settings;" \
447+
"settings" "1" "8"
448+
total=$((total + stage_total))
449+
success=$((success + stage_success))
450+
451+
# Stage 2: Columns
445452
process_stage "Fetch System Columns" \
446453
"COPY INTO @a5c7667401c0c728c2ef9703bdaea66d9ae2d906 FROM system.columns;" \
447-
"columns" "1" "6"
454+
"columns" "2" "8"
448455
total=$((total + stage_total))
449456
success=$((success + stage_success))
450457

451-
# Stage 2: User Functions
458+
# Stage 3: User Functions
452459
process_stage "Fetch User Functions" \
453460
"COPY INTO @a5c7667401c0c728c2ef9703bdaea66d9ae2d906 FROM (SELECT * FROM system.user_functions);" \
454-
"user_functions" "2" "6"
461+
"user_functions" "3" "8"
455462
total=$((total + stage_total))
456463
success=$((success + stage_success))
457464

458-
# Stage 3: Query History - uses query_logs directory to match restore script expectations
465+
# Stage 4: Query History - uses query_logs directory to match restore script expectations
459466
local event_condition
460467
event_condition=$(get_event_time_condition)
461468
process_stage "Fetch Query History" \
462469
"COPY INTO @a5c7667401c0c728c2ef9703bdaea66d9ae2d906 FROM (SELECT * FROM system_history.query_history WHERE $event_condition);" \
463-
"query_logs" "3" "6"
470+
"query_logs" "4" "8"
464471
total=$((total + stage_total))
465472
success=$((success + stage_success))
466473

467-
# Stage 4: Raw Logs - uses query_raw_logs directory to match restore script expectations
474+
# Stage 5: Raw Logs - uses query_raw_logs directory to match restore script expectations
468475
local time_condition
469476
time_condition=$(get_time_condition)
470477
process_stage "Fetch Raw Logs" \
471478
"COPY INTO @a5c7667401c0c728c2ef9703bdaea66d9ae2d906 FROM (SELECT * FROM system_history.log_history WHERE $time_condition);" \
472-
"query_raw_logs" "4" "6"
479+
"query_raw_logs" "5" "8"
480+
total=$((total + stage_total))
481+
success=$((success + stage_success))
482+
483+
# Stage 6: Login History - uses login_history directory to match restore script expectations
484+
local login_event_condition
485+
login_event_condition=$(get_event_time_condition)
486+
process_stage "Fetch Login History" \
487+
"COPY INTO @a5c7667401c0c728c2ef9703bdaea66d9ae2d906 FROM (SELECT * FROM system_history.login_history WHERE $login_event_condition);" \
488+
"login_history" "6" "8"
473489
total=$((total + stage_total))
474490
success=$((success + stage_success))
475491

476-
# Stage 5: Profile Logs - uses query_profile_logs directory to match restore script expectations
492+
# Stage 7: Profile Logs - uses query_profile_logs directory to match restore script expectations
477493
process_stage "Fetch Profile Logs" \
478494
"COPY INTO @a5c7667401c0c728c2ef9703bdaea66d9ae2d906 FROM (SELECT * FROM system_history.profile_history WHERE $time_condition);" \
479-
"query_profile_logs" "5" "6"
495+
"query_profile_logs" "7" "8"
480496
total=$((total + stage_total))
481497
success=$((success + stage_success))
482498

483-
# Stage 6: Create Archive
499+
# Stage 8: Create Archive
484500
echo ""
485-
echo "[6/6] Creating archive and cleanup"
501+
echo "[8/8] Creating archive and cleanup"
486502

487503
echo ""
488504
echo "Summary:"
@@ -499,7 +515,7 @@ main() {
499515
fi
500516

501517
if [[ $success -gt 0 ]]; then
502-
if create_archive "$OUTPUT_DIR/columns" "$OUTPUT_DIR/user_functions" "$OUTPUT_DIR/query_logs" "$OUTPUT_DIR/query_raw_logs" "$OUTPUT_DIR/query_profile_logs"; then
518+
if create_archive "$OUTPUT_DIR/settings" "$OUTPUT_DIR/columns" "$OUTPUT_DIR/user_functions" "$OUTPUT_DIR/query_logs" "$OUTPUT_DIR/query_raw_logs" "$OUTPUT_DIR/login_history" "$OUTPUT_DIR/query_profile_logs"; then
503519
local end_time
504520
end_time=$(date +%s)
505521
local total_time

‎scripts/selfhost/restore_logs.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ log "Extracted ${EXTRACTED_FILES} files from ${LOCAL_TAR_FILE}"
416416

417417
# Step 4: Detect path prefix
418418
log_step "4" "6" "Analyzing directory structure for path prefix"
419-
TARGET_DIRS=("columns" "user_functions" "query_raw_logs" "query_logs" "query_profile_logs")
419+
TARGET_DIRS=("settings" "columns" "user_functions" "query_raw_logs" "query_logs" "login_history" "query_profile_logs")
420420
PREFIX=""
421421

422422
for target_dir in "${TARGET_DIRS[@]}"; do
@@ -494,10 +494,12 @@ log "Created database: ${RESTORE_DATABASE}"
494494

495495
# Restore tables
496496
declare -A TABLE_MAP=(
497+
["settings"]="system.settings:settings"
497498
["columns"]="system.columns:columns"
498499
["user_functions"]="system.user_functions:user_functions"
499500
["log_history"]="system_history.log_history:query_raw_logs"
500501
["query_history"]="system_history.query_history:query_logs"
502+
["login_history"]="system_history.login_history:login_history"
501503
["profile_history"]="system_history.profile_history:query_profile_logs"
502504
)
503505

@@ -515,4 +517,4 @@ done
515517

516518
log "Log restoration completed successfully"
517519
log "Restored database: ${RESTORE_DATABASE}"
518-
log "Tables available: columns, user_functions, log_history, query_history, profile_history"
520+
log "Tables available: settings, columns, user_functions, log_history, query_history, login_history, profile_history"

‎scripts/selfhost/upload_log.sh‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
#!/bin/bash
2+
3+
# Simple logging
4+
log() {
5+
echo "[$(date '+%H:%M:%S')] $1"
6+
}
7+
8+
log_error() {
9+
echo "[$(date '+%H:%M:%S')] ERROR: $1" >&2
10+
}
11+
12+
# Show help information
13+
show_help() {
14+
cat <<EOF
15+
Usage: $0 --stage STAGE [OPTIONS] [TAR_FILE]
16+
17+
Upload Databend log archive to a stage.
18+
19+
OPTIONS:
20+
--dsn DSN Database connection string (overrides BENDSQL_DSN env var)
21+
--stage STAGE Target stage name (required)
22+
--file TAR_FILE Specific tar.gz file to upload (optional if provided as argument)
23+
-h, --help Show this help message
24+
25+
ARGUMENTS:
26+
TAR_FILE Path to tar.gz file to upload
27+
28+
ENVIRONMENT VARIABLES:
29+
BENDSQL_DSN Default database connection string
30+
31+
EXAMPLES:
32+
export BENDSQL_DSN="http://username:password@localhost:8000/database"
33+
34+
# Upload log archive to existing stage
35+
$0 --stage log_stage data_2025-08-23.tar.gz
36+
37+
# Using --file option
38+
$0 --stage log_stage --file data_2025-08-23.tar.gz
39+
40+
NOTE:
41+
- Stage name should not include @ prefix
42+
- Stage must already exist
43+
- File will be uploaded with its original filename
44+
EOF
45+
}
46+
47+
# Parse arguments
48+
STAGE=""
49+
DSN=""
50+
TAR_FILE=""
51+
52+
while [[ $# -gt 0 ]]; do
53+
case "$1" in
54+
-h | --help)
55+
show_help
56+
exit 0
57+
;;
58+
--dsn)
59+
DSN="$2"
60+
shift 2
61+
;;
62+
--stage)
63+
STAGE="$2"
64+
shift 2
65+
;;
66+
--file)
67+
TAR_FILE="$2"
68+
shift 2
69+
;;
70+
*)
71+
if [[ -z "$TAR_FILE" && "$1" =~ \.(tar\.gz|tgz)$ ]]; then
72+
TAR_FILE="$1"
73+
shift
74+
else
75+
log_error "Unknown parameter: $1"
76+
echo "Use -h or --help for usage information." >&2
77+
exit 1
78+
fi
79+
;;
80+
esac
81+
done
82+
83+
# Validate parameters
84+
if [[ -z "$STAGE" ]]; then
85+
log_error "Missing required parameter: --stage"
86+
echo "Use -h or --help for usage information." >&2
87+
exit 1
88+
fi
89+
90+
if [[ -z "$TAR_FILE" ]]; then
91+
log_error "Missing tar.gz file to upload"
92+
echo "Use -h or --help for usage information." >&2
93+
exit 1
94+
fi
95+
96+
# Validate DSN
97+
if [[ -z "$DSN" ]]; then
98+
DSN="$BENDSQL_DSN"
99+
if [[ -z "$DSN" ]]; then
100+
log_error "DSN not provided and BENDSQL_DSN not set"
101+
echo "Use -h or --help for usage information." >&2
102+
exit 1
103+
fi
104+
fi
105+
106+
# Check if file exists
107+
if [[ ! -f "$TAR_FILE" ]]; then
108+
log_error "File not found: $TAR_FILE"
109+
exit 1
110+
fi
111+
112+
# Get file info
113+
FILENAME=$(basename "$TAR_FILE")
114+
FILE_SIZE=$(du -h "$TAR_FILE" | cut -f1)
115+
116+
log "Starting upload of $FILENAME (${FILE_SIZE}) to stage @${STAGE}"
117+
118+
# Generate upload URL
119+
log "Generating presigned upload URL for @${STAGE}/${FILENAME}"
120+
UPLOAD_SQL="PRESIGN UPLOAD @${STAGE}/${FILENAME}"
121+
UPLOAD_URL=$(bendsql --dsn "${DSN}" --query="${UPLOAD_SQL}" | awk '{print $3}')
122+
123+
if [[ -z "$UPLOAD_URL" ]]; then
124+
log_error "Failed to generate upload URL for ${FILENAME}"
125+
log_error "Please check if stage @${STAGE} exists and you have upload permissions"
126+
exit 1
127+
fi
128+
129+
log "Upload URL generated successfully"
130+
131+
# Upload file
132+
log "Uploading ${FILENAME} to @${STAGE}..."
133+
if curl -s -X PUT -T "${TAR_FILE}" "${UPLOAD_URL}"; then
134+
log "Upload completed successfully"
135+
else
136+
log_error "Upload failed"
137+
exit 1
138+
fi
139+
140+
# Verify upload
141+
log "Verifying upload..."
142+
VERIFY_RESULT=$(bendsql --dsn "${DSN}" --query="LIST @${STAGE};" 2>/dev/null | grep "$FILENAME")
143+
144+
if [[ -n "$VERIFY_RESULT" ]]; then
145+
# Extract size from list output (second column)
146+
UPLOADED_SIZE=$(echo "$VERIFY_RESULT" | awk '{print $2}')
147+
log "✓ Upload verified: ${FILENAME} (${UPLOADED_SIZE} bytes) in stage @${STAGE}"
148+
else
149+
log_error "Upload verification failed - file not found in stage"
150+
exit 1
151+
fi
152+
153+
log "Upload operation completed successfully"
154+
log "File: ${FILENAME} -> @${STAGE}/${FILENAME}"

0 commit comments

Comments
 (0)