Skip to content

Commit a7abd8d

Browse files
abbajaj806abbajaj806
authored
Merge pull request qualcomm-linux#191 from smuppand/video-fix
functestlib: harden extract_tar_from_url for minimal builds & flaky TLS
2 parents c31b862 + 1ffc3c0 commit a7abd8d

File tree

2 files changed

+188
-63
lines changed

2 files changed

+188
-63
lines changed

‎Runner/suites/Multimedia/Audio/AudioPlayback/run.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ fi
3838
. "$TOOLS/functestlib.sh"
3939
# shellcheck disable=SC1091
4040
. "$TOOLS/audio_common.sh"
41+
# shellcheck disable=SC1091
42+
. "$TOOLS/lib_video.sh"
4143

4244
TESTNAME="AudioPlayback"
4345
RES_FILE="./${TESTNAME}.res"

‎Runner/utils/functestlib.sh‎

Lines changed: 186 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -325,52 +325,58 @@ extract_tar_from_url() {
325325
mkdir -p "$outdir" 2>/dev/null || true
326326

327327
case "$url" in
328-
/*) tarfile="$url" ;;
329-
file://*) tarfile="${url#file://}" ;;
330-
*) tarfile="$outdir/$(basename "$url")" ;;
328+
/*)
329+
tarfile="$url"
330+
;;
331+
file://*)
332+
tarfile="${url#file://}"
333+
;;
334+
*)
335+
tarfile="$outdir/$(basename "$url")"
336+
;;
331337
esac
332338
markfile="${tarfile}.extracted"
339+
skip_sentinel="${outdir}/.asset_fetch_skipped"
340+
341+
# If a previous run already marked "assets unavailable", honor it and SKIP.
342+
if [ -f "$skip_sentinel" ]; then
343+
log_info "Previous run marked assets unavailable on this system (${skip_sentinel}); skipping download."
344+
return 2
345+
fi
333346

334-
# --- small helper: decide if this archive already looks extracted in $outdir
335347
tar_already_extracted() {
336348
tf="$1"
337349
od="$2"
338-
339-
# Fast path: explicit marker from a previous successful extract.
340-
[ -f "${tf}.extracted" ] && return 0
341-
342-
# Fall back: sniff a few entries from the archive and see if they exist on disk.
343-
# (No reliance on any one filename; majority-of-first-10 rule.)
344-
# NOTE: we intentionally avoid pipes->subshell to keep counters reliable in POSIX sh.
350+
if [ -f "${tf}.extracted" ]; then
351+
return 0
352+
fi
345353
tmp_list="${od}/.tar_ls.$$"
346-
if tar -tf "$tf" 2>/dev/null | head -n 20 >"$tmp_list"; then
347-
total=0; present=0
348-
# shellcheck disable=SC2039
354+
if tar -tf "$tf" 2>/dev/null | head -n 20 > "$tmp_list"; then
355+
total=0
356+
present=0
349357
while IFS= read -r ent; do
350358
[ -z "$ent" ] && continue
351359
total=$((total + 1))
352-
# Normalize (strip trailing slash for directories)
353360
ent="${ent%/}"
354-
# Check both full relative path and basename (covers archives with nested roots)
355361
if [ -e "$od/$ent" ] || [ -e "$od/$(basename "$ent")" ]; then
356362
present=$((present + 1))
357363
fi
358364
done < "$tmp_list"
359365
rm -f "$tmp_list" 2>/dev/null || true
360-
361-
# If we have at least 3 hits or ≥50% of probed entries, assume it's already extracted.
362-
if [ "$present" -ge 3 ] || { [ "$total" -gt 0 ] && [ $((present * 100 / total)) -ge 50 ]; }; then
366+
if [ "$present" -ge 3 ]; then
367+
return 0
368+
fi
369+
if [ "$total" -gt 0 ] && [ $((present * 100 / total)) -ge 50 ]; then
363370
return 0
364371
fi
365372
fi
366373
return 1
367374
}
368375

369376
if command -v check_tar_file >/dev/null 2>&1; then
370-
# Your site-specific validator (returns 0=extracted ok, 2=exists but not extracted, 1=missing/bad)
371-
check_tar_file "$url"; status=$?
377+
check_tar_file "$url"
378+
status=$?
372379
else
373-
# Fallback: if archive exists and looks extracted -> status=0; if exists not-extracted -> 2; else 1
374380
if [ -f "$tarfile" ]; then
375381
if tar_already_extracted "$tarfile" "$outdir"; then
376382
status=0
@@ -382,20 +388,66 @@ extract_tar_from_url() {
382388
fi
383389
fi
384390

385-
ensure_reasonable_clock || { log_warn "Proceeding in limited-network mode."; limited_net=1; }
391+
ensure_reasonable_clock || {
392+
log_warn "Proceeding in limited-network mode."
393+
limited_net=1
394+
}
386395

387-
# ---- helpers ------------------------------------------------------------
388-
is_busybox_wget() { command -v wget >/dev/null 2>&1 && wget --help 2>&1 | head -n1 | grep -qi busybox; }
396+
is_busybox_wget() {
397+
if command -v wget >/dev/null 2>&1; then
398+
if wget --help 2>&1 | head -n 1 | grep -qi busybox; then
399+
return 0
400+
fi
401+
fi
402+
return 1
403+
}
404+
405+
tls_capable_fetcher_available() {
406+
scheme_https=0
407+
case "$url" in
408+
https://*)
409+
scheme_https=1
410+
;;
411+
esac
412+
if [ "$scheme_https" -eq 0 ]; then
413+
return 0
414+
fi
415+
if command -v curl >/dev/null 2>&1; then
416+
if curl -V 2>/dev/null | grep -qiE 'ssl|tls'; then
417+
return 0
418+
fi
419+
fi
420+
if command -v aria2c >/dev/null 2>&1; then
421+
return 0
422+
fi
423+
if command -v wget >/dev/null 2>&1; then
424+
if ! is_busybox_wget; then
425+
return 0
426+
fi
427+
if command -v openssl >/dev/null 2>&1; then
428+
return 0
429+
fi
430+
fi
431+
return 1
432+
}
389433

390434
try_download() {
391-
src="$1"; dst="$2"
435+
src="$1"
436+
dst="$2"
392437
part="${dst}.part.$$"
393438
ca=""
394-
for cand in /etc/ssl/certs/ca-certificates.crt /etc/ssl/cert.pem /system/etc/security/cacerts/ca-certificates.crt; do
395-
[ -r "$cand" ] && ca="$cand" && break
439+
440+
for cand in \
441+
/etc/ssl/certs/ca-certificates.crt \
442+
/etc/ssl/cert.pem \
443+
/system/etc/security/cacerts/ca-certificates.crt
444+
do
445+
if [ -r "$cand" ]; then
446+
ca="$cand"
447+
break
448+
fi
396449
done
397450

398-
# curl first (IPv4, retries, redirects)
399451
if command -v curl >/dev/null 2>&1; then
400452
if [ -n "$ca" ]; then
401453
curl -4 -L --fail --retry 3 --retry-delay 2 --connect-timeout 10 \
@@ -405,92 +457,162 @@ extract_tar_from_url() {
405457
-o "$part" "$src"
406458
fi
407459
rc=$?
408-
if [ $rc -eq 0 ]; then mv -f "$part" "$dst" 2>/dev/null || true; return 0; fi
460+
if [ $rc -eq 0 ]; then
461+
mv -f "$part" "$dst" 2>/dev/null || true
462+
return 0
463+
fi
409464
rm -f "$part" 2>/dev/null || true
410-
case "$rc" in 60|35|22) return 60 ;; esac # TLS-ish / HTTP fail
465+
case "$rc" in
466+
60|35|22)
467+
return 60
468+
;;
469+
esac
411470
fi
412471

413-
# aria2c (if available)
414472
if command -v aria2c >/dev/null 2>&1; then
415473
aria2c -x4 -s4 -m3 --connect-timeout=10 \
416474
-o "$(basename "$part")" --dir="$(dirname "$part")" "$src"
417475
rc=$?
418-
if [ $rc -eq 0 ]; then mv -f "$part" "$dst" 2>/dev/null || true; return 0; fi
476+
if [ $rc -eq 0 ]; then
477+
mv -f "$part" "$dst" 2>/dev/null || true
478+
return 0
479+
fi
419480
rm -f "$part" 2>/dev/null || true
420481
fi
421482

422-
# wget: handle BusyBox vs GNU
423483
if command -v wget >/dev/null 2>&1; then
424484
if is_busybox_wget; then
425-
# BusyBox wget has: -O, -T, --no-check-certificate (no -4, no --tries)
426-
wget -O "$part" -T 15 "$src"; rc=$?
485+
wget -O "$part" -T 15 "$src"
486+
rc=$?
427487
if [ $rc -ne 0 ]; then
428488
log_warn "BusyBox wget failed (rc=$rc); final attempt with --no-check-certificate."
429-
wget -O "$part" -T 15 --no-check-certificate "$src"; rc=$?
489+
wget -O "$part" -T 15 --no-check-certificate "$src"
490+
rc=$?
491+
fi
492+
if [ $rc -eq 0 ]; then
493+
mv -f "$part" "$dst" 2>/dev/null || true
494+
return 0
430495
fi
431-
if [ $rc -eq 0 ]; then mv -f "$part" "$dst" 2>/dev/null || true; return 0; fi
432496
rm -f "$part" 2>/dev/null || true
433497
return 60
434498
else
435-
# GNU wget: can use IPv4 and tries
436499
if [ -n "$ca" ]; then
437-
wget -4 --timeout=15 --tries=3 --ca-certificate="$ca" -O "$part" "$src"; rc=$?
500+
wget -4 --timeout=15 --tries=3 --ca-certificate="$ca" -O "$part" "$src"
501+
rc=$?
438502
else
439-
wget -4 --timeout=15 --tries=3 -O "$part" "$src"; rc=$?
503+
wget -4 --timeout=15 --tries=3 -O "$part" "$src"
504+
rc=$?
440505
fi
441506
if [ $rc -ne 0 ]; then
442507
log_warn "wget failed (rc=$rc); final attempt with --no-check-certificate."
443-
wget -4 --timeout=15 --tries=1 --no-check-certificate -O "$part" "$src"; rc=$?
508+
wget -4 --timeout=15 --tries=1 --no-check-certificate -O "$part" "$src"
509+
rc=$?
510+
fi
511+
if [ $rc -eq 0 ]; then
512+
mv -f "$part" "$dst" 2>/dev/null || true
513+
return 0
444514
fi
445-
if [ $rc -eq 0 ] ; then mv -f "$part" "$dst" 2>/dev/null || true; return 0; fi
446515
rm -f "$part" 2>/dev/null || true
447-
[ $rc -eq 5 ] && return 60
516+
if [ $rc -eq 5 ]; then
517+
return 60
518+
fi
448519
return $rc
449520
fi
450521
fi
451522

452523
return 127
453524
}
454-
# ------------------------------------------------------------------------
525+
455526
if [ "$status" -eq 0 ]; then
456527
log_info "Already extracted. Skipping download."
457528
return 0
458-
elif [ "$status" -eq 2 ]; then
529+
fi
530+
531+
if [ "$status" -eq 2 ]; then
459532
log_info "File exists and is valid, but not yet extracted. Proceeding to extract."
460533
else
461534
case "$url" in
462535
/*|file://*)
463-
if [ ! -f "$tarfile" ]; then log_fail "Local tar file not found: $tarfile"; return 1; fi
536+
if [ ! -f "$tarfile" ]; then
537+
log_fail "Local tar file not found: $tarfile"
538+
return 1
539+
fi
464540
;;
465541
*)
466-
if [ -n "$limited_net" ]; then
467-
log_warn "Limited network; cannot fetch media bundle. Will SKIP decode cases."
468-
return 2
542+
if [ ! -f "$tarfile" ] || [ ! -s "$tarfile" ]; then
543+
prestage_dirs=""
544+
if [ -n "${ASSET_DIR:-}" ]; then prestage_dirs="$prestage_dirs $ASSET_DIR"; fi
545+
if [ -n "${VIDEO_ASSET_DIR:-}" ]; then prestage_dirs="$prestage_dirs $VIDEO_ASSET_DIR"; fi
546+
if [ -n "${AUDIO_ASSET_DIR:-}" ]; then prestage_dirs="$prestage_dirs $AUDIO_ASSET_DIR"; fi
547+
prestage_dirs="$prestage_dirs . $outdir ${ROOT_DIR:-} ${ROOT_DIR:-}/cache /var/Runner /var/Runner/cache"
548+
549+
for d in $prestage_dirs; do
550+
if [ -d "$d" ] && [ -f "$d/$(basename "$tarfile")" ]; then
551+
log_info "Using pre-staged tarball: $d/$(basename "$tarfile")"
552+
cp -f "$d/$(basename "$tarfile")" "$tarfile" 2>/dev/null || true
553+
break
554+
fi
555+
done
556+
557+
if [ ! -s "$tarfile" ]; then
558+
for top in /mnt /media; do
559+
if [ -d "$top" ]; then
560+
for d in "$top"/*; do
561+
if [ -d "$d" ] && [ -f "$d/$(basename "$tarfile")" ]; then
562+
log_info "Using pre-staged tarball: $d/$(basename "$tarfile")"
563+
cp -f "$d/$(basename "$tarfile")" "$tarfile" 2>/dev/null || true
564+
break 2
565+
fi
566+
done
567+
fi
568+
done
569+
fi
469570
fi
470-
log_info "Downloading $url -> $tarfile"
471-
if ! try_download "$url" "$tarfile"; then
472-
rc=$?
473-
if [ $rc -eq 60 ]; then
474-
log_warn "TLS/handshake problem while downloading (cert/clock/firewall). Treating as limited network."
571+
572+
if [ ! -s "$tarfile" ]; then
573+
if [ -n "$limited_net" ]; then
574+
log_warn "Limited network, cannot fetch media bundle. Marking SKIP for callers."
575+
: > "$skip_sentinel" 2>/dev/null || true
475576
return 2
476577
fi
477-
log_fail "Failed to download $(basename "$url")"
478-
return 1
578+
579+
if ! tls_capable_fetcher_available; then
580+
log_warn "No TLS-capable downloader available on this minimal build, cannot fetch: $url"
581+
log_warn "Pre-stage $(basename "$url") locally or use a file:// URL."
582+
: > "$skip_sentinel" 2>/dev/null || true
583+
return 2
584+
fi
585+
586+
log_info "Downloading $url -> $tarfile"
587+
if ! try_download "$url" "$tarfile"; then
588+
rc=$?
589+
if [ $rc -eq 60 ]; then
590+
log_warn "TLS/handshake problem while downloading (cert/clock/firewall or minimal wget). Marking SKIP."
591+
: > "$skip_sentinel" 2>/dev/null || true
592+
return 2
593+
fi
594+
log_fail "Failed to download $(basename "$url")"
595+
return 1
596+
fi
479597
fi
480598
;;
481599
esac
482600
fi
483601

484602
log_info "Extracting $(basename "$tarfile")..."
485603
if tar -xvf "$tarfile"; then
486-
# If we got here, assume success; write marker for future quick skips.
487604
: > "$markfile" 2>/dev/null || true
605+
# Clear the minimal/offline sentinel only if it exists (SC2015-safe)
606+
if [ -f "$skip_sentinel" ]; then
607+
rm -f "$skip_sentinel" 2>/dev/null || true
608+
fi
488609

489-
# Best-effort sanity: verify at least one entry now exists.
490-
first_entry=$(tar -tf "$tarfile" 2>/dev/null | head -n1 | sed 's#/$##')
491-
if [ -n "$first_entry" ] && { [ -e "$first_entry" ] || [ -e "$outdir/$first_entry" ]; }; then
492-
log_pass "Files extracted successfully ($(basename "$first_entry") present)."
493-
return 0
610+
first_entry="$(tar -tf "$tarfile" 2>/dev/null | head -n 1 | sed 's#/$##')"
611+
if [ -n "$first_entry" ]; then
612+
if [ -e "$first_entry" ] || [ -e "$outdir/$first_entry" ]; then
613+
log_pass "Files extracted successfully ($(basename "$first_entry") present)."
614+
return 0
615+
fi
494616
fi
495617
log_warn "Extraction finished but couldn't verify entries. Assuming success."
496618
return 0
@@ -500,6 +622,7 @@ extract_tar_from_url() {
500622
return 1
501623
}
502624

625+
503626
# Function to check if a tar file exists
504627
check_tar_file() {
505628
url="$1"

0 commit comments

Comments
 (0)