Skip to content

Commit 1c08018

Browse files
ipitioipitio
committed
simplify logic
1 parent e054bda commit 1c08018

File tree

2 files changed

+156
-193
lines changed

2 files changed

+156
-193
lines changed

‎src/lib/owner.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ update_owner() {
148148
mv -f "$tmp" "$dst"
149149
fi
150150
' ::: $owner_repos
151-
152151
fi
153152

154153
sed -i '/^\(.*\/\)*'"$owner"'$/d' "$BKG_OWNERS"

‎src/lib/util.sh‎

Lines changed: 156 additions & 192 deletions
Original file line numberDiff line numberDiff line change
@@ -448,201 +448,165 @@ get_membership() {
448448
}
449449

450450
ytox() {
451-
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?><xml>$(yq -ox -I0 "$1" | sed 's/"/\\"/g')</xml>" >"${1%.*}.xml" 2>/dev/null
452-
stat -c %s "${1%.*}.xml" || echo -1
451+
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?><xml>$(yq -ox -I0 "$1" | sed 's/"/\\"/g')</xml>" >"${1%.*}.xml" 2>/dev/null
452+
stat -c %s "${1%.*}.xml" || echo -1
453453
}
454454

455455
ytoxt() {
456-
# ytox + trim: if the json or xml is over 50MB, remove oldest versions
457-
local f="$1"
458-
local tmp
459-
local del_n=1
460-
local last_xml_size=-1
461-
462-
[ -f "$f" ] || return 1
463-
464-
tmp=$(mktemp "${f}.XXXXXX") || return 1
465-
trap 'rm -f "$tmp"' RETURN
466-
467-
while [ -f "$f" ]; do
468-
local json_size
469-
local xml_size
470-
local tmp_size
471-
472-
json_size=$(stat -c %s "$f" 2>/dev/null || echo -1)
473-
474-
if [ "$json_size" -lt 50000000 ]; then
475-
# Only generate/check XML if JSON is already under limit.
476-
xml_size=$(ytox "$f" 2>/dev/null || echo -1)
477-
# If XML size can't be determined, treat it as oversized so we keep trimming.
478-
[ "$xml_size" -ge 0 ] || xml_size=50000000
479-
480-
if [ "$xml_size" -lt 50000000 ]; then
481-
break
482-
fi
483-
484-
# If XML is still too large, keep trimming, but avoid redoing work forever.
485-
if [ "$xml_size" -eq "$last_xml_size" ] && [ "$last_xml_size" -ge 0 ]; then
486-
break
487-
fi
488-
last_xml_size="$xml_size"
489-
490-
# XML still too large: increase trimming aggressiveness as well.
491-
if [ "$del_n" -lt 65536 ]; then
492-
del_n=$((del_n * 2))
493-
fi
494-
else
495-
# JSON is still too large: increase trimming aggressiveness.
496-
if [ "$json_size" -ge 50000000 ]; then
497-
if [ "$del_n" -lt 65536 ]; then
498-
del_n=$((del_n * 2))
499-
fi
500-
fi
501-
fi
502-
503-
if jq -e '
504-
if (type == "array") or (type == "object") then
505-
any(.[]; ((.version // []) | type == "array") and ((.version // []) | length > 0))
506-
else
507-
((.version // []) | type == "array") and ((.version // []) | length > 0)
508-
end
509-
' "$f" >/dev/null; then
510-
jq -c '
511-
def id_to_num:
512-
if type == "number" then .
513-
elif type == "string" then tonumber? // 0
514-
else 0 end;
515-
def vlen:
516-
(.version // []) | if type == "array" then length else 0 end;
517-
def trim_versions($n):
518-
if ((.version // []) | type == "array") and ((.version // []) | length > 0) then
519-
(
520-
.version
521-
| sort_by(.id | id_to_num)
522-
| .[$n:]
523-
) as $v
524-
| .version = $v
525-
else
526-
.
527-
end;
528-
if type == "array" then
529-
(to_entries
530-
| (max_by(.value | vlen) // empty) as $max
531-
| map(
532-
if .key == $max.key and ((.value | vlen) > 0)
533-
then (.value |= trim_versions($n))
534-
else .
535-
end
536-
)
537-
| map(.value))
538-
elif type == "object" then
539-
(to_entries
540-
| (max_by(.value | vlen) // empty) as $max
541-
| map(
542-
if .key == $max.key and ((.value | vlen) > 0)
543-
then (.value |= trim_versions($n))
544-
else .
545-
end
546-
)
547-
| from_entries)
548-
else
549-
trim_versions($n)
550-
end
551-
' --argjson n "$del_n" "$f" >"$tmp"
552-
else
553-
jq -c '
554-
if type == "array" then
555-
(
556-
def to_num:
557-
if type == "number" then .
558-
elif type == "string" then tonumber? // 0
559-
else 0 end;
560-
to_entries
561-
| (min_by([ (.value.raw_downloads // 0 | to_num), (.value.date // "") ]) // null) as $target
562-
| if $target == null then
563-
map(.value)
564-
else
565-
[ .[] | select(.key != $target.key) | .value ]
566-
end
567-
)
568-
elif type == "object" then
569-
(
570-
def to_num:
571-
if type == "number" then .
572-
elif type == "string" then tonumber? // 0
573-
else 0 end;
574-
to_entries
575-
| (min_by([ (.value.raw_downloads // 0 | to_num), (.value.date // "") ]) // null) as $target
576-
| if $target == null then
577-
from_entries
578-
else
579-
([ .[] | select(.key != $target.key) ] | from_entries)
580-
end
581-
)
582-
else
583-
.
584-
end
585-
' "$f" >"$tmp"
586-
fi
587-
588-
tmp_size=$(stat -c %s "$tmp" 2>/dev/null || echo -1)
589-
590-
# If trimming didn't reduce size, retry with more aggressive deletion instead of stalling.
591-
if [ "$json_size" -ge 0 ] && [ "$tmp_size" -ge 0 ] && [ "$tmp_size" -ge "$json_size" ]; then
592-
rm -f "$tmp"
593-
594-
if [ "$del_n" -lt 65536 ]; then
595-
del_n=$((del_n * 2))
596-
continue
597-
fi
598-
599-
# If we're already at max aggressiveness, fall back to trimming whole packages once.
600-
jq -c '
601-
if type == "array" then
602-
(
603-
def to_num:
604-
if type == "number" then .
605-
elif type == "string" then tonumber? // 0
606-
else 0 end;
607-
to_entries
608-
| (min_by([ (.value.raw_downloads // 0 | to_num), (.value.date // "") ]) // null) as $target
609-
| if $target == null then
610-
map(.value)
611-
else
612-
[ .[] | select(.key != $target.key) | .value ]
613-
end
614-
)
615-
elif type == "object" then
616-
(
617-
def to_num:
618-
if type == "number" then .
619-
elif type == "string" then tonumber? // 0
620-
else 0 end;
621-
to_entries
622-
| (min_by([ (.value.raw_downloads // 0 | to_num), (.value.date // "") ]) // null) as $target
623-
| if $target == null then
624-
from_entries
625-
else
626-
([ .[] | select(.key != $target.key) ] | from_entries)
627-
end
628-
)
629-
else
630-
.
631-
end
632-
' "$f" >"$tmp"
633-
634-
tmp_size=$(stat -c %s "$tmp" 2>/dev/null || echo -1)
635-
if [ "$json_size" -ge 0 ] && [ "$tmp_size" -ge 0 ] && [ "$tmp_size" -ge "$json_size" ]; then
636-
rm -f "$tmp"
637-
break
638-
fi
639-
fi
640-
641-
mv "$tmp" "$f"
642-
done
643-
644-
# Ensure the XML output corresponds to the final JSON.
645-
ytox "$f" >/dev/null 2>&1
456+
# ytox + trim: if the json or xml is over 50MB, remove oldest versions
457+
local f="$1"
458+
local tmp
459+
local del_n=1
460+
local last_xml_size=-1
461+
462+
[ -f "$f" ] || return 1
463+
464+
tmp=$(mktemp "${f}.XXXXXX") || return 1
465+
trap 'rm -f "$tmp"' RETURN
466+
467+
while [ -f "$f" ]; do
468+
local json_size
469+
local xml_size
470+
local tmp_size
471+
472+
json_size=$(stat -c %s "$f" 2>/dev/null || echo -1)
473+
474+
if [ "$json_size" -lt 50000000 ]; then
475+
# Only generate/check XML if JSON is already under limit.
476+
xml_size=$(ytox "$f" 2>/dev/null || echo -1)
477+
# If XML size can't be determined, treat it as oversized so we keep trimming.
478+
[ "$xml_size" -ge 0 ] || xml_size=50000000
479+
480+
if [ "$xml_size" -lt 50000000 ]; then
481+
break
482+
fi
483+
484+
# If XML is still too large, keep trimming, but avoid redoing work forever.
485+
if [ "$xml_size" -eq "$last_xml_size" ] && [ "$last_xml_size" -ge 0 ]; then
486+
break
487+
fi
488+
last_xml_size="$xml_size"
489+
490+
# XML still too large: increase trimming aggressiveness as well.
491+
if [ "$del_n" -lt 65536 ]; then
492+
del_n=$((del_n * 2))
493+
fi
494+
else
495+
# JSON is still too large: increase trimming aggressiveness.
496+
if [ "$json_size" -ge 50000000 ]; then
497+
if [ "$del_n" -lt 65536 ]; then
498+
del_n=$((del_n * 2))
499+
fi
500+
fi
501+
fi
502+
503+
if jq -e '
504+
def has_versions:
505+
if (type == "object") then
506+
if ((keys | length) == 1) and ((.[keys[0]] | type) == "array") then
507+
any(.[keys[0]][]?; ((.version // []) | type == "array") and ((.version // []) | length > 0))
508+
else
509+
((.version // []) | type == "array") and ((.version // []) | length > 0)
510+
end
511+
else
512+
false
513+
end;
514+
has_versions
515+
' "$f" >/dev/null; then
516+
jq -c '
517+
def id_to_num:
518+
if type == "number" then .
519+
elif type == "string" then tonumber? // 0
520+
else 0 end;
521+
def vlen:
522+
(.version // []) | if type == "array" then length else 0 end;
523+
def trim_versions($n):
524+
if ((.version // []) | type == "array") and ((.version // []) | length > 0) then
525+
.version |= (sort_by(.id | id_to_num) | .[$n:])
526+
else
527+
.
528+
end;
529+
530+
if ((keys | length) == 1) and ((.[keys[0]] | type) == "array") then
531+
. as $root
532+
| (.[keys[0]] | max_by(vlen) // empty) as $max
533+
| if ($max | vlen) > 0 then
534+
.[keys[0]] |= map(
535+
if . == $max then trim_versions($n) else . end
536+
)
537+
else
538+
.
539+
end
540+
else
541+
trim_versions($n)
542+
end
543+
' --argjson n "$del_n" "$f" >"$tmp"
544+
else
545+
jq -c '
546+
def to_num:
547+
if type == "number" then .
548+
elif type == "string" then tonumber? // 0
549+
else 0 end;
550+
551+
if ((keys | length) == 1) and ((.[keys[0]] | type) == "array") then
552+
. as $root
553+
| .[keys[0]] as $arr
554+
| ($arr | (min_by([ (.raw_downloads // 0 | to_num), (.date // "") ]) // empty)) as $target
555+
| if ($target | length) == 0 then
556+
.
557+
else
558+
.[keys[0]] = [ $arr[] | select(. != $target) ]
559+
end
560+
else
561+
.
562+
end
563+
' "$f" >"$tmp"
564+
fi
565+
566+
tmp_size=$(stat -c %s "$tmp" 2>/dev/null || echo -1)
567+
568+
# If trimming didn't reduce size, retry with more aggressive deletion instead of stalling.
569+
if [ "$json_size" -ge 0 ] && [ "$tmp_size" -ge 0 ] && [ "$tmp_size" -ge "$json_size" ]; then
570+
rm -f "$tmp"
571+
572+
if [ "$del_n" -lt 65536 ]; then
573+
del_n=$((del_n * 2))
574+
continue
575+
fi
576+
577+
# If we're already at max aggressiveness, fall back to trimming whole packages once.
578+
jq -c '
579+
def to_num:
580+
if type == "number" then .
581+
elif type == "string" then tonumber? // 0
582+
else 0 end;
583+
584+
if ((keys | length) == 1) and ((.[keys[0]] | type) == "array") then
585+
. as $root
586+
| .[keys[0]] as $arr
587+
| ($arr | (min_by([ (.raw_downloads // 0 | to_num), (.date // "") ]) // empty)) as $target
588+
| if ($target | length) == 0 then
589+
.
590+
else
591+
.[keys[0]] = [ $arr[] | select(. != $target) ]
592+
end
593+
else
594+
.
595+
end
596+
' "$f" >"$tmp"
597+
598+
tmp_size=$(stat -c %s "$tmp" 2>/dev/null || echo -1)
599+
if [ "$json_size" -ge 0 ] && [ "$tmp_size" -ge 0 ] && [ "$tmp_size" -ge "$json_size" ]; then
600+
rm -f "$tmp"
601+
break
602+
fi
603+
fi
604+
605+
mv "$tmp" "$f"
606+
done
607+
608+
# Ensure the XML output corresponds to the final JSON.
609+
ytox "$f" >/dev/null 2>&1
646610

647611
# If either JSON or XML is > 100MB, empty each one that is too large:
648612
[ "$(stat -c %s "$f" 2>/dev/null || echo -1)" -lt 100000000 ] || echo "{}" >"$f"

0 commit comments

Comments
 (0)