ytoxt

Author: ipitio

src/lib/owner.sh

@@ -123,13 +123,13 @@ update_owner() {
         echo "Creating $owner array..."
         find "$BKG_INDEX_DIR/$owner" -type f -name '*.json' ! -name '.*' -print0 | xargs -0 jq -cs '[.] | add' >"$BKG_INDEX_DIR/$owner/.json.tmp"
         jq -cs '{ ("package"): . }' "$BKG_INDEX_DIR/$owner/.json.tmp" >"$BKG_INDEX_DIR/$owner/.json"
-        ytox "$BKG_INDEX_DIR/$owner/.json"
+        ytoxt "$BKG_INDEX_DIR/$owner/.json"
         mv -f "$BKG_INDEX_DIR/$owner/.json.tmp" "$BKG_INDEX_DIR/$owner/.json"
 
         echo "Creating $owner repo arrays..."
         parallel "jq -c --arg repo {} '[.[] | select(.repo == \$repo)]' \"$BKG_INDEX_DIR/$owner/.json\" > \"$BKG_INDEX_DIR/$owner/{}/.json.tmp\"" <<<"$owner_repos"
         xargs -I {} bash -c "jq -cs '{ (\"package\"): . }' \"$BKG_INDEX_DIR/$owner/{}/.json.tmp\" > \"$BKG_INDEX_DIR/$owner/{}/.json\"" <<<"$owner_repos"
-        xargs -I {} bash -c "ytox \"$BKG_INDEX_DIR/$owner/{}/.json\"" <<<"$owner_repos"
+        xargs -I {} bash -c "ytoxt \"$BKG_INDEX_DIR/$owner/{}/.json\"" <<<"$owner_repos"
         xargs -I {} mv -f "$BKG_INDEX_DIR/$owner/{}/.json.tmp" "$BKG_INDEX_DIR/$owner/{}/.json" <<<"$owner_repos"
     fi
 

src/lib/package.sh

@@ -230,15 +230,7 @@ update_package() {
     [[ ! -f "$json_file".abs || ! -s "$json_file".abs ]] || jq -c --arg newest "$version_newest_id" --arg latest "$latest_version" '.version |= map(if .id == ($newest | tonumber) then .newest = true else . end | if .id == ($latest | tonumber) then .latest = true else . end)' "$json_file".abs >"$json_file".rel
     [[ ! -f "$json_file".rel || ! -s "$json_file".rel ]] || mv "$json_file".rel "$json_file".abs
     [[ ! -f "$json_file".abs || ! -s "$json_file".abs ]] || mv "$json_file".abs "$json_file"
-
-    # if the json is over 50MB, remove oldest versions from the packages with the most versions
-    while [ -f "$json_file" ] && [ "$(stat -c %s "$json_file")" -ge 50000000 ]; do
-        jq -e 'map(.version | length > 0) | any' "$json_file" || break
-        jq -c 'sort_by(.versions | tonumber) | reverse | map(select(.versions > 0)) | map(.version |= sort_by(.id | tonumber) | del(.version[0]))' "$json_file" >"$json_file".tmp
-        mv "$json_file".tmp "$json_file"
-    done
-
-    ytox "$json_file"
+	ytoxt "$json_file"
     rm -rf "$BKG_INDEX_DIR/$owner/$repo/$package.d"
     echo "Refreshed $owner/$package"
 }

src/lib/util.sh

@@ -435,7 +435,45 @@ get_membership() {
 }
 
 ytox() {
-    echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?><xml>$(yq -ox -I0 "$1" | sed 's/"/\\"/g')</xml>" >"${1%.*}.xml"
+    echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?><xml>$(yq -ox -I0 "$1" | sed 's/"/\\"/g')</xml>" >"${1%.*}.xml" 2>/dev/null
+    stat -c %s "${1%.*}.xml" || echo -1
+}
+
+ytoxt() {
+    # ytox + trim if the json or xml is over 50MB, remove oldest versions
+    while [ -f "$1" ] && [[ "$(ytox "$1")" -ge 50000000 || "$(stat -c %s "$1")" -ge 50000000 ]]; do
+        jq -e '
+            if type == "array" then
+                reduce .[] as $pkg (false; . or (($pkg.version // []) | length > 0))
+            else
+                (.version // []) | length > 0
+            end
+        ' "$1" || break
+        jq -c '
+            def trim_versions:
+                if (.version // []) | length > 0 then
+                    .version |= (
+                        sort_by(
+                            .id | (if type == "string" and test("^-?[0-9]+$") then tonumber else . end)
+                        )
+                        | del(.[0])
+                    )
+                else
+                    .
+                end;
+            def version_count:
+                (if has("raw_versions") then .raw_versions else (.versions // 0) end)
+                | (if type == "string" then (if test("^-?[0-9]+$") then tonumber else 0 end) else . end);
+            if type == "array" then
+                sort_by(version_count) | reverse
+                | map(select((.version // []) | length > 0))
+                | map(trim_versions)
+            else
+                trim_versions
+            end
+        ' "$1" >"$1".tmp
+        mv "$1".tmp "$1"
+    done
 }
 
 ytoy() {