Merge remote-tracking branch 'upstream/master'

Author: ipitio

.github/workflows/manual.yml

@@ -39,6 +39,15 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Tune dpkg
+        uses: abbbi/github-actions-tune@v1
+
+      - name: Set dpkg
+        run: sudo rm -f /usr/local/bin/dpkg
+
+      - name: Tune apt
+        uses: firebuild/apt-eatmydata-action@v1
+
       - name: Check for release
         id: release
         continue-on-error: true
@@ -48,6 +57,7 @@ jobs:
 
       - name: Fetch all data
         if: ${{ steps.release.outcome == 'success' }}
+        continue-on-error: true
         uses: robinraju/release-downloader@v1
         with:
           latest: true
@@ -60,11 +70,11 @@ jobs:
         run: |
           docker run -v $PWD:/app --env-file <(env | grep GITHUB) \
             ghcr.io/$GITHUB_OWNER/$GITHUB_REPO:master \
-            bash src/test/update.sh; echo "updated=$?" >> "$GITHUB_OUTPUT"
+            src/test/update.sh; echo "updated=$?" >> "$GITHUB_OUTPUT"
 
       - name: Get date
         id: date
-        run: echo "tag=$(date -u +%-Y.%-m).$((($(date -u +%-d)-1)/7))" >> "$GITHUB_OUTPUT"
+        run: echo "tag=$(date -u +%-Y.%-m).$((($(date -u +%-d)-1)/14))" >> "$GITHUB_OUTPUT"
 
       - name: Save database
         if: ${{ steps.update.outputs.updated == '0' }}

.github/workflows/publish.yml

@@ -1,6 +1,8 @@
 name: build
 
 on:
+  workflow_dispatch:
+
   push:
     branches:
       - master

.github/workflows/update.yml

@@ -31,6 +31,15 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Tune dpkg
+        uses: abbbi/github-actions-tune@v1
+
+      - name: Set dpkg
+        run: sudo rm -f /usr/local/bin/dpkg
+
+      - name: Tune apt
+        uses: firebuild/apt-eatmydata-action@v1
+
       - name: Check for release
         id: release
         continue-on-error: true
@@ -40,6 +49,7 @@ jobs:
 
       - name: Fetch all data
         if: ${{ steps.release.outcome == 'success' }}
+        continue-on-error: true
         uses: robinraju/release-downloader@v1
         with:
           latest: true
@@ -52,11 +62,11 @@ jobs:
         run: |
           docker run -v $PWD:/app --env-file <(env | grep GITHUB) \
             ghcr.io/ipitio/backage:master \
-            bash src/test/update.sh; echo "updated=$?" >> "$GITHUB_OUTPUT"
+            src/test/update.sh; echo "updated=$?" >> "$GITHUB_OUTPUT"
 
       - name: Get date
         id: date
-        run: echo "tag=$(date -u +%-Y.%-m).$((($(date -u +%-d)-1)/7))" >> "$GITHUB_OUTPUT"
+        run: echo "tag=$(date -u +%-Y.%-m).$((($(date -u +%-d)-1)/14))" >> "$GITHUB_OUTPUT"
 
       - name: Save database
         if: ${{ steps.update.outputs.updated == '0' }}

Dockerfile

@@ -1,4 +1,4 @@
 FROM ubuntu:22.04
 WORKDIR /app
-COPY src src
+COPY . .
 RUN cd src && bash bkg.sh

README.md

@@ -20,15 +20,84 @@ If this is [`ipitio/backage`](https://github.com/ipitio/backage), all you have t
 
 Otherwise, if this is a fork, you'd prefer an alternative method, or your packages weren't added to the [index](https://github.com/pkgforge-dev/backage/tree/index) after a day, enter the case-sensitive name of each missing user or organization on a new line at the top of `owners.txt` [here](https://github.com/pkgforge-dev/backage/edit/master/owners.txt) and make a pull request. Please submit just the name(s) -- ids, repos, and packages will be found automatically!
 
-New packages won't be added until *all* existing ones are refreshed; you should also create an independent instance that'll update faster and more frequently, up to hourly. Simply fork just the `master` branch, enable Actions from its tab and all disabled workflows, and use the [Alternative URL](#alternative-url) when it changes. Your own packages will be picked up automatically! If you need to edit `owners.txt`, do so after the first run. This centralized repo will then serve as a backup for all subsets of packages not in `optout.txt`.
+Otherwise, if this is a fork, you'd prefer an alternative method, or your packages weren't added to the [index](https://github.com/ipitio/backage/tree/index) after a day, enter the case-sensitive name of each missing user or organization on a new line at the top of `owners.txt` [here](https://github.com/ipitio/backage/edit/master/owners.txt) and make a pull request. Don't worry -- while my Contribution Graph is an uptime monitor of sorts, yours won't be. See the top of `bkg.sh` for details about available options, which must come last when passed to `update.sh`, as shown in `Self-Host`.
+
+> [!TIP]
+> You only need to add the name(s), IDs are fetched as needed.
+
+New packages won't be added until *all* existing ones are refreshed; you should also create an independent instance that'll update faster and more frequently. Simply fork just the `master` branch, choose one of the following options, and use the [Alternative URL](#alternative-url) when it changes. This centralized repo will then serve as a backup for all subsets of packages not in `optout.txt`.
+
+> [!IMPORTANT]
+> Your own packages will be picked up automatically! If you need to edit `owners.txt`, do so after the first run.
+
+<details>
+<summary>With Actions</summary>
+
+This will use a lot of minutes on GitHub-hosted runners, so you may want to use your own.
+
+1. Enable Actions from its tab
+2. Enable all disabled workflows
+
+</details>
+
+<details>
+<summary>Self-Host</summary>
+
+This is an example for `systemd`; adapt it to your needs. Please note:
+
+- Docker needs to be installed
+- You don't need to set `GITHUB_TOKEN` if you're logged in with `gh` or you'll first use your PAT to run (replace `*` with `https` or `ssh`):
+
+```bash
+git clone --depth=1 -b master --single-branch *://<PAT>@github.com/ipitio/backage /opt/backage/master
+```
+
+- `-m 0` ensures only the public packages of the owners you've added are updated (default)
+  - You'll need the proper permissions to update private packages
+- `-d 0` allows everything to be updated in one go
+  - A graceful restart is initiated every 4.5 hours by default
+
+```bash
+echo "[Unit]
+Description=Run Backage
+After=network.target
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+Restart=always
+RestartSec=5
+ExecStart=/usr/bin/sh -c '                   \\
+  GITHUB_TOKEN=<PAT>                        ;\\
+  GITHUB_OWNER=<username>                   ;\\
+  GITHUB_REPO=backage                       ;\\
+  GITHUB_BRANCH=master                      ;\\
+  BKG_PATH=\$GITHUB_REPO/\$GITHUB_BRANCH    ;\\
+  mkdir -p /opt/\$BKG_PATH                  ;\\
+  docker run -v /opt/\$BKG_PATH:/app         \\
+    --env-file <(env | grep GITHUB)          \\
+    ghcr.io/\$GITHUB_OWNER/\${BKG_PATH////:} \\
+    src/test/update.sh -m 0 -d 0'
+
+[Install]
+WantedBy=multi-user.target
+" | sudo tee /etc/systemd/system/bkg.service
+sudo systemctl daemon-reload
+sudo systemctl enable --now bkg
+```
+
+</details>
 
 ## The Endpoint
 
 ```py
 https://ipitio.github.io/backage/OWNER/[REPO/[PACKAGE]].FORMAT
 ```
 
-Once the packages you're interested in have been added, replace the parameters with their respective values, scoping to your parsing needs, then access the latest data however you want. The format can be either `json` or `xml`. Use something like [shields.io/json](https://shields.io/badges/dynamic-json-badge) or [shields.io/xml](https://shields.io/badges/dynamic-xml-badge) to make badges like [this one](https://github.com/badges/shields/issues/5594#issuecomment-2157626147); you'll need the latter to evaluate expressions, like filters ([issue](https://github.com/ipitio/backage/issues/23)).
+Once the packages you're interested in have been added, replace the parameters with their respective values, scoping to your parsing needs, then access the latest data however you want. The format can be either `json` or `xml`.
+
+> [!NOTE]
+> Use something like [shields.io/json](https://shields.io/badges/dynamic-json-badge) or [shields.io/xml](https://shields.io/badges/dynamic-xml-badge) to make badges like [this one](https://github.com/badges/shields/issues/5594#issuecomment-2157626147). You'll need the latter to evaluate expressions, like filters ([issue](https://github.com/ipitio/backage/issues/23)).
 
 ### Available Properties
 
@@ -164,7 +233,7 @@ As can packages in `owner[/repo]/.xml` files:
 https://github.com/pkgforge-dev/backage/raw/index/OWNER/[REPO/[PACKAGE]].FORMAT
 ```
 
-The endpoint is also available here!
+The endpoint is also available here! This will change to your fork once it updates.
 
 ## JSON2XML Proxy
 

src/bkg.sh

@@ -1,15 +1,19 @@
 #!/bin/bash
-# shellcheck disable=SC1091,SC2015
+# shellcheck disable=SC1091,SC2015,SC2034
 #
 # Modes:
-# 0 - Update all public (default)
-# 1 - Update own public
-# 2 - Just clean up
-# 3 - Update all public and own private
-# 4 - Update own public and private
-# 5 - Update own private
+# 0 - Update all public pkgs (default)
+# 1 - Update own public pkgs
+# 2 - Just clean the dir
+# 3 - Update all public and own private pkgs
+# 4 - Update own public and private pkgs
+# 5 - Update own private pkgs
 #
-# Usage: source bkg.sh && main [-m <mode>]
+# Duration:
+#  < 0 - Unlimited
+# >= 0 - Run for this many seconds
+#
+# Usage: source bkg.sh && main [-d <duration>] [-m <mode>]
 
 source lib/owner.sh
 
@@ -25,11 +29,15 @@ main() {
     local return_code=0
     local opted_out
     local opted_out_before
+    local len_conn
     connections=$(mktemp) || exit 1
     temp_connections=$(mktemp) || exit 1
 
-    while getopts "m:" flag; do
+    while getopts "d:m:" flag; do
         case ${flag} in
+        d)
+            BKG_MAX_LEN=$((OPTARG))
+            ;;
         m)
             BKG_MODE=$((OPTARG))
             ;;
@@ -120,14 +128,13 @@ main() {
                     cat "$temp_connections" >>"$connections"
 
                     sed -i 's/^[[:space:]]*//;s/[[:space:]]*$//; /^$/d; /^0\/$/d' "$connections"
-                    [[ "$(wc -l <"$BKG_OWNERS")" -ge $(($(sort -u "$connections" | wc -l) + 100)) ]] || seq 1 2 | env_parallel --lb --halt soon,fail=1 page_owner
+                    # shellcheck disable=SC2319
+                    BKG_PAGE_ALL=$( (($(wc -l <"$BKG_OWNERS") < $(($(sort -u "$connections" | wc -l) + 100)))); echo "$?")
+                    seq 1 2 | env_parallel --lb --halt soon,fail=1 page_owner
                 else
                     get_membership "$GITHUB_OWNER" >"$connections"
-                    curl "https://raw.githubusercontent.com/ipitio/backage/refs/heads/$GITHUB_BRANCH/optout.txt" > base_out
-                    curl "https://raw.githubusercontent.com/ipitio/backage/refs/heads/$GITHUB_BRANCH/owners.txt" > base_own
-                    ! diff -q "$BKG_OWNERS" base_own || : > "$BKG_OWNERS"
-                    ! diff -q "$BKG_OPTOUT" base_out || : > "$BKG_OPTOUT"
-                    rm -f base_out base_own
+                    [ "$BKG_IS_FIRST" = "false" ] || : > "$BKG_OWNERS"
+                    [ "$BKG_IS_FIRST" = "false" ] || : > "$BKG_OPTOUT"
                 fi
 
                 sort "$connections" | uniq -c | sort -nr | awk '{print $2}' >"$connections".bak
@@ -177,9 +184,10 @@ main() {
 
                 rm -f all_owners_in_db all_owners_tu
                 clean_owners "$BKG_OWNERS"
-                head -n $(($(wc -l <"$connections") + 2)) "$BKG_OWNERS" | env_parallel --lb save_owner
+                len_conn=$(wc -l <"$connections")
+                head -n $(( len_conn < 100 ? len_conn + 100 : len_conn * 3 / 2 )) "$BKG_OWNERS" | env_parallel --lb save_owner
                 awk -F'|' '{print $1"/"$2}' packages_to_update | sort -uR 2>/dev/null | head -n1000 | env_parallel --lb save_owner
-                parallel "sed -i '\,^{}$,d' $BKG_OWNERS" <"$connections"
+                parallel "sed -i '\,^{}$,d' $BKG_OWNERS" ::: "$(sed 's/"/\\"/g' "$connections")"
                 sed -i '/^0\//d' "$BKG_OWNERS"
                 set_BKG BKG_DIFF "$db_size_curr"
             fi

src/lib/owner.sh

@@ -62,7 +62,7 @@ page_owner() {
     jq -e '.[].login' <<<"$owners_more" &>/dev/null || return 2
     local owners_lines
     owners_lines=$(jq -r '.[] | @base64' <<<"$owners_more")
-    run_parallel request_owner "$owners_lines"
+    (( BKG_PAGE_ALL > 0 )) && request_owner "$(echo "$owners_lines" | head -n1)" || run_parallel request_owner "$owners_lines"
     echo "Checked owners page $1"
     [ "$(wc -l <<<"$owners_lines")" -gt 1 ] || return 2
 }

src/lib/package.sh

@@ -144,7 +144,7 @@ update_package() {
     fi
 
     # calculate the overall downloads and size
-    size=$(sqlite3 "$BKG_INDEX_DB" "select size from '$table_version_name' where id in (select id from '$table_version_name' order by id desc limit 1) order by date desc limit 1;")
+    size=$(sqlite3 "$BKG_INDEX_DB" "select size from '$table_version_name' where size > 1 order by id desc, date desc limit 1;")
     raw_all=$(sqlite3 "$BKG_INDEX_DB" "select sum(downloads), sum(downloads_month), sum(downloads_week), sum(downloads_day) from '$table_version_name' where date in (select date from '$table_version_name' order by date desc limit 1);")
     summed_raw_downloads=$(cut -d'|' -f1 <<<"$raw_all")
     raw_downloads_month=$(cut -d'|' -f2 <<<"$raw_all")