don't restart too early

Author: ipitio

src/bkg.sh

@@ -160,29 +160,20 @@ main() {
                     awk -F'|' '{print $2}' packages_to_update
                 )" | sort -u >all_owners_tu
 
-                grep -Fxf all_owners_tu "$connections" >connections_tu
-
-                if [ ! -s connections_tu ]; then
-                    set_BKG BKG_BATCH_FIRST_STARTED "$today"
-                    rm -f packages_to_update
-                    \cp packages_all packages_to_update
-                    : >packages_already_updated
-                fi
-
                 echo "$(
                     echo "0/$GITHUB_OWNER"
 
                     # new connections
                     cat "$temp_connections"
 
                     # connections that have to be updated
-                    cat connections_tu
+                    grep -Fxf all_owners_tu "$connections"
 
                     # requests
                     cat "$BKG_OWNERS"
                 )" >"$BKG_OWNERS"
 
-                rm -f all_owners_in_db all_owners_tu connections_tu
+                rm -f all_owners_in_db all_owners_tu
                 clean_owners "$BKG_OWNERS"
                 head -n $(($(wc -l <"$connections") + 2)) "$BKG_OWNERS" | env_parallel --lb save_owner
                 awk -F'|' '{print $1"/"$2}' packages_to_update | sort -uR 2>/dev/null | head -n1000 | env_parallel --lb save_owner

src/lib/owner.sh

@@ -27,7 +27,9 @@ request_owner() {
         return_code=2
     elif $paging && [ -n "$id" ]; then
         echo "Requested $owner"
-        set_BKG BKG_LAST_SCANNED_ID "$id"
+        local last_id
+        last_id=$(get_BKG BKG_LAST_SCANNED_ID)
+        (( id <= last_id )) || set_BKG BKG_LAST_SCANNED_ID "$id"
     fi
 
     rm -f "$BKG_OWNERS.lock"
@@ -49,8 +51,10 @@ page_owner() {
 
     if [ -n "$GITHUB_TOKEN" ]; then
         echo "Checking owners page $1..."
-        users_more=$(query_api "users?per_page=100&page=$1&since=$(get_BKG BKG_LAST_SCANNED_ID)")
-        orgs_more=$(query_api "organizations?per_page=100&page=$1&since=$(get_BKG BKG_LAST_SCANNED_ID)")
+        local last_id
+        last_id=$(get_BKG BKG_LAST_SCANNED_ID)
+        users_more=$(query_api "users?per_page=100&page=$1&since=$last_id")
+        orgs_more=$(query_api "organizations?per_page=100&page=$1&since=$last_id")
         owners_more=$(jq --argjson users "$users_more" --argjson orgs "$orgs_more" -n '$users + $orgs | unique_by(.login)')
     fi