test nrd

Author: Azathothas

.github/workflows/sync.yaml

@@ -48,6 +48,12 @@ jobs:
             script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/sync_trickest_hf.sh"
             merge-script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/merge_trickest_hf.sh"
 
+          - runner: "ubuntu-latest"
+            source: "nrd"
+            dest: "ghcr-hf"
+            script: "https://raw.githubusercontent.com/pkgforge-security/domains/refs/heads/main/scripts/sync_nrd_ghcr_hf.sh"
+            merge-script: "N/A"
+
           - runner: "ubuntu-latest"
             source: "sni-ip-ranges"
             dest: "ghcr"

scripts/merge_certstream_ghcr_hf.sh

@@ -307,7 +307,7 @@ pushd "${TMPDIR}" &>/dev/null
   echo -e "[+] Cleaning up & Merging ${HF_REPO_LOCAL}/DATA/certstream/all_latest.txt ==> ${HF_REPO_LOCAL}/DATA/certstream/latest.txt"
   sort --version-sort --unique "${HF_REPO_LOCAL}/DATA/certstream/all_latest.txt" --output "${HF_REPO_LOCAL}/DATA/certstream/all_latest.txt"
   du -sh "${HF_REPO_LOCAL}/DATA/certstream/all_latest.txt"
-  cat "${HF_REPO_LOCAL}/DATA/certstream/all_latest.txt" | tr -s '[:space:]' '\n' > "${HF_REPO_LOCAL}/DATA/certstream/latest.txt"
+  cat "${HF_REPO_LOCAL}/DATA/certstream/all_latest.txt" | sed '/^[[:space:]]*#/d' | tr -s '[:space:]' '\n' > "${HF_REPO_LOCAL}/DATA/certstream/latest.txt"
   echo -e "[+] Filtering ..."
    cleanup_domains "${HF_REPO_LOCAL}/DATA/certstream/latest.txt"
    sort --version-sort --unique "${HF_REPO_LOCAL}/DATA/certstream/latest.txt" --output "${HF_REPO_LOCAL}/DATA/certstream/latest.txt"
@@ -343,7 +343,7 @@ pushd "${TMPDIR}" &>/dev/null
   echo -e "[+] Cleaning up & Merging ${HF_REPO_LOCAL}/DATA/certstream/all_weekly.txt ==> ${HF_REPO_LOCAL}/DATA/certstream/weekly.txt"
   sort --version-sort --unique "${HF_REPO_LOCAL}/DATA/certstream/all_weekly.txt" --output "${HF_REPO_LOCAL}/DATA/certstream/all_weekly.txt"
   du -sh "${HF_REPO_LOCAL}/DATA/certstream/all_weekly.txt"
-  cat "${HF_REPO_LOCAL}/DATA/certstream/all_weekly.txt" | tr -s '[:space:]' '\n' > "${HF_REPO_LOCAL}/DATA/certstream/weekly.txt"
+  cat "${HF_REPO_LOCAL}/DATA/certstream/all_weekly.txt" | sed '/^[[:space:]]*#/d' | tr -s '[:space:]' '\n' > "${HF_REPO_LOCAL}/DATA/certstream/weekly.txt"
   echo -e "[+] Filtering ..."
    cleanup_domains "${HF_REPO_LOCAL}/DATA/certstream/weekly.txt"
    sort --version-sort --unique "${HF_REPO_LOCAL}/DATA/certstream/weekly.txt" --output "${HF_REPO_LOCAL}/DATA/certstream/weekly.txt"

scripts/merge_sni-ip-ranges_ghcr.sh

@@ -185,7 +185,7 @@ pushd "${TMPDIR}" &>/dev/null
  if [[ -s "${ORAS_LOCAL}/DATA/sni-ip-ranges/all.txt" && $(stat -c%s "${ORAS_LOCAL}/DATA/sni-ip-ranges/all.txt") -gt 100000 ]]; then
   echo -e "[+] Cleaning up & Merging ${ORAS_LOCAL}/DATA/sni-ip-ranges/all.txt ==> ${ORAS_LOCAL}/DATA/sni-ip-ranges/domains.txt"
   sort --version-sort --unique "${ORAS_LOCAL}/DATA/sni-ip-ranges/all.txt" --output "${ORAS_LOCAL}/DATA/sni-ip-ranges/all.txt"
-  cat "${ORAS_LOCAL}/DATA/sni-ip-ranges/all.txt" |\
+  cat "${ORAS_LOCAL}/DATA/sni-ip-ranges/all.txt" | sed '/^[[:space:]]*#/d' |\
     awk -F '[[:space:]]*--[[:space:]]*\\[|\\]' '{print $2}' | tr -s '[:space:]' '\n' |\
     sed -E '/^[[:space:]]*$/d; s/^[[:space:]]*\*\.?[[:space:]]*//; s/[A-Z]/\L&/g' |\
     sed -E '/([0-9].*){40}/d; s/^[[:space:]]*//; s/[[:space:]]*$//; s/[${}%]//g' | sed 's/[()]//g' |\

scripts/merge_sni-ip-ranges_hf.sh

@@ -174,7 +174,7 @@ pushd "${TMPDIR}" &>/dev/null
  if [[ -s "${HF_REPO_LOCAL}/DATA/sni-ip-ranges/all.txt" && $(stat -c%s "${HF_REPO_LOCAL}/DATA/sni-ip-ranges/all.txt") -gt 100000 ]]; then
   echo -e "[+] Cleaning up & Merging ${HF_REPO_LOCAL}/DATA/sni-ip-ranges/all.txt ==> ${HF_REPO_LOCAL}/DATA/sni-ip-ranges/domains.txt"
   sort --version-sort --unique "${HF_REPO_LOCAL}/DATA/sni-ip-ranges/all.txt" --output "${HF_REPO_LOCAL}/DATA/sni-ip-ranges/all.txt"
-  cat "${HF_REPO_LOCAL}/DATA/sni-ip-ranges/all.txt" |\
+  cat "${HF_REPO_LOCAL}/DATA/sni-ip-ranges/all.txt" | sed '/^[[:space:]]*#/d' |\
     awk -F '[[:space:]]*--[[:space:]]*\\[|\\]' '{print $2}' | tr -s '[:space:]' '\n' |\
     sed -E '/^[[:space:]]*$/d; s/^[[:space:]]*\*\.?[[:space:]]*//; s/[A-Z]/\L&/g' |\
     sed -E '/([0-9].*){40}/d; s/^[[:space:]]*//; s/[[:space:]]*$//; s/[${}%]//g' | sed 's/[()]//g' |\