Add caching proxy for libcdb debuginfod files to CI (#2487)

Author: peace-maker

.github/workflows/ci.yml

@@ -12,6 +12,21 @@ jobs:
             os: ubuntu-22.04
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
+    services:
+      libcdb-cache:
+        image: nginx
+        volumes:
+          - /home/runner/libcdb-cache:/var/cache/nginx
+        ports:
+          - 3000:3000  # https://debuginfod.elfutils.org proxy cache
+          - 3001:3001  # https://libc.rip/ proxy cache
+          - 3002:3002  # http://archive.ubuntu.com/ proxy cache
+          - 3003:3003  # https://gitlab.com/ proxy cache
+    env:
+      DEBUGINFOD_URLS: http://localhost:3000/
+      PWN_LIBCRIP_URL: http://localhost:3001/
+      PWN_UBUNTU_ARCHIVE_URL: http://localhost:3002/
+      PWN_GITLAB_LIBCDB_URL:  http://localhost:3003/
     steps:
     - uses: actions/checkout@v4
       with:
@@ -21,6 +36,28 @@ jobs:
       run: |
         git fetch origin
         git log --oneline --graph -10
+    
+    - name: Fix libcdb-cache permissions
+      id: fix-perms
+      run: |
+        sudo chown -R runner:runner /home/runner/libcdb-cache
+        echo "date=$(/bin/date -u "+%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT
+
+    - name: Cache for libcdb requests
+      uses: actions/cache@v4
+      with:
+        path: ~/libcdb-cache
+        key: libcdb-python${{ matrix.python_version }}-${{ steps.fix-perms.outputs.date }}
+        restore-keys: |
+          libcdb-python${{ matrix.python_version }}-
+          libcdb-
+
+    - name: Install libcdb-cache service config
+      run: |
+        sudo chown -R 101:101 /home/runner/libcdb-cache
+        container_id=$(docker ps --all --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}")
+        docker cp ./travis/libcdb_nginx_cache.conf $container_id:/etc/nginx/nginx.conf
+        docker restart $container_id
 
     - name: Install RPyC for gdb
       run: |
@@ -29,11 +66,10 @@ jobs:
         sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver
         /usr/bin/python -m pip install --break-system-packages rpyc || /usr/bin/python -m pip install rpyc
         gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)'
-
+    
     - name: Cache for pip
       uses: actions/cache@v4
       if: matrix.python_version == '2.7'
-      id: cache-pip
       with:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }}
@@ -224,6 +260,12 @@ jobs:
         name: coverage-${{ matrix.python_version }}
         path: .coverage*
         include-hidden-files: true
+    
+    - name: Fix libcdb-cache permissions
+      run: |
+        container_id=$(docker ps --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}")
+        docker stop $container_id
+        sudo chown -R runner:runner /home/runner/libcdb-cache
 
   windows-test:
     runs-on: windows-latest

pwnlib/dynelf.py

@@ -652,7 +652,7 @@ def _dynamic_load_dynelf(self, libname):
                 break
 
             if name:
-                self.status('Skipping %s' % name)
+                self.status('Skipping %r' % name)
 
             cur = leak.field(cur, LinkMap.l_next)
         else:

pwnlib/libcdb.py

@@ -72,6 +72,10 @@ def _turbofast_extract_build_id(path):
     urls = os.environ['DEBUGINFOD_URLS'].split(' ')
     DEBUGINFOD_SERVERS = urls + DEBUGINFOD_SERVERS
 
+# Allow to override url with a caching proxy in CI
+LIBC_RIP_URL = os.environ.get("PWN_LIBCRIP_URL", "https://libc.rip").rstrip("/")
+GITLAB_LIBCDB_URL = os.environ.get("PWN_GITLAB_LIBCDB_URL", "https://gitlab.com").rstrip("/")
+
 # Retry failed lookups after some time
 NEGATIVE_CACHE_EXPIRY = 60 * 60 * 24 * 7 # 1 week
 
@@ -86,7 +90,7 @@ def provider_libcdb(hex_encoded_id, search_type):
     from six.moves import urllib
 
     # Build the URL using the requested hash type
-    url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % search_type
+    url_base = "{}/libcdb/libcdb/raw/master/hashes/{}/".format(GITLAB_LIBCDB_URL, search_type)
     url      = urllib.parse.urljoin(url_base, hex_encoded_id)
 
     data     = b""
@@ -111,7 +115,7 @@ def query_libc_rip(params):
     # Deferred import because it's slow
     import requests
 
-    url = "https://libc.rip/api/find"
+    url = "{}/api/find".format(LIBC_RIP_URL)
     try:
         result = requests.post(url, json=params, timeout=20)
         result.raise_for_status()
@@ -143,6 +147,7 @@ def provider_libc_rip(search_target, search_type):
 
     url = libc_match[0]['download_url']
     log.debug("Downloading data from libc.rip: %s", url)
+    url = url.replace("https://libc.rip", LIBC_RIP_URL)
     data = wget(url, timeout=20)
 
     if not data:
@@ -529,7 +534,9 @@ def _find_libc_package_lib_url(libc):
     libc_match = query_libc_rip({'buildid': enhex(libc.buildid)})
     if libc_match is not None:
         for match in libc_match:
-            yield match['libs_url']
+            # Allow to override url with a caching proxy in CI
+            ubuntu_archive_url = os.environ.get('PWN_UBUNTU_ARCHIVE_URL', 'http://archive.ubuntu.com').rstrip('/')
+            yield match['libs_url'].replace('http://archive.ubuntu.com', ubuntu_archive_url)
 
     # Check launchpad.net if it's an Ubuntu libc
     # GNU C Library (Ubuntu GLIBC 2.36-0ubuntu4)

travis/libcdb_nginx_cache.conf

@@ -0,0 +1,69 @@
+events {
+    worker_connections 1024;
+}
+
+http {
+    proxy_cache_path /var/cache/nginx keys_zone=my_cache:1m max_size=1g inactive=12w use_temp_path=off;
+    log_format cache_st '$remote_addr - $remote_user - $upstream_cache_status [$time_local]  '
+                    '"$request" $status $body_bytes_sent '
+                    '"$http_referer" "$http_user_agent"';
+    access_log /dev/stdout cache_st;
+
+    server {
+        listen 3000;
+        proxy_cache my_cache;
+
+        location / {
+            proxy_set_header Host debuginfod.elfutils.org;
+            proxy_cache_revalidate on;
+            proxy_cache_key $scheme://$host$uri$is_args$query_string;
+            proxy_cache_valid 200 404 12w;
+            proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429;
+            proxy_pass https://debuginfod.elfutils.org/;
+        }
+    }
+
+    server {
+        listen 3001;
+        proxy_cache my_cache;
+
+        location / {
+            proxy_set_header Host libc.rip;
+            proxy_cache_methods GET HEAD POST;
+            proxy_cache_revalidate on;
+            proxy_cache_key $scheme://$host$uri$is_args$query_string$request_body;
+            proxy_cache_valid 200 404 12w;
+            proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429;
+            proxy_pass https://libc.rip/;
+        }
+    }
+
+    server {
+        listen 3002;
+        proxy_cache my_cache;
+
+        location / {
+            proxy_set_header Host archive.ubuntu.com;
+            proxy_cache_revalidate on;
+            proxy_cache_key $scheme://$host$uri$is_args$query_string;
+            proxy_cache_valid 200 404 12w;
+            proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429;
+            proxy_pass http://archive.ubuntu.com/;
+        }
+    }
+
+    server {
+        listen 3003;
+        proxy_cache my_cache;
+
+        location / {
+            proxy_set_header Host gitlab.com;
+            proxy_ssl_server_name on;
+            proxy_cache_revalidate on;
+            proxy_cache_key $scheme://$host$uri$is_args$query_string;
+            proxy_cache_valid 200 404 12w;
+            proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429;
+            proxy_pass https://gitlab.com/;
+        }
+    }
+}