build_parameters.py: use local cache and server cache to get files

Author: khancyr

build_parameters.py

@@ -44,6 +44,14 @@
 parser.add_argument("--ardupilotRepoFolder", dest='gitFolder', default="../ardupilot", help="Ardupilot git folder. ")
 parser.add_argument("--destination", dest='destFolder', default="../../../../new_params_mversion", help="Parameters*.rst destination folder.")  # noqa: E501
 parser.add_argument('--vehicle', dest='single_vehicle', help="If you just want to copy to one vehicle, you can do this. Otherwise it will work for all vehicles (Copter, Plane, Rover, AntennaTracker, Sub, Blimp)")  # noqa: E501
+
+DEFAULT_CACHE_TIME = 6 * 3600
+
+# Get the directory where this script is located
+script_dir = os.path.dirname(os.path.abspath(__file__))
+default_cache_dir = os.path.join(script_dir, '.cache')
+
+parser.add_argument("--cache-dir", dest='cache_dir', default=default_cache_dir, help="Directory to cache HTTP responses")
 args = parser.parse_args()
 
 error_count = 0
@@ -83,6 +91,95 @@ def format(self, record):
     'Connection': 'keep-alive'
 })
 
+
+def get_cached_url(url, cache_dir=None):
+    """Get URL content with caching to avoid repeated downloads"""
+    if cache_dir is None:
+        cache_dir = args.cache_dir
+
+    os.makedirs(cache_dir, exist_ok=True)
+
+    # Create cache filename from URL
+    cache_filename = re.sub(r'[^\w\-_.]', '_', url) + '.cache'
+    cache_path = os.path.join(cache_dir, cache_filename)
+    cache_meta_path = cache_path + '.meta'
+
+    def load_cached_content():
+        with open(cache_path, 'r', encoding='utf-8') as f:
+            return f.read()
+
+    def load_cache_metadata():
+        if not os.path.exists(cache_meta_path):
+            return {}
+        try:
+            with open(cache_meta_path, 'r', encoding='utf-8') as f:
+                return json.load(f)
+        except OSError:
+            return {}
+
+    def save_cache(content, response):
+        with open(cache_path, 'w', encoding='utf-8') as f:
+            f.write(content)
+
+        metadata = {}
+        etag = response.headers.get('ETag')
+        last_modified = response.headers.get('Last-Modified')
+        if etag:
+            metadata['etag'] = etag
+        if last_modified:
+            metadata['last_modified'] = last_modified
+        if metadata:
+            with open(cache_meta_path, 'w', encoding='utf-8') as f:
+                json.dump(metadata, f)
+
+    if os.path.exists(cache_path):
+        cache_age = time.time() - os.path.getmtime(cache_path)
+        if cache_age < DEFAULT_CACHE_TIME:
+            debug(f"Using cached content for {url}")
+            return load_cached_content()
+
+    cache_metadata = {}
+    headers = {}
+    if os.path.exists(cache_path):
+        cache_metadata = load_cache_metadata()
+        if cache_metadata.get('etag'):
+            headers['If-None-Match'] = cache_metadata['etag']
+        if cache_metadata.get('last_modified'):
+            headers['If-Modified-Since'] = cache_metadata['last_modified']
+
+    if headers:
+        try:
+            debug(f"HEAD checking server for {url}")
+            head_response = session.head(url, timeout=30, headers=headers, allow_redirects=True)
+            if head_response.status_code == 304:
+                debug(f"Cache still valid for {url}")
+                return load_cached_content()
+
+            head_response.raise_for_status()
+            if (head_response.headers.get('ETag') == cache_metadata.get('etag') and
+                    head_response.headers.get('Last-Modified') == cache_metadata.get('last_modified')):
+                debug(f"Server metadata unchanged for {url}, using local cache")
+                return load_cached_content()
+        except requests.RequestException as e:
+            debug(f"HEAD request failed for {url}: {e}")
+            # Fallback to GET if the HEAD request is unsupported or fails.
+
+    try:
+        debug(f"Fetching full content from {url}")
+        response = session.get(url, timeout=30, allow_redirects=True)
+        response.raise_for_status()
+        content = response.text
+    except requests.RequestException as e:
+        error(f"Failed to fetch {url}: {e}")
+        if os.path.exists(cache_path):
+            debug(f"Using stale cached content for {url}")
+            return load_cached_content()
+        raise
+
+    save_cache(content, response)
+    return content
+
+
 # Parameters
 COMMITFILE = "git-version.txt"
 BASEURL = "https://firmware.ardupilot.org/"
@@ -361,9 +458,7 @@ def handle_starttag(self, tag, attrs):
         try:
             debug(f"Fetching {firmware_url}{vehicle}")
 
-            response = session.get(firmware_url + vehicle, timeout=30)
-            response.raise_for_status()
-            content = response.text
+            content = get_cached_url(firmware_url + vehicle)
             html_parser.feed(content)
         except Exception as e:
             error(f"Vehicles folders list download error: {e}")
@@ -415,9 +510,7 @@ def handle_starttag(self, tag, attrs):
         try:
             debug(f"Fetching {url}")
 
-            response = session.get(url, timeout=30)
-            response.raise_for_status()
-            content = response.text
+            content = get_cached_url(url)
             html_parser.feed(content)
         except Exception as e:
             error(f"Board folders list download error: {e}")
@@ -438,11 +531,9 @@ def fetch_commit_hash(version_link, board, file):
         progress(f"Processing link...\t{fetch_link}")
 
         try:
-            response = session.get(fetch_link, timeout=30)
-            response.raise_for_status()
-            fecth_response = response.text
+            fetch_response = get_cached_url(fetch_link)
 
-            commit_details = fecth_response.split("\n")
+            commit_details = fetch_response.split("\n")
             commit_hash = commit_details[0][7:]
             # version =  commit_details[6] the sizes cary
             version = commit_details.pop(-2)