Allow externally hosted contrib ports (#24303)

The logic that this PR implements is the following:

* the external port is loaded like the other ports and "flagged" as
being external (`port.is_external = hasattr(port, 'EXTERNAL_PORT'`)
* if the port is never needed, then nothing else happens
* if the port is needed (because the user wants to use it via
`--use-port=contrib.emdawn`, or it is a dependency of another port),
then the code fetches the artifact and uses the port file included in
the artifact to load the actual port file as if it was a local port, and
replaces it in the ports array and `ports_by_name` map.

=> as a result, all calls using `ports_by_name` have been changed to
call `get_port_by_name` to detect whether the port is remote or not and
act appropriately

An external port looks like this: it only contains info about where the
port is located, the actual logic is coming from `PORT_FILE`

```
TAG = 'v20250509.171557'

EXTERNAL_PORT = f'https://github.com/google/dawn/releases/download/{TAG}/emdawnwebgpu_pkg-{TAG}.zip'
SHA512 = '4b66bf0f64b9616a6420abdad636b3ecefab892dde8f67cd941147bfddf7920f5523ff10160c9a563ef377a0f88b2dfc033527591b2d0753d531de5cbbabde59'
PORT_FILE = 'emdawnwebgpu_pkg/emdawnwebgpu.port.py'

# contrib port information (required)
URL = 'https://dawn.googlesource.com/dawn'
DESCRIPTION = 'Dawn is an open-source and cross-platform implementation of the WebGPU standard'
LICENSE = 'BSD 3-Clause License'
```

Author: ypujante

test/test_other.py

@@ -2592,6 +2592,10 @@ def test_contrib_ports(self):
     # with a different contrib port when there is another one
     self.emcc(test_file('other/test_contrib_ports.cpp'), ['--use-port=contrib.glfw3'])
 
+  @requires_network
+  def test_remote_ports(self):
+    self.emcc(test_file('hello_world.c'), ['--use-port=contrib.emdawnwebgpu'])
+
   @crossplatform
   def test_external_ports_simple(self):
     if config.FROZEN_CACHE:

tools/ports/__init__.py

@@ -13,7 +13,7 @@
 from inspect import signature
 import sys
 import subprocess
-from typing import Set
+from typing import Set, Dict
 from urllib.request import urlopen
 
 from tools import cache
@@ -26,7 +26,7 @@
 
 ports = []
 
-ports_by_name = {}
+ports_by_name: Dict[str, object] = {}
 
 ports_needed = set()
 
@@ -39,11 +39,28 @@
 logger = logging.getLogger('ports')
 
 
+def get_port_by_name(name):
+  port = ports_by_name[name]
+  if port.is_external:
+    load_external_port(port)
+    return ports_by_name[name]
+  else:
+    return port
+
+
 def init_port(name, port):
   ports.append(port)
   port.is_contrib = name.startswith('contrib.')
+  port.is_external = hasattr(port, 'EXTERNAL_PORT')
   port.name = name
   ports_by_name[port.name] = port
+  if port.is_external:
+    init_external_port(name, port)
+  else:
+    init_local_port(name, port)
+
+
+def init_local_port(name, port):
   if not hasattr(port, 'needed'):
     port.needed = lambda s: name in ports_needed
   else:
@@ -71,15 +88,41 @@ def init_port(name, port):
   validate_port(port)
 
 
+def load_port_module(module_name, port_file):
+  spec = importlib.util.spec_from_file_location(module_name, port_file)
+  port = importlib.util.module_from_spec(spec)
+  spec.loader.exec_module(port)
+  return port
+
+
+def load_external_port(external_port):
+  name = external_port.name
+  up_to_date = Ports.fetch_port_artifact(name, external_port.EXTERNAL_PORT, external_port.SHA512)
+  port_file = os.path.join(Ports.get_dir(), name, external_port.PORT_FILE)
+  local_port = load_port_module(f'tools.ports.external.{name}', port_file)
+  ports.remove(external_port)
+  for a in ['URL', 'DESCRIPTION', 'LICENSE']:
+    if not hasattr(local_port, a):
+      setattr(local_port, a, getattr(external_port, a))
+  init_port(name, local_port)
+  if not up_to_date:
+    Ports.clear_project_build(name)
+
+
+def init_external_port(name, port):
+  expected_attrs = ['SHA512', 'PORT_FILE', 'URL', 'DESCRIPTION', 'LICENSE']
+  for a in expected_attrs:
+    assert hasattr(port, a), 'port %s is missing %s' % (port, a)
+  port.needed = lambda s: name in ports_needed
+  port.show = lambda: f'{port.name} (--use-port={port.name}; {port.LICENSE})'
+
+
 def load_port(path, name=None):
   if not name:
     name = shared.unsuffixed_basename(path)
   if name in ports_by_name:
     utils.exit_with_error(f'port path [`{path}`] is invalid: duplicate port name `{name}`')
-  module_name = f'tools.ports.{name}'
-  spec = importlib.util.spec_from_file_location(module_name, path)
-  port = importlib.util.module_from_spec(spec)
-  spec.loader.exec_module(port)
+  port = load_port_module(f'tools.ports.{name}', path)
   init_port(name, port)
   return name
 
@@ -251,7 +294,8 @@ def get_build_dir():
   name_cache: Set[str] = set()
 
   @staticmethod
-  def fetch_project(name, url, sha512hash=None):
+  def fetch_port_artifact(name, url, sha512hash=None):
+    """This function only fetches the port and returns True when the port is up to date, False otherwise"""
     # To compute the sha512 hash, run `curl URL | sha512sum`.
     fullname = Ports.get_dir(name)
 
@@ -291,18 +335,17 @@ def fetch_project(name, url, sha512hash=None):
           # before acquiring the lock we have an early out if the port already exists
           if os.path.exists(target) and dir_is_newer(path, target):
             logger.warning(uptodate_message)
-            return
+            return True
           with cache.lock('unpack local port'):
             # Another early out in case another process unpackage the library while we were
             # waiting for the lock
             if os.path.exists(target) and not dir_is_newer(path, target):
               logger.warning(uptodate_message)
-              return
+              return True
             logger.warning(f'grabbing local port: {name} from {path} to {fullname} (subdir: {subdir})')
             utils.delete_dir(fullname)
             shutil.copytree(path, target)
-            Ports.clear_project_build(name)
-          return
+            return False
 
     url_filename = url.rsplit('/')[-1]
     ext = url_filename.split('.', 1)[1]
@@ -343,16 +386,17 @@ def up_to_date():
 
     # before acquiring the lock we have an early out if the port already exists
     if up_to_date():
-      return
+      return True
 
     # main logic. do this under a cache lock, since we don't want multiple jobs to
     # retrieve the same port at once
+    cache.ensure() # TODO: find a better place for this (necessary at the moment)
     with cache.lock('unpack port'):
       if os.path.exists(fullpath):
         # Another early out in case another process unpackage the library while we were
         # waiting for the lock
         if up_to_date():
-          return
+          return True
         # file exists but tag is bad
         logger.warning('local copy of port is not correct, retrieving from remote server')
         utils.delete_dir(fullname)
@@ -361,12 +405,17 @@ def up_to_date():
       retrieve()
       unpack()
 
+      return False
+
+  @staticmethod
+  def fetch_project(name, url, sha512hash=None):
+    if not Ports.fetch_port_artifact(name, url, sha512hash):
       # we unpacked a new version, clear the build in the cache
       Ports.clear_project_build(name)
 
   @staticmethod
   def clear_project_build(name):
-    port = ports_by_name[name]
+    port = get_port_by_name(name)
     port.clear(Ports, settings, shared)
     build_dir = os.path.join(Ports.get_build_dir(), name)
     logger.debug(f'clearing port build: {name} {build_dir}')
@@ -449,7 +498,7 @@ def add_deps(node):
       d, _ = split_port_options(d)
       if d not in ports_by_name:
         utils.exit_with_error(f'unknown dependency `{d}` for port `{node.name}`')
-      dep = ports_by_name[d]
+      dep = get_port_by_name(d)
       if dep not in port_set:
         port_set.add(dep)
         add_deps(dep)
@@ -479,7 +528,7 @@ def show_port_help_and_exit(port):
 
 # extract dict and delegate to port.handle_options for handling (format is 'option1=value1:option2=value2')
 def handle_port_options(name, options, error_handler):
-  port = ports_by_name[name]
+  port = get_port_by_name(name)
   if options == 'help':
     show_port_help_and_exit(port)
   if not hasattr(port, 'handle_options'):
@@ -501,7 +550,7 @@ def handle_port_options(name, options, error_handler):
 
 # handle port dependencies (ex: deps=['sdl2_image:formats=jpg'])
 def handle_port_deps(name, error_handler):
-  port = ports_by_name[name]
+  port = get_port_by_name(name)
   for dep in port.deps:
     dep_name, dep_options = split_port_options(dep)
     if dep_name not in ports_by_name:
@@ -541,21 +590,21 @@ def error_handler(message):
 
 def get_needed_ports(settings, cflags_only=False):
   # Start with directly needed ports, and transitively add dependencies
-  needed = OrderedSet(p for p in ports if p.needed(settings))
+  needed = OrderedSet(get_port_by_name(p.name) for p in ports if p.needed(settings))
   resolve_dependencies(needed, settings, cflags_only)
   return needed
 
 
 def build_port(port_name, settings):
-  port = ports_by_name[port_name]
+  port = get_port_by_name(port_name)
   port_set = OrderedSet([port])
   resolve_dependencies(port_set, settings)
   for port in dependency_order(port_set):
     port.get(Ports, settings, shared)
 
 
 def clear_port(port_name, settings):
-  port = ports_by_name[port_name]
+  port = get_port_by_name(port_name)
   port.clear(Ports, settings, shared)
 
 

tools/ports/contrib/emdawnwebgpu.py

@@ -0,0 +1,16 @@
+# Copyright 2025 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+
+
+TAG = 'v20250516.124039'
+
+EXTERNAL_PORT = f'https://github.com/google/dawn/releases/download/{TAG}/emdawnwebgpu_pkg-{TAG}.zip'
+SHA512 = '994eac4be5f69d8ec83838af9c7b4cc87f15fa22bede589517c169320dd69ab5cf164528f7bd6ec6503b1ef178da3d87df0565d16445dac2a69f98450083dd8f'
+PORT_FILE = 'emdawnwebgpu_pkg/emdawnwebgpu.port.py'
+
+# contrib port information (required)
+URL = 'https://dawn.googlesource.com/dawn'
+DESCRIPTION = 'Dawn is an open-source and cross-platform implementation of the WebGPU standard'
+LICENSE = 'Some files: BSD 3-Clause License. Other files: Emscripten\'s license (available under both MIT License and University of Illinois/NCSA Open Source License)'