Package headers into pip wheel

Summary: Add a new BaseExtension - HeaderFile. It takes a source
directory relative to ExecuTorch root and copies all headers recursively
into pip wheel during packaging.

The destination is hardcoded to `executorch/include/executorch` in the
pip wheel.

Test Plan:

```bash
python setup.py bdist_wheel
```

This prints out the following lines:

```
creating pip-out/bdist.linux-x86_64/wheel/executorch/include/executorch/runtime
creating pip-out/bdist.linux-x86_64/wheel/executorch/include/executorch/runtime/core
copying pip-out/lib.linux-x86_64-cpython-311/executorch/include/executorch/runtime/core/array_ref.h -> pip-out/bdist.linux-x86_64/wheel/executorch/include/executorch/runtime/core
...
```
Then install the wheel:

```bash
pip install dist/executorch-0.5.0a0+52d5218-cp311-cp311-linux_x86_64.whl
```

And we can find the headers in site-packages

```
/data/users/larryliu/executorch/pip-out/lib.linux-x86_64-cpython-311/executorch/include/executorch/runtime/core/array_ref.h
```

Reviewers:

Subscribers:

Tasks:

Tags:

Author: larryliu0820

setup.py

@@ -191,7 +191,14 @@ def get_executable_name(name: str) -> str:
 class _BaseExtension(Extension):
     """A base class that maps an abstract source to an abstract destination."""
 
-    def __init__(self, src: str, dst: str, name: str):
+    def __init__(
+        self,
+        src: str,
+        dst: str,
+        name: str,
+        is_cmake_built: bool = True,
+        is_dst_dir: bool = False,
+    ):
         # Source path; semantics defined by the subclass.
         self.src: str = src
 
@@ -206,6 +213,14 @@ def __init__(self, src: str, dst: str, name: str):
         # that doesn't look like a module path.
         self.name: str = name
 
+        # If True, the file is built by cmake. If False, the file is copied from
+        # the source tree.
+        self.is_cmake_built: bool = is_cmake_built
+
+        # If True, the destination is a directory. If False, the destination is a
+        # file.
+        self.is_dst_dir: bool = is_dst_dir
+
         super().__init__(name=self.name, sources=[])
 
     def src_path(self, installer: "InstallerBuildExt") -> Path:
@@ -215,26 +230,25 @@ def src_path(self, installer: "InstallerBuildExt") -> Path:
             installer: The InstallerBuildExt instance that is installing the
                 file.
         """
-        # Share the cmake-out location with CustomBuild.
-        cmake_cache_dir = Path(installer.get_finalized_command("build").cmake_cache_dir)
+        if self.is_cmake_built:
+            # Share the cmake-out location with CustomBuild.
+            cmake_cache_dir = Path(
+                installer.get_finalized_command("build").cmake_cache_dir
+            )
+
+            cfg = get_build_type(installer.debug)
 
-        cfg = get_build_type(installer.debug)
+            if os.name == "nt":
+                # Replace %BUILD_TYPE% with the current build type.
+                self.src = self.src.replace("%BUILD_TYPE%", cfg)
+            else:
+                # Remove %BUILD_TYPE% from the path.
+                self.src = self.src.replace("/%BUILD_TYPE%", "")
 
-        if os.name == "nt":
-            # Replace %BUILD_TYPE% with the current build type.
-            self.src = self.src.replace("%BUILD_TYPE%", cfg)
+            # Construct the full source path, do not resolve globs.
+            return cmake_cache_dir / Path(self.src)
         else:
-            # Remove %BUILD_TYPE% from the path.
-            self.src = self.src.replace("/%BUILD_TYPE%", "")
-
-        # Construct the full source path, resolving globs. If there are no glob
-        # pattern characters, this will just ensure that the source file exists.
-        srcs = tuple(cmake_cache_dir.glob(self.src))
-        if len(srcs) != 1:
-            raise ValueError(
-                f"Expected exactly one file matching '{self.src}'; found {repr(srcs)}"
-            )
-        return srcs[0]
+            return Path(self.src)
 
 
 class BuiltFile(_BaseExtension):
@@ -302,6 +316,54 @@ def dst_path(self, installer: "InstallerBuildExt") -> Path:
             return dst_root / Path(self.dst)
 
 
+class HeaderFile(_BaseExtension):
+    """An extension that installs headers in a directory.
+
+    This isn't technically a `build_ext` style python extension, but there's no
+    dedicated command for installing arbitrary data. It's convenient to use
+    this, though, because it lets us manage the files to install as entries in
+    `ext_modules`.
+    """
+
+    def __init__(
+        self,
+        src_dir: str,
+        src_name: Optional[str] = None,
+        dst_dir: Optional[str] = None,
+    ):
+        """Initializes a BuiltFile.
+
+        Args:
+            src_dir: The directory of the headers to install, relative to the root
+                ExecuTorch source directory. Under the hood, we glob all the headers
+                using `*.h` patterns.
+            src_name: The name of the header to install. If not specified, all the
+                headers in the src_dir will be installed.
+            dst_dir: The directory to install to, relative to the root of the pip
+                package. If not specified, defaults to `include/executorch/<src_dir>`.
+        """
+        if src_name is None:
+            src_name = "*.h"
+        src = os.path.join(src_dir, src_name)
+        if dst_dir is None:
+            dst = "executorch/include/executorch/"
+        else:
+            dst = dst_dir
+        super().__init__(
+            src=src, dst=dst, name=src_name, is_cmake_built=False, is_dst_dir=True
+        )
+
+    def dst_path(self, installer: "InstallerBuildExt") -> Path:
+        """Returns the path to the destination file.
+
+        Args:
+            installer: The InstallerBuildExt instance that is installing the
+                file.
+        """
+        dst_root = Path(installer.build_lib).resolve()
+        return dst_root / Path(self.dst)
+
+
 class BuiltExtension(_BaseExtension):
     """An extension that installs a python extension that was built by cmake."""
 
@@ -364,19 +426,30 @@ def build_extension(self, ext: _BaseExtension) -> None:
         src_file: Path = ext.src_path(self)
         dst_file: Path = ext.dst_path(self)
 
-        # Ensure that the destination directory exists.
-        self.mkpath(os.fspath(dst_file.parent))
-
         # Copy the file.
-        self.copy_file(os.fspath(src_file), os.fspath(dst_file))
+        src_list = src_file.parent.rglob(src_file.name)
+        for src in src_list:
+            if ext.is_dst_dir:
+                # Destination is a prefix directory. Copy the file to the
+                # destination directory.
+                dst = dst_file / src
+            else:
+                # Destination is a file. Copy the file to the destination file.
+                dst = dst_file
+
+            # Ensure that the destination directory exists.
+            if not dst.parent.exists():
+                self.mkpath(os.fspath(dst.parent))
+
+            self.copy_file(os.fspath(src), os.fspath(dst))
 
-        # Ensure that the destination file is writable, even if the source was
-        # not. build_py does this by passing preserve_mode=False to copy_file,
-        # but that would clobber the X bit on any executables. TODO(dbort): This
-        # probably won't work on Windows.
-        if not os.access(src_file, os.W_OK):
-            # Make the file writable. This should respect the umask.
-            os.chmod(src_file, os.stat(src_file).st_mode | 0o222)
+            # Ensure that the destination file is writable, even if the source was
+            # not. build_py does this by passing preserve_mode=False to copy_file,
+            # but that would clobber the X bit on any executables. TODO(dbort): This
+            # probably won't work on Windows.
+            if not os.access(src, os.W_OK):
+                # Make the file writable. This should respect the umask.
+                os.chmod(src, os.stat(src).st_mode | 0o222)
 
 
 class CustomBuildPy(build_py):
@@ -618,6 +691,21 @@ def run(self):
 def get_ext_modules() -> List[Extension]:
     """Returns the set of extension modules to build."""
     ext_modules = []
+
+    # Copy all the necessary headers into include/executorch/ so that they can
+    # be found in the pip package. This is a subset of the headers that are
+    # essential for building custom ops extensions
+    for include_dir in [
+        "runtime/core/",
+        "runtime/kernel/",
+        "runtime/platform/",
+        "extension/kernel_util/",
+    ]:
+        ext_modules.append(
+            HeaderFile(
+                src_dir=include_dir,
+            )
+        )
     if ShouldBuild.flatc():
         ext_modules.append(
             BuiltFile(