Merge pull request #4 from pythonbpf/perfbuf

Add userspace utils for Structs and maps, starting with PERF_EVENT_ARRAY

Author: r41k0u

.github/workflows/pip.yml

@@ -45,7 +45,7 @@ jobs:
       run: pip install --verbose .[test]
 
     - name: Test import
-      run: python -c "import pylibbpf; print('Import successful')"
+      run: python -I -c "import pylibbpf; print('Import successful')"
 
     - name: Test
-      run: python -m pytest -v
+      run: python -I -m pytest -v

CMakeLists.txt

@@ -1,17 +1,31 @@
 cmake_minimum_required(VERSION 4.0)
 project(pylibbpf)
 
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
 # pybind11
 include_directories(${CMAKE_SOURCE_DIR}/src)
 add_subdirectory(pybind11)
 pybind11_add_module(
   pylibbpf
+  # Core
   src/core/bpf_program.h
   src/core/bpf_exception.h
   src/core/bpf_map.h
-  src/bindings/main.cpp
+  src/core/bpf_object.h
   src/core/bpf_program.cpp
-  src/core/bpf_map.cpp)
+  src/core/bpf_map.cpp
+  src/core/bpf_object.cpp
+  # Maps
+  src/maps/perf_event_array.h
+  src/maps/perf_event_array.cpp
+  # Utils
+  src/utils/struct_parser.h
+  src/utils/struct_parser.cpp
+  # Bindings
+  src/bindings/main.cpp)
 
 # --- libbpf build rules ---
 set(LIBBPF_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libbpf/src)

examples/execve.py

@@ -1,10 +1,11 @@
 import time
 from ctypes import c_int32, c_int64, c_uint64, c_void_p
 
-from pylibbpf import BpfMap
 from pythonbpf import BPF, bpf, bpfglobal, map, section
 from pythonbpf.maps import HashMap
 
+from pylibbpf import BpfMap
+
 
 @bpf
 @map

pylibbpf/__init__.py

@@ -0,0 +1,46 @@
+import logging
+
+from .ir_to_ctypes import convert_structs_to_ctypes, is_pythonbpf_structs
+from .pylibbpf import (
+    BpfException,
+    BpfMap,
+    BpfProgram,
+    PerfEventArray,
+    StructParser,
+)
+from .pylibbpf import (
+    BpfObject as _BpfObject,  # C++ object (internal)
+)
+from .wrappers import BpfObjectWrapper
+
+logger = logging.getLogger(__name__)
+
+
+class BpfObject(BpfObjectWrapper):
+    """BpfObject with automatic struct conversion"""
+
+    def __init__(self, object_path: str, structs=None):
+        """Create a BPF object"""
+        if structs is None:
+            structs = {}
+        elif is_pythonbpf_structs(structs):
+            logger.info(f"Auto-converting {len(structs)} PythonBPF structs to ctypes")
+            structs = convert_structs_to_ctypes(structs)
+
+        # Create C++ BpfObject with converted structs
+        cpp_obj = _BpfObject(object_path, structs)
+
+        # Initialize wrapper
+        super().__init__(cpp_obj)
+
+
+__all__ = [
+    "BpfObject",
+    "BpfProgram",
+    "BpfMap",
+    "PerfEventArray",
+    "StructParser",
+    "BpfException",
+]
+
+__version__ = "0.0.6"

pylibbpf/ir_to_ctypes.py

@@ -0,0 +1,105 @@
+import ctypes
+import logging
+from typing import Dict, Type
+
+from llvmlite import ir
+
+logger = logging.getLogger(__name__)
+
+
+def ir_type_to_ctypes(ir_type):
+    """Convert LLVM IR type to ctypes type."""
+    if isinstance(ir_type, ir.IntType):
+        width = ir_type.width
+        type_map = {
+            8: ctypes.c_uint8,
+            16: ctypes.c_uint16,
+            32: ctypes.c_uint32,
+            64: ctypes.c_uint64,
+        }
+        if width not in type_map:
+            raise ValueError(f"Unsupported integer width: {width}")
+        return type_map[width]
+
+    elif isinstance(ir_type, ir.ArrayType):
+        count = ir_type.count
+        element_type_ir = ir_type.element
+
+        if isinstance(element_type_ir, ir.IntType) and element_type_ir.width == 8:
+            # Use c_char for string fields (will have .decode())
+            return ctypes.c_char * count
+        else:
+            element_type = ir_type_to_ctypes(element_type_ir)
+            return element_type * count
+    elif isinstance(ir_type, ir.PointerType):
+        return ctypes.c_void_p
+
+    else:
+        raise TypeError(f"Unsupported IR type: {ir_type}")
+
+
+def _make_repr(struct_name: str, fields: list):
+    """Create a __repr__ function for a struct"""
+
+    def __repr__(self):
+        field_strs = []
+        for field_name, _ in fields:
+            value = getattr(self, field_name)
+            field_strs.append(f"{field_name}={value}")
+        return f"<{struct_name} {' '.join(field_strs)}>"
+
+    return __repr__
+
+
+def convert_structs_to_ctypes(structs_sym_tab) -> Dict[str, Type[ctypes.Structure]]:
+    """Convert PythonBPF's structs_sym_tab to ctypes.Structure classes."""
+    if not structs_sym_tab:
+        return {}
+
+    ctypes_structs = {}
+
+    for struct_name, struct_type_obj in structs_sym_tab.items():
+        try:
+            fields = []
+            for field_name, field_ir_type in struct_type_obj.fields.items():
+                field_ctypes = ir_type_to_ctypes(field_ir_type)
+                fields.append((field_name, field_ctypes))
+
+            repr_func = _make_repr(struct_name, fields)
+
+            struct_class = type(
+                struct_name,
+                (ctypes.Structure,),
+                {
+                    "_fields_": fields,
+                    "__module__": "pylibbpf.ir_to_ctypes",
+                    "__doc__": f"Auto-generated ctypes structure for {struct_name}",
+                    "__repr__": repr_func,
+                },
+            )
+
+            ctypes_structs[struct_name] = struct_class
+            # Pretty print field info
+            field_info = ", ".join(f"{name}: {typ.__name__}" for name, typ in fields)
+            logger.debug(f"  {struct_name}({field_info})")
+        except Exception as e:
+            logger.error(f"Failed to convert struct '{struct_name}': {e}")
+            raise
+    logger.info(f"Converted struct '{struct_name}' to ctypes")
+    return ctypes_structs
+
+
+def is_pythonbpf_structs(structs) -> bool:
+    """Check if structs dict is from PythonBPF."""
+    if not isinstance(structs, dict) or not structs:
+        return False
+
+    first_value = next(iter(structs.values()))
+    return (
+        hasattr(first_value, "ir_type")
+        and hasattr(first_value, "fields")
+        and hasattr(first_value, "size")
+    )
+
+
+__all__ = ["convert_structs_to_ctypes", "is_pythonbpf_structs"]

pylibbpf/py.typed

@@ -0,0 +1,77 @@
+from typing import Callable, Optional
+
+
+class PerfEventArrayHelper:
+    """Fluent wrapper for PERF_EVENT_ARRAY maps."""
+
+    def __init__(self, bpf_map):
+        self._map = bpf_map
+        self._perf_buffer = None
+
+    def open_perf_buffer(
+        self,
+        callback: Callable,
+        struct_name: str = "",
+        page_cnt: int = 8,
+        lost_callback: Optional[Callable] = None,
+    ):
+        """Open perf buffer with auto-deserialization."""
+        from .pylibbpf import PerfEventArray
+
+        if struct_name:
+            self._perf_buffer = PerfEventArray(
+                self._map,
+                page_cnt,
+                callback,
+                struct_name,
+                lost_callback or (lambda cpu, cnt: None),
+            )
+        else:
+            self._perf_buffer = PerfEventArray(
+                self._map, page_cnt, callback, lost_callback or (lambda cpu, cnt: None)
+            )
+
+        return self
+
+    def poll(self, timeout_ms: int = -1) -> int:
+        if not self._perf_buffer:
+            raise RuntimeError("Call open_perf_buffer() first")
+        return self._perf_buffer.poll(timeout_ms)
+
+    def consume(self) -> int:
+        if not self._perf_buffer:
+            raise RuntimeError("Call open_perf_buffer() first")
+        return self._perf_buffer.consume()
+
+    def __getattr__(self, name):
+        return getattr(self._map, name)
+
+
+class BpfObjectWrapper:
+    """Smart wrapper that returns map-specific helpers."""
+
+    BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4
+    BPF_MAP_TYPE_RINGBUF = 27
+
+    def __init__(self, bpf_object):
+        self._obj = bpf_object
+        self._map_helpers = {}
+
+    def __getitem__(self, name: str):
+        """Return appropriate helper based on map type."""
+        if name in self._map_helpers:
+            return self._map_helpers[name]
+
+        map_obj = self._obj[name]
+        map_type = map_obj.get_type()
+
+        if map_type == self.BPF_MAP_TYPE_PERF_EVENT_ARRAY:
+            helper = PerfEventArrayHelper(map_obj)
+        else:
+            helper = map_obj
+
+        self._map_helpers[name] = helper
+        return helper
+
+    def __getattr__(self, name):
+        return getattr(self._obj, name)

pylibbpf/wrappers.py

@@ -4,12 +4,13 @@ requires = [
     "wheel",
     "ninja",
     "cmake>=4.0",
+    "pybind11>=2.10",
 ]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "pylibbpf"
-version = "0.0.5"
+version = "0.0.6"
 description = "Python Bindings for Libbpf"
 authors = [
     { name = "r41k0u", email = "[email protected]" },
@@ -32,14 +33,17 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Topic :: System :: Operating System Kernels :: Linux",
 ]
+dependencies = [
+    "llvmlite>=0.40.0",
+]
 
 [project.optional-dependencies]
 test = ["pytest>=6.0"]
 
 [project.urls]
-Homepage = "https://github.com/varun-r-mallya/pylibbpf"
-Repository = "https://github.com/varun-r-mallya/pylibbpf"
-Issues = "https://github.com/varun-r-mallya/pylibbpf/issues"
+Homepage = "https://github.com/pythonbpf/pylibbpf"
+Repository = "https://github.com/pythonbpf/pylibbpf"
+Issues = "https://github.com/pythonbpf/pylibbpf/issues"
 
 [tool.mypy]
 files = "setup.py"

pyproject.toml

@@ -3,7 +3,7 @@
 import sys
 from pathlib import Path
 
-from setuptools import Extension, setup
+from setuptools import Extension, find_packages, setup
 from setuptools.command.build_ext import build_ext
 
 # Convert distutils Windows platform specifiers to CMake -A arguments
@@ -129,8 +129,11 @@ def build_extension(self, ext: CMakeExtension) -> None:
     description="Python Bindings for Libbpf",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    url="https://github.com/varun-r-mallya/pylibbpf",
-    ext_modules=[CMakeExtension("pylibbpf")],
+    url="https://github.com/pythonbpf/pylibbpf",
+    packages=find_packages(where="."),
+    package_dir={"": "."},
+    py_modules=[],  # Empty since we use packages
+    ext_modules=[CMakeExtension("pylibbpf.pylibbpf")],
     cmdclass={"build_ext": CMakeBuild},
     zip_safe=False,
     classifiers=[
@@ -147,6 +150,16 @@ def build_extension(self, ext: CMakeExtension) -> None:
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Topic :: System :: Operating System Kernels :: Linux",
     ],
+    install_requires=[
+        "llvmlite>=0.40.0",  # Required for struct conversion
+    ],
     extras_require={"test": ["pytest>=6.0"]},
     python_requires=">=3.8",
+    package_data={
+        "pylibbpf": [
+            "*.py",
+            "py.typed",  # For type hints
+        ],
+    },
+    include_package_data=True,
 )