Merge pull request #6 from jpsamaroo/jps/decent-api

Make API more decent

Author: jpsamaroo

FILTER_LICENSES.md

@@ -1,5 +1,11 @@
 # This file is machine-generated - editing it directly is not advised
 
+[[ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+
+[[Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
@@ -32,24 +38,38 @@ uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
+[[Downloads]]
+deps = ["ArgTools", "LibCURL", "NetworkOptions"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+
 [[GPUCompiler]]
-deps = ["DataStructures", "InteractiveUtils", "LLVM", "Libdl", "TimerOutputs", "UUIDs"]
-path = "/home/jpsamaroo/.julia/dev/GPUCompiler-bpf/"
+deps = ["DataStructures", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "Serialization", "TimerOutputs", "UUIDs"]
+git-tree-sha1 = "80100dc01602c15fe1abf83b70bf2cd8961fb0e5"
+repo-rev = "master"
+repo-url = "https://github.com/JuliaGPU/GPUCompiler.jl.git"
 uuid = "61eb1bfa-7361-4325-ad38-22787b887f55"
-version = "0.7.1"
+version = "0.9.0"
 
 [[InteractiveUtils]]
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
 [[LLVM]]
 deps = ["CEnum", "Libdl", "Printf", "Unicode"]
-git-tree-sha1 = "70070a0131f17fcffc5fc004f5f73f037bd217c5"
+git-tree-sha1 = "000a737732aa4eb996414c4685368f6a74b41d14"
 uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
-version = "3.2.0"
+version = "3.4.0"
+
+[[LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+
+[[LibCURL_jll]]
+deps = ["Libdl"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
 
 [[LibGit2]]
-deps = ["Printf"]
+deps = ["NetworkOptions", "Printf"]
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
 [[Libdl]]
@@ -69,21 +89,27 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
+[[MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+
+[[NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+
 [[OrderedCollections]]
 git-tree-sha1 = "293b70ac1780f9584c89268a6e2a560d938a7065"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 version = "1.3.0"
 
 [[Pkg]]
-deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
+deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [[Printf]]
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
 [[REPL]]
-deps = ["InteractiveUtils", "Markdown", "Sockets"]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
 uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 
 [[Random]]
@@ -93,6 +119,12 @@ uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 [[SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
+[[Scratch]]
+deps = ["Dates"]
+git-tree-sha1 = "ad4b278adb62d185bbcb6864dc24959ab0627bf6"
+uuid = "6c6a2e73-6563-6170-7368-637461726353"
+version = "1.0.3"
+
 [[Serialization]]
 uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 
@@ -111,8 +143,16 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 deps = ["LinearAlgebra", "SparseArrays"]
 uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
+[[TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+
+[[Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+
 [[Test]]
-deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [[TimerOutputs]]

Manifest.toml

@@ -10,9 +10,9 @@ LLVM = "929cbde3-209d-540e-8aea-75f648917ca0"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [compat]
-GPUCompiler = "0.7"
-LLVM = "3"
-julia = "1.3"
+GPUCompiler = "0.9"
+LLVM = "3.4"
+julia = "1.6"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

Project.toml

@@ -1,17 +1,32 @@
 # BPFnative.jl
 
-## Instructions for use
-* Recompile Julia's LLVM with the appropriate line in deps/llvm.mk as such:
-  `LLVM_TARGETS := host;NVPTX;AMDGPU;BPF`
-* `] build LLVM` to rebuild LLVM
-* Run `bpfgen(io, license, f, tt)` to generate a BPF kernel with license
-  `license`, from function `f` with input types `tt`. The kernel's object file
-  (ELF format) will be written to `io`.
-* If your kernel is an XDP filter, load it on the appropriate interface with:
-  `ip link set dev myiface xdp obj mybpfkernel.o verbose`. `iproute2` will
-  output some information on your kernel, which is useful for debugging when
-  the Linux kernel refuses to load your kernel (this is common). When you're
-  done with that kernel, unload it with `ip link set dev myiface xdp off`.
+BPFnative provides the ability to write eBPF filters in Julia. Additionally,
+wrappers to the [libbpf](https://github.com/libbpf/libbpf) library are provided
+to make it easy to load eBPF programs into the Linux kernel in for a variety of
+use cases.
+
+## Filter Licensing
+
+Disclaimer: I am not a lawyer, so the following information may be completely
+incorrect. If you are a lawyer or have equivalent experience, and believe the
+following information to be misleading or incorrect, please file an issue/PR.
+
+Because the Linux kernel exposes BPF helpers functions which are only
+available to GPL-licensed programs, BPFnative provides the option to allow BPF
+kernels to be generated under the GPL license, or whatever license is
+provided to `bpffunction()`. The subsequently generated kernel and
+source that generates it would then be considered to be licensed as
+specified. The default license is the empty string "", which may be construed
+to imply a lack of a license (proprietary).
+
+BPFnative itself is of course just a compiler, so it may retain its MIT
+license. However, users should keep in mind that whatever license they specify
+to `bpffunction()` is the license that they must adhere to. This means
+that, for example, if a user were to generate a GPL-licensed BPF kernel with
+BPFnative's compiler, the user would be obligated to adhere to the terms of
+the GPL license, and specifically, would be required to provide the source
+code used to generate their BPF kernel, as well as distributing a copy or
+reference to the GPL license itself.
 
 ## Acknowledgments
 Thanks to @vchuravy for MCAnalyzer.jl, and @maleadt for CUDAnative.jl, both of

README.md

@@ -5,14 +5,34 @@ using InteractiveUtils
 using Libdl
 using GPUCompiler
 
-include("c_helpers.jl")
+# Host API
+module API
+include("common.jl")
 include("libbpf.jl")
-include("maps.jl")
+end
+
+# Runtime API
+module RT
+import ..API
+include("runtime_maps.jl")
+include("misc.jl")
+end
+
+# Host API
+module Host
+import ..API
+include("syscall.jl")
+include("host_maps.jl")
+end
+
+# Compiler
 include("compiler.jl")
-include("execution.jl")
 include("reflection.jl")
 
-# Some useful utilities
+# Easy-to-use probes
+include("probes.jl")
+
+# Useful utilities
 include("xdp.jl")
 
 end # module

src/BPFnative.jl

@@ -1,5 +1,3 @@
-module API
-
 export BPFCmd, BPFMapType, BPFProgType, BPFAttachType, BPFHelper
 
 @enum BPFCmd begin
@@ -257,4 +255,35 @@ end
     sk_assign
 end
 
+## Perf/Ptrace
+
+# TODO: pt_regs for other architectures
+struct pt_regs
+    ebx::Clong
+    ecx::Clong
+    edx::Clong
+    esi::Clong
+    edi::Clong
+    ebp::Clong
+    eax::Clong
+    xds::Cint
+    xes::Cint
+    xfs::Cint
+    xgs::Cint
+    orig_eax::Clong
+    eip::Clong
+    xcs::Cint
+    eflags::Clong
+    esp::Clong
+    xss::Cint
+end
+
+## XDP
+
+struct xdp_md
+    data::UInt32
+    data_end::UInt32
+    data_meta::UInt32
+    ingress_ifindex::UInt32
+    rx_queue_index::UInt32
 end

src/c_helpers.jl renamed to src/common.jl

@@ -1,5 +1,51 @@
+export bpffunction, bpfasm
+
+# Compilation support
+
 struct BPFCompilerParams <: AbstractCompilerParams end
 
 BPFCompilerJob = CompilerJob{BPFCompilerTarget,BPFCompilerParams}
 
 GPUCompiler.runtime_module(::BPFCompilerJob) = BPFnative
+
+
+## host-side API
+
+"""
+    bpffunction(f, tt=Tuple{}; kwargs...)
+
+Compiles a function invocation into eBPF bytecode.
+
+The following keyword arguments are supported:
+- `name`: override the name that the kernel will have in the generated code
+- `license::String=""`: license for the kernel source code and resulting object
+- `prog_section::String=""`: ELF section that the kernel will be placed in
+- `btf::Bool=true`: Whether to generate BTF debuginfo or not
+
+The output of this function is automatically cached, i.e. you can simply call
+`bpffunction` in a hot path without degrading performance. New code will be
+generated automatically when the function changes or when different types or
+keyword arguments are provided.
+"""
+function bpffunction(f::Core.Function, tt::Type=Tuple{}; name=nothing, kwargs...)
+    source = FunctionSpec(f, tt, false, name)
+    GPUCompiler.cached_compilation(bpffunction_cache,
+                                   bpffunction_compile,
+                                   bpffunction_link,
+                                   source; kwargs...)
+end
+
+const bpffunction_cache = Dict{UInt,Any}()
+
+# actual compilation
+function bpffunction_compile(source::FunctionSpec; format=:obj, license="",
+                             prog_section="prog", btf=true, kwargs...)
+    # compile to BPF
+    target = BPFCompilerTarget(; license, prog_section)
+    params = BPFCompilerParams()
+    job = CompilerJob(target, source, params)
+    args = GPUCompiler.compile(format, job; validate=true, libraries=false, strip=!btf)
+    format == :llvm && return collect.(codeunits.(string.(args))) # TODO: Make more efficient
+    return collect(codeunits(args[1]))
+end
+bpffunction_link(@nospecialize(source::FunctionSpec), exe; kwargs...) = exe