daemon: cgen: route matcher codegen through flavor-specific dispatch

Replace gen_inline_get_mark and gen_inline_get_skb in
bf_flavor_ops with a single gen_inline_matcher callback that gives
each flavor full control over matcher codegen. Packet-based
flavors intercept matcher types they handle specially (e.g.,
meta.mark on __sk_buff-based flavors) and delegate the rest to
bf_matcher_generate_packet().

Extract bf_matcher_generate_packet() from program.c into cgen/
matcher/packet.{h,c} where it belongs alongside the matcher
codegen it dispatches to.

Export bf_matcher_generate_meta_mark() and
bf_matcher_generate_meta_flow_hash() as comparison-only
helpers. Callers load the value into BPF_REG_1 before calling.

This is a prerequisite for CGROUP_SOCK_ADDR support, where the
flavor needs to intercept matchers like ip4.daddr and generate
bytecode that reads from bpf_sock_addr fields instead of packet
headers.

Author: yaakov-stein

src/bpfilter/CMakeLists.txt

@@ -27,6 +27,7 @@ add_executable(bpfilter
     ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/tcp.h       ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/tcp.c
     ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/udp.h       ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/udp.c
     ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/meta.h      ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/meta.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/packet.h    ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/packet.c
     ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/set.h       ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/set.c
     ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/icmp.h      ${CMAKE_CURRENT_SOURCE_DIR}/cgen/matcher/icmp.c
     ${CMAKE_CURRENT_SOURCE_DIR}/cgen/nf.h                ${CMAKE_CURRENT_SOURCE_DIR}/cgen/nf.c

src/bpfilter/cgen/cgroup_skb.c

@@ -15,9 +15,12 @@
 #include <bpfilter/btf.h>
 #include <bpfilter/flavor.h>
 #include <bpfilter/helper.h>
+#include <bpfilter/matcher.h>
 #include <bpfilter/verdict.h>
 
 #include "cgen/cgen.h"
+#include "cgen/matcher/meta.h"
+#include "cgen/matcher/packet.h"
 #include "cgen/program.h"
 #include "cgen/stub.h"
 #include "cgen/swich.h"
@@ -114,30 +117,33 @@ static int _bf_cgroup_skb_gen_inline_set_mark(struct bf_program *program,
     return 0;
 }
 
-static int _bf_cgroup_skb_gen_inline_get_mark(struct bf_program *program,
-                                              int reg)
+static int _bf_cgroup_skb_gen_inline_matcher(struct bf_program *program,
+                                             const struct bf_matcher *matcher)
 {
-    EMIT(program,
-         BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, BF_PROG_CTX_OFF(arg)));
-    EMIT(program,
-         BPF_LDX_MEM(BPF_W, reg, BPF_REG_1, offsetof(struct __sk_buff, mark)));
-
-    return 0;
-}
-
-static int _bf_cgroup_skb_gen_inline_get_skb(struct bf_program *program,
-                                             int reg)
-{
-    EMIT(program, BPF_LDX_MEM(BPF_DW, reg, BPF_REG_10, BF_PROG_CTX_OFF(arg)));
-
-    return 0;
+    assert(program);
+    assert(matcher);
+
+    switch (bf_matcher_get_type(matcher)) {
+    case BF_MATCHER_META_MARK:
+        EMIT(program,
+             BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, BF_PROG_CTX_OFF(arg)));
+        EMIT(program, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1,
+                                  offsetof(struct __sk_buff, mark)));
+        return bf_matcher_generate_meta_mark(program, matcher);
+    case BF_MATCHER_META_FLOW_HASH:
+        EMIT(program,
+             BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, BF_PROG_CTX_OFF(arg)));
+        return bf_matcher_generate_meta_flow_hash(program, matcher);
+    default:
+        return bf_matcher_generate_packet(program, matcher);
+    }
 }
 
 /**
  * Convert a standard verdict into a return value.
  *
  * @param verdict Verdict to convert. Must be valid.
- * @return TC return code corresponding to the verdict, as an integer.
+ * @return Cgroup return code corresponding to the verdict, as an integer.
  */
 static int _bf_cgroup_skb_get_verdict(enum bf_verdict verdict)
 {
@@ -155,7 +161,6 @@ const struct bf_flavor_ops bf_flavor_ops_cgroup_skb = {
     .gen_inline_prologue = _bf_cgroup_skb_gen_inline_prologue,
     .gen_inline_epilogue = _bf_cgroup_skb_gen_inline_epilogue,
     .gen_inline_set_mark = _bf_cgroup_skb_gen_inline_set_mark,
-    .gen_inline_get_mark = _bf_cgroup_skb_gen_inline_get_mark,
-    .gen_inline_get_skb = _bf_cgroup_skb_gen_inline_get_skb,
     .get_verdict = _bf_cgroup_skb_get_verdict,
+    .gen_inline_matcher = _bf_cgroup_skb_gen_inline_matcher,
 };

src/bpfilter/cgen/matcher/meta.c

@@ -141,15 +141,15 @@ static int _bf_matcher_generate_meta_port(struct bf_program *program,
     return 0;
 }
 
-static int _bf_matcher_generate_meta_mark(struct bf_program *program,
-                                          const struct bf_matcher *matcher)
+int bf_matcher_generate_meta_mark(struct bf_program *program,
+                                  const struct bf_matcher *matcher)
 {
-    uint32_t mark = *(uint32_t *)bf_matcher_payload(matcher);
-    int r;
+    uint32_t mark;
 
-    r = program->runtime.ops->gen_inline_get_mark(program, BPF_REG_1);
-    if (r)
-        return bf_err_r(r, "failed to get inline mark");
+    assert(program);
+    assert(matcher);
+
+    mark = *(uint32_t *)bf_matcher_payload(matcher);
 
     switch (bf_matcher_get_op(matcher)) {
     case BF_MATCHER_EQ:
@@ -169,15 +169,15 @@ static int _bf_matcher_generate_meta_mark(struct bf_program *program,
     return 0;
 }
 
-static int _bf_matcher_generate_meta_flow_hash(struct bf_program *program,
-                                               const struct bf_matcher *matcher)
+int bf_matcher_generate_meta_flow_hash(struct bf_program *program,
+                                       const struct bf_matcher *matcher)
 {
-    uint32_t *hash = (uint32_t *)bf_matcher_payload(matcher);
-    int r;
+    uint32_t *hash;
 
-    r = program->runtime.ops->gen_inline_get_skb(program, BPF_REG_1);
-    if (r)
-        return bf_err_r(r, "failed to get inline skb");
+    assert(program);
+    assert(matcher);
+
+    hash = (uint32_t *)bf_matcher_payload(matcher);
 
     EMIT(program, BPF_EMIT_CALL(BPF_FUNC_get_hash_recalc));
 
@@ -276,12 +276,6 @@ int bf_matcher_generate_meta(struct bf_program *program,
     case BF_MATCHER_META_DPORT:
         r = _bf_matcher_generate_meta_port(program, matcher);
         break;
-    case BF_MATCHER_META_MARK:
-        r = _bf_matcher_generate_meta_mark(program, matcher);
-        break;
-    case BF_MATCHER_META_FLOW_HASH:
-        r = _bf_matcher_generate_meta_flow_hash(program, matcher);
-        break;
     case BF_MATCHER_META_FLOW_PROBABILITY:
         r = _bf_matcher_generate_meta_flow_probability(program, matcher);
         break;

src/bpfilter/cgen/matcher/meta.h

@@ -10,3 +10,31 @@ struct bf_program;
 
 int bf_matcher_generate_meta(struct bf_program *program,
                              const struct bf_matcher *matcher);
+
+/**
+ * @brief Generate bytecode to compare a mark value already loaded in
+ * @c BPF_REG_1 against the matcher's payload.
+ *
+ * The caller is responsible for loading the mark value into @c BPF_REG_1
+ * before calling this function.
+ *
+ * @param program Program being generated. Can't be NULL.
+ * @param matcher Matcher to generate comparison for. Can't be NULL.
+ * @return 0 on success, negative errno on error.
+ */
+int bf_matcher_generate_meta_mark(struct bf_program *program,
+                                  const struct bf_matcher *matcher);
+
+/**
+ * @brief Generate bytecode to compute and compare a flow hash. The skb pointer
+ * must already be loaded in @c BPF_REG_1 before calling this function.
+ *
+ * Calls @c bpf_get_hash_recalc on the skb, then compares the result against
+ * the matcher's payload.
+ *
+ * @param program Program being generated. Can't be NULL.
+ * @param matcher Matcher to generate comparison for. Can't be NULL.
+ * @return 0 on success, negative errno on error.
+ */
+int bf_matcher_generate_meta_flow_hash(struct bf_program *program,
+                                       const struct bf_matcher *matcher);

src/bpfilter/cgen/matcher/packet.c

@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
+ */
+
+#include "cgen/matcher/packet.h"
+
+#include <errno.h>
+
+#include <bpfilter/helper.h>
+#include <bpfilter/logger.h>
+#include <bpfilter/matcher.h>
+
+#include "cgen/matcher/icmp.h"
+#include "cgen/matcher/ip4.h"
+#include "cgen/matcher/ip6.h"
+#include "cgen/matcher/meta.h"
+#include "cgen/matcher/set.h"
+#include "cgen/matcher/tcp.h"
+#include "cgen/matcher/udp.h"
+#include "cgen/program.h"
+
+int bf_matcher_generate_packet(struct bf_program *program,
+                               const struct bf_matcher *matcher)
+{
+    int r;
+
+    assert(program);
+    assert(matcher);
+
+    switch (bf_matcher_get_type(matcher)) {
+    case BF_MATCHER_META_IFACE:
+    case BF_MATCHER_META_L3_PROTO:
+    case BF_MATCHER_META_L4_PROTO:
+    case BF_MATCHER_META_PROBABILITY:
+    case BF_MATCHER_META_SPORT:
+    case BF_MATCHER_META_DPORT:
+    case BF_MATCHER_META_FLOW_PROBABILITY:
+        r = bf_matcher_generate_meta(program, matcher);
+        if (r)
+            return r;
+        break;
+    case BF_MATCHER_META_MARK:
+    case BF_MATCHER_META_FLOW_HASH:
+        return bf_err_r(-ENOTSUP,
+                        "matcher '%s' is not supported by this flavor",
+                        bf_matcher_type_to_str(bf_matcher_get_type(matcher)));
+    case BF_MATCHER_IP4_SADDR:
+    case BF_MATCHER_IP4_SNET:
+    case BF_MATCHER_IP4_DADDR:
+    case BF_MATCHER_IP4_DNET:
+    case BF_MATCHER_IP4_PROTO:
+    case BF_MATCHER_IP4_DSCP:
+        r = bf_matcher_generate_ip4(program, matcher);
+        if (r)
+            return r;
+        break;
+    case BF_MATCHER_IP6_SADDR:
+    case BF_MATCHER_IP6_SNET:
+    case BF_MATCHER_IP6_DADDR:
+    case BF_MATCHER_IP6_DNET:
+    case BF_MATCHER_IP6_NEXTHDR:
+    case BF_MATCHER_IP6_DSCP:
+        r = bf_matcher_generate_ip6(program, matcher);
+        if (r)
+            return r;
+        break;
+    case BF_MATCHER_TCP_SPORT:
+    case BF_MATCHER_TCP_DPORT:
+    case BF_MATCHER_TCP_FLAGS:
+        r = bf_matcher_generate_tcp(program, matcher);
+        if (r)
+            return r;
+        break;
+    case BF_MATCHER_UDP_SPORT:
+    case BF_MATCHER_UDP_DPORT:
+        r = bf_matcher_generate_udp(program, matcher);
+        if (r)
+            return r;
+        break;
+    case BF_MATCHER_ICMP_TYPE:
+    case BF_MATCHER_ICMP_CODE:
+    case BF_MATCHER_ICMPV6_TYPE:
+    case BF_MATCHER_ICMPV6_CODE:
+        r = bf_matcher_generate_icmp(program, matcher);
+        if (r)
+            return r;
+        break;
+    case BF_MATCHER_SET:
+        r = bf_matcher_generate_set(program, matcher);
+        if (r)
+            return r;
+        break;
+    default:
+        return bf_err_r(-EINVAL, "unknown matcher type %d",
+                        bf_matcher_get_type(matcher));
+    };
+
+    return 0;
+}

src/bpfilter/cgen/matcher/packet.h

@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
+ */
+
+#pragma once
+
+struct bf_matcher;
+struct bf_program;
+
+/**
+ * @brief Generate bytecode for a packet-based matcher.
+ *
+ * Dispatches to the appropriate matcher codegen function based on the matcher
+ * type. Handles all matchers that operate on packet headers or metadata common
+ * to all packet-based flavors.
+ *
+ * `BF_MATCHER_META_MARK` and `BF_MATCHER_META_FLOW_HASH` are not supported
+ * by this function and return `-ENOTSUP`, as they require flavor-specific
+ * context access.
+ *
+ * @param program Program being generated. Can't be NULL.
+ * @param matcher Matcher to generate code for. Can't be NULL.
+ * @return 0 on success, negative errno on error.
+ */
+int bf_matcher_generate_packet(struct bf_program *program,
+                               const struct bf_matcher *matcher);

src/bpfilter/cgen/nf.c

@@ -19,9 +19,12 @@
 #include <bpfilter/flavor.h>
 #include <bpfilter/helper.h>
 #include <bpfilter/hook.h>
+#include <bpfilter/matcher.h>
 #include <bpfilter/verdict.h>
 
 #include "cgen/jmp.h"
+#include "cgen/matcher/meta.h"
+#include "cgen/matcher/packet.h"
 #include "cgen/program.h"
 #include "cgen/stub.h"
 #include "cgen/swich.h"
@@ -124,39 +127,42 @@ static int _bf_nf_gen_inline_epilogue(struct bf_program *program)
     return 0;
 }
 
-static int _bf_nf_gen_inline_get_mark(struct bf_program *program, int reg)
+static int _bf_nf_gen_inline_matcher(struct bf_program *program,
+                                     const struct bf_matcher *matcher)
 {
     int offset;
 
-    EMIT(program,
-         BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, BF_PROG_CTX_OFF(arg)));
-    if ((offset = bf_btf_get_field_off("bpf_nf_ctx", "skb")) < 0)
-        return offset;
-    EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, offset));
-    if ((offset = bf_btf_get_field_off("sk_buff", "mark")) < 0)
-        return offset;
-    EMIT(program, BPF_LDX_MEM(BPF_W, reg, BPF_REG_2, offset));
-
-    return 0;
-}
-
-static int _bf_nf_gen_inline_get_skb(struct bf_program *program, int reg)
-{
-    int offset;
-
-    EMIT(program, BPF_LDX_MEM(BPF_DW, reg, BPF_REG_10, BF_PROG_CTX_OFF(arg)));
-    if ((offset = bf_btf_get_field_off("bpf_nf_ctx", "skb")) < 0)
-        return offset;
-    EMIT(program, BPF_LDX_MEM(BPF_DW, reg, reg, offset));
+    assert(program);
+    assert(matcher);
 
-    return 0;
+    switch (bf_matcher_get_type(matcher)) {
+    case BF_MATCHER_META_MARK:
+        EMIT(program,
+             BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, BF_PROG_CTX_OFF(arg)));
+        if ((offset = bf_btf_get_field_off("bpf_nf_ctx", "skb")) < 0)
+            return offset;
+        EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, offset));
+        if ((offset = bf_btf_get_field_off("sk_buff", "mark")) < 0)
+            return offset;
+        EMIT(program, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_2, offset));
+        return bf_matcher_generate_meta_mark(program, matcher);
+    case BF_MATCHER_META_FLOW_HASH:
+        EMIT(program,
+             BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, BF_PROG_CTX_OFF(arg)));
+        if ((offset = bf_btf_get_field_off("bpf_nf_ctx", "skb")) < 0)
+            return offset;
+        EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, offset));
+        return bf_matcher_generate_meta_flow_hash(program, matcher);
+    default:
+        return bf_matcher_generate_packet(program, matcher);
+    }
 }
 
 /**
  * Convert a standard verdict into a return value.
  *
  * @param verdict Verdict to convert. Must be valid.
- * @return TC return code corresponding to the verdict, as an integer.
+ * @return Netfilter return code corresponding to the verdict, as an integer.
  */
 static int _bf_nf_get_verdict(enum bf_verdict verdict)
 {
@@ -173,7 +179,6 @@ static int _bf_nf_get_verdict(enum bf_verdict verdict)
 const struct bf_flavor_ops bf_flavor_ops_nf = {
     .gen_inline_prologue = _bf_nf_gen_inline_prologue,
     .gen_inline_epilogue = _bf_nf_gen_inline_epilogue,
-    .gen_inline_get_mark = _bf_nf_gen_inline_get_mark,
-    .gen_inline_get_skb = _bf_nf_gen_inline_get_skb,
     .get_verdict = _bf_nf_get_verdict,
+    .gen_inline_matcher = _bf_nf_gen_inline_matcher,
 };