diff --git a/backends/xnnpack/third-party/xnnpack.buck.bzl b/backends/xnnpack/third-party/xnnpack.buck.bzl
index 8556fde3d8a..c01512eed26 100644
--- a/backends/xnnpack/third-party/xnnpack.buck.bzl
+++ b/backends/xnnpack/third-party/xnnpack.buck.bzl
@@ -4,8 +4,8 @@ load(
     "OPERATOR_SRCS",
     "SUBGRAPH_SRCS",
     "TABLE_SRCS",
-    "XNNPACK_SRCS",
     "get_xnnpack_headers",
+    "get_ukernel_config_srcs",
     "prod_srcs_for_arch_wrapper",
 )
 
@@ -1070,7 +1070,7 @@ def define_xnnpack():
     # @lint-ignore BUCKLINT: native and fb_native are explicitly forbidden in fbcode.
     native.cxx_library(
         name = "XNNPACK",
-        srcs = XNNPACK_SRCS + LOGGING_SRCS + [
+        srcs = get_ukernel_config_srcs() + LOGGING_SRCS + [
             "XNNPACK/src/init.c",
             "XNNPACK/src/params.c",
             "XNNPACK/src/configs/hardware-config.c",
diff --git a/backends/xnnpack/third-party/xnnpack_src_defs.bzl b/backends/xnnpack/third-party/xnnpack_src_defs.bzl
index cb1f635e79e..25477e8c718 100644
--- a/backends/xnnpack/third-party/xnnpack_src_defs.bzl
+++ b/backends/xnnpack/third-party/xnnpack_src_defs.bzl
@@ -9,32 +9,6 @@ load("//backends/xnnpack/third-party/XNNPACK/gen:microkernels.bzl", "prod_srcs_f
 load("@fbsource//xplat/executorch/third-party:glob_defs.bzl", "subdir_glob")
 
 # To get from XNNPACK:build_srcs.bzl in the future
-_XNNPACK_SRCS = [
-    "src/configs/argmaxpool-config.c",
-    "src/configs/avgpool-config.c",
-    "src/configs/binary-elementwise-config.c",
-    "src/configs/cmul-config.c",
-    "src/configs/conv-hwc2chw-config.c",
-    "src/configs/dwconv-config.c",
-    "src/configs/dwconv2d-chw-config.c",
-    "src/configs/gemm-config.c",
-    "src/configs/ibilinear-chw-config.c",
-    "src/configs/ibilinear-config.c",
-    "src/configs/lut32norm-config.c",
-    "src/configs/maxpool-config.c",
-    "src/configs/pack-lh-config.c",
-    "src/configs/raddstoreexpminusmax-config.c",
-    "src/configs/reduce-config.c",
-    "src/configs/spmm-config.c",
-    "src/configs/transpose-config.c",
-    "src/configs/unary-elementwise-config.c",
-    "src/configs/unpool-config.c",
-    "src/configs/vmulcaddc-config.c",
-    "src/configs/x8-lut-config.c",
-    "src/configs/xx-fill-config.c",
-    "src/configs/xx-pad-config.c",
-]
-
 def define_xnnpack_build_src(xnnpack_build_src):
     return ["XNNPACK/{}".format(src) for src in xnnpack_build_src]
 
@@ -56,8 +30,12 @@ def get_xnnpack_headers():
     ])
     return src_headers | include_headers | ukernel_headers
 
+def get_ukernel_config_srcs():
+    return subdir_glob([
+        ("XNNPACK/src/configs", "*.c"),
+    ]).values()
+
 OPERATOR_SRCS = define_xnnpack_build_src(_OPERATOR_SRCS)
 SUBGRAPH_SRCS = define_xnnpack_build_src(_SUBGRAPH_SRCS)
 TABLE_SRCS = define_xnnpack_build_src(_TABLE_SRCS)
-XNNPACK_SRCS = define_xnnpack_build_src(_XNNPACK_SRCS)
 LOGGING_SRCS = define_xnnpack_build_src(_LOGGING_SRCS)