diff --git a/bpf/process/policy_filter.h b/bpf/process/policy_filter.h
index 2baa916bde7..a8d252e4b97 100644
--- a/bpf/process/policy_filter.h
+++ b/bpf/process/policy_filter.h
@@ -20,7 +20,7 @@ struct {
 
 struct {
 	__uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
-	__uint(max_entries, POLICY_FILTER_MAX_POLICIES);
+	__uint(max_entries, 1); // will be resized by agent when needed
 	__type(key, u32); /* policy id */
 	__array(
 		values, struct {
diff --git a/docs/data/tetragon_flags.yaml b/docs/data/tetragon_flags.yaml
index 35432c615d3..7eb1d09798b 100644
--- a/docs/data/tetragon_flags.yaml
+++ b/docs/data/tetragon_flags.yaml
@@ -203,6 +203,10 @@ options:
     - name: netns-dir
       default_value: /var/run/docker/netns/
       usage: Network namespace dir
+    - name: policy-filter-map-entries
+      default_value: "128"
+      usage: |
+        Set maximum number of policies in policy_filter_maps. This map restricts tracing policies to specific pods/containers. Increase if you have many policies, decrease to save memory if you have few policies.
     - name: pprof-address
       usage: |
         Serves runtime profile data via HTTP (e.g. 'localhost:6060'). Disabled by default
diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go
index 834b146b7d6..f765ed513a9 100644
--- a/pkg/defaults/defaults.go
+++ b/pkg/defaults/defaults.go
@@ -59,6 +59,9 @@ const (
 
 	// defaults for the {k,u}retprobes lru cache
 	DefaultRetprobesCacheSize = 4096
+
+	// defaults for the policy filter map
+	DefaultPolicyFilterMapEntries = 128
 )
 
 var (
diff --git a/pkg/defaults/defaults_windows.go b/pkg/defaults/defaults_windows.go
index f4eb7d90a67..bd7043750a0 100644
--- a/pkg/defaults/defaults_windows.go
+++ b/pkg/defaults/defaults_windows.go
@@ -51,4 +51,7 @@ const (
 
 	// defaults for the {k,u}retprobes lru cache
 	DefaultRetprobesCacheSize = 4096
+
+	// defaults for the policy filter map
+	DefaultPolicyFilterMapEntries = 128
 )
diff --git a/pkg/option/config.go b/pkg/option/config.go
index dc24e587c98..f52f1217e17 100644
--- a/pkg/option/config.go
+++ b/pkg/option/config.go
@@ -134,6 +134,8 @@ type config struct {
 	ExecveMapSize    string
 
 	RetprobesCacheSize int
+
+	PolicyFilterMapEntries int
 }
 
 var (
@@ -159,6 +161,9 @@ var (
 
 		// Set default value for {k,u}retprobes lru events cache
 		RetprobesCacheSize: defaults.DefaultRetprobesCacheSize,
+
+		// set default value for the policy filter map
+		PolicyFilterMapEntries: defaults.DefaultPolicyFilterMapEntries,
 	}
 )
 
diff --git a/pkg/option/flags.go b/pkg/option/flags.go
index f37b41fa60e..49a15385151 100644
--- a/pkg/option/flags.go
+++ b/pkg/option/flags.go
@@ -139,6 +139,8 @@ const (
 	KeyExecveMapSize    = "execve-map-size"
 
 	KeyRetprobesCacheSize = "retprobes-cache-size"
+
+	KeyPolicyFilterMapEntries = "policy-filter-map-entries"
 )
 
 type UsernameMetadaCode int
@@ -305,6 +307,8 @@ func ReadAndSetFlags() error {
 	Config.ExecveMapSize = viper.GetString(KeyExecveMapSize)
 
 	Config.RetprobesCacheSize = viper.GetInt(KeyRetprobesCacheSize)
+
+	Config.PolicyFilterMapEntries = viper.GetInt(KeyPolicyFilterMapEntries)
 	return nil
 }
 
@@ -504,4 +508,6 @@ func AddFlags(flags *pflag.FlagSet) {
 	flags.String(KeyExecveMapSize, "", "Set size for execve_map table (allows K/M/G suffix)")
 
 	flags.Int(KeyRetprobesCacheSize, defaults.DefaultRetprobesCacheSize, "Set {k,u}retprobes events cache maximum size")
+
+	flags.Int(KeyPolicyFilterMapEntries, defaults.DefaultPolicyFilterMapEntries, "Set maximum number of policies in policy_filter_maps. This map restricts tracing policies to specific pods/containers. Increase if you have many policies, decrease to save memory if you have few policies.")
 }
diff --git a/pkg/policyfilter/map.go b/pkg/policyfilter/map.go
index b30b26d23e3..bc45c7ad290 100644
--- a/pkg/policyfilter/map.go
+++ b/pkg/policyfilter/map.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/cilium/tetragon/pkg/bpf"
 	"github.com/cilium/tetragon/pkg/config"
+	"github.com/cilium/tetragon/pkg/option"
 )
 
 const (
@@ -72,6 +73,10 @@ func newPfMap(enableCgroupMap bool) (PfMap, error) {
 		return PfMap{}, fmt.Errorf("loading spec for %s failed: %w", objPath, err)
 	}
 
+	if _, ok := spec.Maps["policy_filter_maps"]; ok {
+		spec.Maps["policy_filter_maps"].MaxEntries = uint32(option.Config.PolicyFilterMapEntries)
+	}
+
 	var ret PfMap
 	if ret.policyMap, err = openMap(spec, MapName, polMapSize); err != nil {
 		return PfMap{}, fmt.Errorf("opening map %s failed: %w", MapName, err)
@@ -80,7 +85,7 @@ func newPfMap(enableCgroupMap bool) (PfMap, error) {
 	if enableCgroupMap {
 		if ret.cgroupMap, err = openMap(spec, CgroupMapName, polMaxPolicies); err != nil {
 			releaseMap(ret.policyMap)
-			return PfMap{}, fmt.Errorf("opening cgroup map %s failed: %w", MapName, err)
+			return PfMap{}, fmt.Errorf("opening cgroup map %s failed: %w", CgroupMapName, err)
 		}
 	}
 
diff --git a/pkg/sensors/program/loader_linux.go b/pkg/sensors/program/loader_linux.go
index 5d9e5bdff7b..f6d9ee35222 100644
--- a/pkg/sensors/program/loader_linux.go
+++ b/pkg/sensors/program/loader_linux.go
@@ -18,6 +18,7 @@ import (
 	cachedbtf "github.com/cilium/tetragon/pkg/btf"
 	"github.com/cilium/tetragon/pkg/logger"
 	"github.com/cilium/tetragon/pkg/logger/logfields"
+	"github.com/cilium/tetragon/pkg/option"
 	"github.com/cilium/tetragon/pkg/sensors/unloader"
 )
 
@@ -961,6 +962,11 @@ func doLoadProgram(
 		}
 	}
 
+	// TODO: remove this special case handling (see #4398)
+	if ms, ok := spec.Maps["policy_filter_maps"]; ok {
+		ms.MaxEntries = uint32(option.Config.PolicyFilterMapEntries)
+	}
+
 	// Find all the maps referenced by the program, so we'll rewrite only
 	// the ones used.
 	var progSpec *ebpf.ProgramSpec