diff --git a/pkg/config/config.go b/pkg/config/config.go
index 81ea3ea..be33051 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -317,6 +317,65 @@ func getFromPrunerConfigResourceLevelwithSelector(namespacesSpec map[string]Name
 	return nil, ""
 }
 
+// getMatchingSelectorFromConfig retrieves the ConfigMap's selector that matches a resource
+func getMatchingSelectorFromConfig(namespacesSpec map[string]NamespaceSpec, namespace, name string, selector SelectorSpec, resourceType PrunerResourceType) *SelectorSpec {
+	prunerResourceSpec, found := namespacesSpec[namespace]
+	if !found {
+		return nil
+	}
+
+	var resourceSpecs []ResourceSpec
+	switch resourceType {
+	case PrunerResourceTypePipelineRun:
+		resourceSpecs = prunerResourceSpec.PipelineRuns
+	case PrunerResourceTypeTaskRun:
+		resourceSpecs = prunerResourceSpec.TaskRuns
+	}
+
+	if len(selector.MatchAnnotations) == 0 && len(selector.MatchLabels) == 0 {
+		return nil
+	}
+
+	for _, resourceSpec := range resourceSpecs {
+		for _, selectorSpec := range resourceSpec.Selector {
+			annotationsMatch := true
+			labelsMatch := true
+
+			if len(selectorSpec.MatchAnnotations) > 0 {
+				if len(selector.MatchAnnotations) == 0 {
+					annotationsMatch = false
+				} else {
+					for key, value := range selectorSpec.MatchAnnotations {
+						if resourceAnnotationValue, exists := selector.MatchAnnotations[key]; !exists || resourceAnnotationValue != value {
+							annotationsMatch = false
+							break
+						}
+					}
+				}
+			}
+
+			if len(selectorSpec.MatchLabels) > 0 {
+				if len(selector.MatchLabels) == 0 {
+					labelsMatch = false
+				} else {
+					for key, value := range selectorSpec.MatchLabels {
+						if resourceLabelValue, exists := selector.MatchLabels[key]; !exists || resourceLabelValue != value {
+							labelsMatch = false
+							break
+						}
+					}
+				}
+			}
+
+			if annotationsMatch && labelsMatch {
+				return &selectorSpec
+			}
+		}
+	}
+
+	return nil
+}
+
 // getResourceFieldData retrieves configuration field values based on enforcedConfigLevel
 // Design principle: Selector support ONLY for namespace-level ConfigMaps, NOT global ConfigMaps
 //
@@ -650,6 +709,20 @@ func (ps *prunerConfigStore) GetTaskFailedHistoryLimitCount(namespace, name stri
 	return getResourceFieldData(ps.globalConfig, ps.namespaceConfig, namespace, name, selector, PrunerResourceTypeTaskRun, PrunerFieldTypeFailedHistoryLimit, enforcedConfigLevel)
 }
 
+// GetPipelineMatchingSelector returns the ConfigMap's selector that matches a PipelineRun.
+func (ps *prunerConfigStore) GetPipelineMatchingSelector(namespace, name string, selector SelectorSpec) *SelectorSpec {
+	ps.mutex.RLock()
+	defer ps.mutex.RUnlock()
+	return getMatchingSelectorFromConfig(ps.namespaceConfig, namespace, name, selector, PrunerResourceTypePipelineRun)
+}
+
+// GetTaskMatchingSelector returns the ConfigMap's selector that matches a TaskRun.
+func (ps *prunerConfigStore) GetTaskMatchingSelector(namespace, name string, selector SelectorSpec) *SelectorSpec {
+	ps.mutex.RLock()
+	defer ps.mutex.RUnlock()
+	return getMatchingSelectorFromConfig(ps.namespaceConfig, namespace, name, selector, PrunerResourceTypeTaskRun)
+}
+
 // ValidateGlobalConfig validates a GlobalConfig struct directly without ConfigMap conversion
 // This is a convenience function for validating global config and all nested namespace configs
 // without the overhead of serialization/deserialization through ConfigMaps.
diff --git a/pkg/config/history_limiter.go b/pkg/config/history_limiter.go
index a2aa646..808f00d 100644
--- a/pkg/config/history_limiter.go
+++ b/pkg/config/history_limiter.go
@@ -50,6 +50,7 @@ type HistoryLimiterResourceFuncs interface {
 	IsCompleted(resource metav1.Object) bool
 	GetDefaultLabelKey() string
 	GetEnforcedConfigLevel(namespace, name string, selectors SelectorSpec) EnforcedConfigLevel
+	GetMatchingSelector(namespace, name string, selectors SelectorSpec) *SelectorSpec
 }
 
 // HistoryLimiter is a struct that encapsulates functionality for managing resources
@@ -276,15 +277,45 @@ func (hl *HistoryLimiter) doResourceCleanup(ctx context.Context, resource metav1
 		label := fmt.Sprintf("%s=%s", labelKey, resourceName)
 		resources, err = hl.resourceFn.List(ctx, resource.GetNamespace(), label)
 	case "identifiedBy_resource_selector":
-		// Filter by selector labels (namespace-level enforcement with selectors)
+		// Filter by the ConfigMap's selector labels only
+		matchingSelector := hl.resourceFn.GetMatchingSelector(resource.GetNamespace(), resourceName, resourceSelectors)
 		labelSelector := ""
-		for k, v := range resourceLabels {
-			if labelSelector != "" {
-				labelSelector += ","
+		if matchingSelector != nil && len(matchingSelector.MatchLabels) > 0 {
+			for k, v := range matchingSelector.MatchLabels {
+				if labelSelector != "" {
+					labelSelector += ","
+				}
+				labelSelector += fmt.Sprintf("%s=%s", k, v)
 			}
-			labelSelector += fmt.Sprintf("%s=%s", k, v)
 		}
+		logger.Debugw("listing resources with selector from ConfigMap",
+			"resource", hl.resourceFn.Type(),
+			"namespace", resource.GetNamespace(),
+			"labelSelector", labelSelector)
 		resources, err = hl.resourceFn.List(ctx, resource.GetNamespace(), labelSelector)
+		if err != nil {
+			return err
+		}
+		if matchingSelector != nil && len(matchingSelector.MatchAnnotations) > 0 {
+			filteredResources := []metav1.Object{}
+			for _, res := range resources {
+				resAnnotations := res.GetAnnotations()
+				if resAnnotations == nil {
+					continue
+				}
+				matches := true
+				for k, v := range matchingSelector.MatchAnnotations {
+					if resAnnotations[k] != v {
+						matches = false
+						break
+					}
+				}
+				if matches {
+					filteredResources = append(filteredResources, res)
+				}
+			}
+			resources = filteredResources
+		}
 	case "identifiedBy_resource_ann":
 		// Filter by annotations (converted to labels for listing)
 		labelSelector := ""
diff --git a/pkg/config/history_limiter_test.go b/pkg/config/history_limiter_test.go
index deddacf..a187a8f 100644
--- a/pkg/config/history_limiter_test.go
+++ b/pkg/config/history_limiter_test.go
@@ -99,6 +99,10 @@ func (m *mockResourceFuncs) GetEnforcedConfigLevel(_, _ string, _ SelectorSpec)
 	return m.enforceLevel
 }
 
+func (m *mockResourceFuncs) GetMatchingSelector(_, _ string, _ SelectorSpec) *SelectorSpec {
+	return nil // Return nil to list all resources in namespace for tests
+}
+
 func TestNewHistoryLimiter(t *testing.T) {
 	tests := []struct {
 		name       string
diff --git a/pkg/reconciler/pipelinerun/reconciler.go b/pkg/reconciler/pipelinerun/reconciler.go
index 1d958bc..cf36b6b 100644
--- a/pkg/reconciler/pipelinerun/reconciler.go
+++ b/pkg/reconciler/pipelinerun/reconciler.go
@@ -452,3 +452,8 @@ func (prf *PrFuncs) GetFailedHistoryLimitCount(namespace, name string, selectors
 func (prf *PrFuncs) GetEnforcedConfigLevel(namespace, name string, selectors config.SelectorSpec) config.EnforcedConfigLevel {
 	return config.PrunerConfigStore.GetPipelineEnforcedConfigLevel(namespace, name, selectors)
 }
+
+// GetMatchingSelector returns the ConfigMap's selector that matches a PipelineRun.
+func (prf *PrFuncs) GetMatchingSelector(namespace, name string, selectors config.SelectorSpec) *config.SelectorSpec {
+	return config.PrunerConfigStore.GetPipelineMatchingSelector(namespace, name, selectors)
+}
diff --git a/pkg/reconciler/taskrun/reconciler.go b/pkg/reconciler/taskrun/reconciler.go
index ba773db..056871e 100644
--- a/pkg/reconciler/taskrun/reconciler.go
+++ b/pkg/reconciler/taskrun/reconciler.go
@@ -444,3 +444,8 @@ func (trf *TrFuncs) GetFailedHistoryLimitCount(namespace, name string, selectors
 func (trf *TrFuncs) GetEnforcedConfigLevel(namespace, name string, selectors config.SelectorSpec) config.EnforcedConfigLevel {
 	return config.PrunerConfigStore.GetTaskEnforcedConfigLevel(namespace, name, selectors)
 }
+
+// GetMatchingSelector returns the ConfigMap's selector that matches a TaskRun.
+func (trf *TrFuncs) GetMatchingSelector(namespace, name string, selectors config.SelectorSpec) *config.SelectorSpec {
+	return config.PrunerConfigStore.GetTaskMatchingSelector(namespace, name, selectors)
+}