Revert "Make Add non-blocking"

This reverts commit ce23de5.

Speedup is tiny and at the expense of increased mem usage (which due to
increasing GC pressure is likely the explanation why its so small), so
doesn't seem worth it overall:
```
goos: darwin
goarch: arm64
pkg: sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue
cpu: Apple M2 Pro
                    │ blocking.txt │          non-blocking.txt          │
                    │    sec/op    │   sec/op     vs base               │
AddGetDone-10          1.320m ± 1%   1.410m ± 0%  +6.81% (p=0.000 n=10)
AddOnly-10             373.9µ ± 1%   343.2µ ± 1%  -8.22% (p=0.000 n=10)
AddLockContended-10    375.8µ ± 1%   342.8µ ± 1%  -8.78% (p=0.000 n=10)
geomean                570.3µ        549.4µ       -3.66%

                    │ blocking.txt │            non-blocking.txt             │
                    │     B/op     │     B/op      vs base                   │
AddGetDone-10         109.9Ki ± 0%   164.2Ki ± 0%     +49.42% (p=0.000 n=10)
AddOnly-10              553.0 ± 2%   56045.0 ± 0%  +10034.72% (p=0.000 n=10)
AddLockContended-10     569.0 ± 6%   56045.0 ± 0%   +9749.74% (p=0.000 n=10)
geomean               3.207Ki        78.94Ki        +2361.60%

                    │ blocking.txt │            non-blocking.txt             │
                    │  allocs/op   │  allocs/op    vs base                   │
AddGetDone-10          3.013k ± 0%    5.001k ± 0%     +65.98% (p=0.000 n=10)
AddOnly-10              16.00 ± 6%   2000.00 ± 0%  +12400.00% (p=0.000 n=10)
AddLockContended-10     16.00 ± 6%   2000.00 ± 0%  +12400.00% (p=0.000 n=10)
geomean                 91.71         2.715k        +2860.01%
```

Author: alvaroaleman

pkg/controller/priorityqueue/priorityqueue.go

@@ -60,8 +60,6 @@ func New[T comparable](name string, o ...Opt[T]) PriorityQueue[T] {
 		items:   map[T]*item[T]{},
 		queue:   btree.NewG(32, less[T]),
 		metrics: newQueueMetrics[T](opts.MetricProvider, name, clock.RealClock{}),
-		// Use a buffered channel to try and not block callers
-		adds: make(chan *itemsWithOpts[T], 1000),
 		// itemOrWaiterAdded indicates that an item or
 		// waiter was added. It must be buffered, because
 		// if we currently process items we can't tell
@@ -90,7 +88,6 @@ type priorityqueue[T comparable] struct {
 	items   map[T]*item[T]
 	queue   bTree[*item[T]]
 	metrics queueMetrics[T]
-	adds    chan *itemsWithOpts[T]
 
 	// addedCounter is a counter of elements added, we need it
 	// because unixNano is not guaranteed to be unique.
@@ -120,67 +117,54 @@ type priorityqueue[T comparable] struct {
 }
 
 func (w *priorityqueue[T]) AddWithOpts(o AddOpts, items ...T) {
-	w.adds <- &itemsWithOpts[T]{opts: o, items: items}
-}
-
-func (w *priorityqueue[T]) drainAddsLocked(added *itemsWithOpts[T]) {
-	for {
-		for _, key := range added.items {
-			if added.opts.RateLimited {
-				after := w.rateLimiter.When(key)
-				if added.opts.After == 0 || after < added.opts.After {
-					added.opts.After = after
-				}
-			}
-
-			var readyAt *time.Time
-			if added.opts.After > 0 {
-				readyAt = ptr.To(w.now().Add(added.opts.After))
-				w.metrics.retry()
-			}
-			if _, ok := w.items[key]; !ok {
-				item := &item[T]{
-					key:             key,
-					addedAtUnixNano: w.now().UnixNano(),
-					addedCounter:    w.addedCounter,
-					priority:        added.opts.Priority,
-					readyAt:         readyAt,
-				}
-				w.items[key] = item
-				w.queue.ReplaceOrInsert(item)
-				w.metrics.add(key)
-				w.addedCounter++
-				continue
-			}
+	w.lock.Lock()
+	defer w.lock.Unlock()
 
-			// The b-tree de-duplicates based on ordering and any change here
-			// will affect the order - Just delete and re-add.
-			item, _ := w.queue.Delete(w.items[key])
-			if added.opts.Priority > item.priority {
-				item.priority = added.opts.Priority
+	for _, key := range items {
+		if o.RateLimited {
+			after := w.rateLimiter.When(key)
+			if o.After == 0 || after < o.After {
+				o.After = after
 			}
+		}
 
-			if item.readyAt != nil && (readyAt == nil || readyAt.Before(*item.readyAt)) {
-				item.readyAt = readyAt
+		var readyAt *time.Time
+		if o.After > 0 {
+			readyAt = ptr.To(w.now().Add(o.After))
+			w.metrics.retry()
+		}
+		if _, ok := w.items[key]; !ok {
+			item := &item[T]{
+				key:             key,
+				addedAtUnixNano: w.now().UnixNano(),
+				addedCounter:    w.addedCounter,
+				priority:        o.Priority,
+				readyAt:         readyAt,
 			}
-
+			w.items[key] = item
 			w.queue.ReplaceOrInsert(item)
+			w.metrics.add(key)
+			w.addedCounter++
+			continue
 		}
 
-		// Drain the remainder of the channel. Has to be at the end,
-		// because the first item is read in spin() and passed on
-		// to us.
-		select {
-		case added = <-w.adds:
-		default:
-			return
+		// The b-tree de-duplicates based on ordering and any change here
+		// will affect the order - Just delete and re-add.
+		item, _ := w.queue.Delete(w.items[key])
+		if o.Priority > item.priority {
+			item.priority = o.Priority
+		}
+
+		if item.readyAt != nil && (readyAt == nil || readyAt.Before(*item.readyAt)) {
+			item.readyAt = readyAt
 		}
+
+		w.queue.ReplaceOrInsert(item)
 	}
-}
 
-type itemsWithOpts[T comparable] struct {
-	opts  AddOpts
-	items []T
+	if len(items) > 0 {
+		w.notifyItemOrWaiterAdded()
+	}
 }
 
 func (w *priorityqueue[T]) notifyItemOrWaiterAdded() {
@@ -195,13 +179,11 @@ func (w *priorityqueue[T]) spin() {
 	var nextReady <-chan time.Time
 	nextReady = blockForever
 
-	var addedItem *itemsWithOpts[T]
 	for {
 		select {
 		case <-w.done:
 			return
 		case <-w.itemOrWaiterAdded:
-		case addedItem = <-w.adds:
 		case <-nextReady:
 		}
 
@@ -211,11 +193,6 @@ func (w *priorityqueue[T]) spin() {
 			w.lock.Lock()
 			defer w.lock.Unlock()
 
-			if addedItem != nil {
-				w.drainAddsLocked(addedItem)
-			}
-			addedItem = nil
-
 			w.lockedLock.Lock()
 			defer w.lockedLock.Unlock()