arglist: optimize flush_pre_post(), and __iadd__() with it

Unless an argument is marked as Dedup.OVERRIDDEN, pre_flush_set and
post_flush_set will always be empty and the loops in flush_pre_post()
will not be doing anything interesting:

        for a in self.pre:
            dedup = self._can_dedup(a)
            if a not in pre_flush_set:
                # This just makes new a copy of self.pre
                new.append(a)
                if dedup is Dedup.OVERRIDDEN:
                    # this never happens
                    pre_flush_set.add(a)

        for a in reversed(self.post):
            dedup = self._can_dedup(a)
            if a not in post_flush_set:
                # Here self.post is reversed twice
                post_flush.appendleft(a)
                if dedup is Dedup.OVERRIDDEN:
                    # this never happens
                    post_flush_set.add(a)
        new.extend(post_flush)

In this case it's possible to avoid expensive calls and loops, instead
relying as much on Python builtins as possible.  Track whether any options
have that flag and if not just concatenate pre, _container and post.

Before:

   ncalls  tottime  cumtime
    45127    0.251    4.530 arglist.py:142(__iter__)
    81866    3.623    5.013 arglist.py:108(flush_pre_post)
    76618    3.793    5.338 arglist.py:273(__iadd__)

After:

    35647    0.156    0.627 arglist.py:160(__iter__)
    78998    2.627    3.603 arglist.py:116(flush_pre_post)
    73774    3.605    5.049 arglist.py:292(__iadd__)

The time in __iadd__ is reduced because it calls __iter__, which flushes
pre and post.

Signed-off-by: Paolo Bonzini <[email protected]>

Author: bonzini

mesonbuild/arglist.py

@@ -104,14 +104,25 @@ def __init__(self, compiler: T.Union['Compiler', 'StaticLinker'],
             # pass the underlying list to list() directly, instead of an iterator
             iterable = iterable._container
         self._container: T.List[str] = list(iterable) if iterable is not None else []
+
         self.pre: T.Deque[str] = collections.deque()
         self.post: T.Deque[str] = collections.deque()
+        self.needs_override_check: bool = False
 
     # Flush the saved pre and post list into the _container list
     #
     # This correctly deduplicates the entries after _can_dedup definition
     # Note: This function is designed to work without delete operations, as deletions are worsening the performance a lot.
     def flush_pre_post(self) -> None:
+        if not self.needs_override_check:
+            if self.pre:
+                self._container[0:0] = self.pre
+                self.pre.clear()
+            if self.post:
+                self._container.extend(self.post)
+                self.post.clear()
+            return
+
         new: T.List[str] = []
         pre_flush_set: T.Set[str] = set()
         post_flush: T.Deque[str] = collections.deque()
@@ -133,17 +144,15 @@ def flush_pre_post(self) -> None:
 
         #pre and post will overwrite every element that is in the container
         #only copy over args that are in _container but not in the post flush or pre flush set
-        if pre_flush_set or post_flush_set:
-            for a in self._container:
-                if a not in post_flush_set and a not in pre_flush_set:
-                    new.append(a)
-        else:
-            new.extend(self._container)
+        for a in self._container:
+            if a not in post_flush_set and a not in pre_flush_set:
+                new.append(a)
         new.extend(post_flush)
 
         self._container = new
         self.pre.clear()
         self.post.clear()
+        self.needs_override_check = False
 
     def __iter__(self) -> T.Iterator[str]:
         # see also __init__, where this method is essentially inlined
@@ -295,6 +304,8 @@ def __iadd__(self, args: T.Iterable[str]) -> 'CompilerArgs':
                 # Argument already exists and adding a new instance is useless
                 if arg in self._container or arg in self.pre or arg in self.post:
                     continue
+            elif dedup is Dedup.OVERRIDDEN:
+                self.needs_override_check = True
             if self._should_prepend(arg):
                 tmp_pre.appendleft(arg)
             else: