Cleanup codeobject-interpretation benchmark

Author: msimacek

graalpython/com.oracle.graal.python.benchmarks/python/micro/codeobject-interpretation.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -37,175 +37,66 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 import marshal
-import os
-import sys
-
-
-IS_GRAAL = sys.implementation.name == "graalpython"
-DIR0 = os.path.dirname(__file__)
-
-
-if IS_GRAAL:
-    get_code = lambda n,s: __graalpython__.compile(s, n, "pyc")
-else:
-    def get_code(n,s):
-        c = compile(s,n,"exec")
-        import dis
-        dis.dis(c)
-        return c
-
+import pydoc_data.topics
 
-CODE = get_code("bench.py", """
+# pollute the profile of the bytecode loop
+exec("""
 import sys
-def foo():
-  pass
+def foo(): pass
 len(sys.__name__)
 print(sys, flush=False)
 pass
 """)
 
-# pollute the profile of the bytecode loop
-exec(CODE)
-
-
-# We're making sure that each code object is a separate call target. This is to
-# simulate we are loading SIMULATED_FILECOUNT modules, because each module is
-# potentially a lot of code but only gets executed once
-RETURN_NONE = [100, 0, 83, 0]
-BYTECODE_COUNT = 200
-SIMULATED_FILECOUNT = 1000
-CODE = []
-
-
-def generate_code(name, code_to_repeat, **kwargs):
-    filename = os.path.join(DIR0, name)
-    if not IS_GRAAL and False:
-        # generate our code files only with cpython
-        def foo(): pass
-        code = foo.__code__
-        cnt = int(2 * BYTECODE_COUNT / len(code_to_repeat))
-        newcode = code.replace(co_code=bytes(bytearray(code_to_repeat * cnt + RETURN_NONE)), **kwargs)
-        with open(filename, "wb") as f:
-            marshal.dump(newcode, f)
-    # pollute the profile for the bytecode loop
-    with open(filename, "rb") as f:
-        exec(marshal.load(f))
-    return filename
-
-
-BYTECODE_FILES = [
-    generate_code("nop", [9, 0]),
-    generate_code("pushpop", [100, 0, 1, 0]),
-    generate_code("negative_one", [100, 1, 11, 0, 1, 0], co_consts=(None, 1,)),
-    generate_code("load_fast", [
-        100, 0, # load None
-        125, 0, # store fast 0
-        124, 0, # load fast 0
-        1, 0, # pop top
-    ], co_varnames=('x',)),
-]
-
-
-
-for _ in range(0, SIMULATED_FILECOUNT, len(BYTECODE_FILES)):
-    for filename in BYTECODE_FILES:
-        with open(filename, "rb") as f:
-            CODE.append(marshal.load(f))
-
-
 CODESTR1 = "\n".join(["""
 # import sys
 def foo(): pass
 len(foo.__name__)
 """] * 100)
+
 CODESTR2 = "\n".join(["""
 def bar(): pass
 x = None
 len([])
 y = x
 """] * 100)
-import pydoc_data.topics
+
 with open(pydoc_data.topics.__file__, "r") as f:
     CODESTR3 = f.read()
-TEST_WITH_BYTECODE = False # False to test with AST
-if IS_GRAAL and TEST_WITH_BYTECODE:
-    JUST_PYC_1 = __graalpython__.compile(CODESTR1, "1", "pyc-nocompile")
-    JUST_PYC_2 = __graalpython__.compile(CODESTR1, "2", "pyc-nocompile")
-    JUST_PYC_3 = __graalpython__.compile(CODESTR3, pydoc_data.topics.__file__, "pyc-nocompile")
-else:
-    JUST_PYC_1 = marshal.dumps(compile(CODESTR1, "1", "exec"))
-    JUST_PYC_2 = marshal.dumps(compile(CODESTR2, "2", "exec"))
-    JUST_PYC_3 = marshal.dumps(compile(CODESTR3, pydoc_data.topics.__file__, "exec"))
 
+JUST_PYC_1 = marshal.dumps(compile(CODESTR1, "1", "exec"))
+JUST_PYC_2 = marshal.dumps(compile(CODESTR2, "2", "exec"))
+JUST_PYC_3 = marshal.dumps(compile(CODESTR3, pydoc_data.topics.__file__, "exec"))
 
-MORE_CODEOBJECTS = []
+CODEOBJECTS = []
 
 
 def __setup__(num):
     __cleanup__(num)
 
 
 def __cleanup__(num):
-    import time
-    s = time.time()
-    MORE_CODEOBJECTS.clear()
+    CODEOBJECTS.clear()
     for _ in range(0, num, 3):
-        MORE_CODEOBJECTS.append(marshal.loads(JUST_PYC_1))
-        MORE_CODEOBJECTS.append(marshal.loads(JUST_PYC_2))
-        MORE_CODEOBJECTS.append(marshal.loads(JUST_PYC_3))
-    print(f"Resetting {len(MORE_CODEOBJECTS)} took {time.time() - s} seconds")
-
-
-BYTECODE_FILE_DATA = []
-for filename in BYTECODE_FILES:
-    with open(filename, "rb") as f:
-        BYTECODE_FILE_DATA.append(f.read())
+        CODEOBJECTS.append(marshal.loads(JUST_PYC_1))
+        CODEOBJECTS.append(marshal.loads(JUST_PYC_2))
+        CODEOBJECTS.append(marshal.loads(JUST_PYC_3))
 
 
 def measure(num):
     for i in range(num):
-        # Enable this to benchmark GraalPython code deserialization SST vs
-        # bytecode. Switch out the mode in the global setup above
-        # marshal.loads(JUST_PYC_1); marshal.loads(JUST_PYC_2); marshal.loads(JUST_PYC_3)
-
-        # Enable this to measure executing different modules in AST vs bytecode
-        exec(MORE_CODEOBJECTS[i])
+        exec(CODEOBJECTS[i])
 
-        # Enable this to measure just unmarshalling
-        # marshal.loads(BYTECODE_FILE_DATA[i % len(BYTECODE_FILE_DATA)])
 
-        # Enable this to measure just loading file data
-        # with open(BYTECODE_FILES[i % len(BYTECODE_FILES)], "rb") as f:
-        #     f.read()
-
-        # Enable this to measure loading all the modules
-        # with open(BYTECODE_FILES[i % len(BYTECODE_FILES)], "rb") as f:
-        #     marshal.loads(f.read())
-
-        # Enable this to measure executing different modules
-        # exec(CODE[i % len(CODE)])
-
-        # Enable this to measure unmarshalling and executing code
-        # exec(marshal.loads(BYTECODE_FILE_DATA[i % len(BYTECODE_FILE_DATA)]))
-
-        # Enable this to measure all three, reading file, loading data, and executing
-        # with open(BYTECODE_FILES[i % len(BYTECODE_FILES)], "rb") as f:
-        #     exec(marshal.loads(f.read()))
-
-
-def __benchmark__(num=10_000):
+def __benchmark__(num=2000):
     measure(num)
 
-
-print("Benchmark file loaded")
-
-
 # I've written the bytecode benchmark to simulate loading thousands of modules. I
-# always execute 10_000 iterations, to make it somewhat like loading a big
+# always execute thousands of iterations, to make it somewhat like loading a big
 # project with many pyc files, but not so large that the compiler would have time
 # to compile many of the operations involved in loading code.
 #
-# If we compare unmarshaling a code object from a bytes that has bytecode vs one
+# If we compare unmarshalling a code object from a bytes that has bytecode vs one
 # that has SST, we see that unmarshalling 10_000 bytecode code objects is 2-3x
 # faster than unmarshalling the SST. This is for two bits of code with 300 lines
 # of statements (creating functions, imports, assignments, calling
@@ -227,7 +118,7 @@ def __benchmark__(num=10_000):
 # Opening the files *and* loading the bytecode data yields that CPython is ~14x
 # faster than us.
 #
-# Now, if we preload that artificial code before the benchmark and just execut
+# Now, if we preload that artificial code before the benchmark and just execute
 # those code objects 10_000 times, CPython is a whopping 20-30x faster than we
 # are. But the numbers are so small, it's hard to say (CPython 0.006-0.008s,
 # Graal 0.16-0.19s). OTOH, it's hard to argue that we would ever load more

mx.graalpython/mx_graalpython_bench_param.py

@@ -106,7 +106,7 @@
     'tuple-indexing-from-constructor': ITER_10 + ['10000000'],
     'tuple-indexing-from-literal': ITER_10 + ['10000000'],
     'repeated-import': ITER_10 + ['10000000'],
-    'codeobject-interpretation': ITER_100 + ['10000'],
+    'codeobject-interpretation': ITER_10 + ['2000'],
 }
 
 MICRO_BENCHMARKS_SMALL = {