Enable Bytecode regen on Linux/MacOS (#6425)

Introduce a Force32BitByteCode flag usable in Debug and Test builds that forces a 64bit CC build to generate 32bit bytecode for the known differences. (All related logic is #ifdef'd out in release builds)

Add a python script that will use that flag to perform the regen on macos & linux.

Author: rhuanjl

RegenAllByteCode.cmd

@@ -4,6 +4,7 @@
 ::-------------------------------------------------------------------------------------------------------
 
 :: Regenerate all bytecode.
+:: Note, this script is windows only, on linux or macOS please use tools/xplatRegenByteCode.py
 :: ch.exe is used to generate Intl bytecodes.
 :: ch.exe (NoJIT variety) is used to generate NoJIT Intl bytecodes.
 :: Each set of bytecode requires an x86_debug and x64_debug binary.

RegenAllByteCodeNoBuild.cmd

@@ -10,6 +10,7 @@
 :: every flavor of ChakraCore.dll when there are no relevant changes is a waste of time.
 :: Please ensure that you use buddy builds to validate the results.
 
+:: Note, this script is windows only, on linux or macOS please use tools/xplatRegenByteCode.py
 :: Regenerate all bytecode (without rebuilding each flavor of ch.exe)
 :: ch.exe is used to generate Intl bytecodes.
 :: ch.exe (NoJIT variety) is used to generate NoJIT Intl bytecodes.

lib/Common/ConfigFlagsList.h

@@ -1101,6 +1101,9 @@ FLAGR(Boolean, SkipSplitOnNoResult, "If the result of Regex split isn't used, sk
 #ifdef TEST_ETW_EVENTS
 FLAGNR(String,  TestEtwDll            , "Path of the TestEtwEventSink DLL", nullptr)
 #endif
+#ifdef ENABLE_TEST_HOOKS
+FLAGNR(Boolean, Force32BitByteCode, "Force CC to generate 32bit bytecode intended only for regenerating bytecode headers.", false)
+#endif
 
 FLAGNR(Boolean, CollectGarbage        , "Enable CollectGarbage API", DEFAULT_CONFIG_CollectGarbage)
 

lib/Runtime/Base/FunctionBody.cpp

@@ -4410,7 +4410,13 @@ namespace Js
     void FunctionBody::RecordIntConstant(RegSlot location, unsigned int val)
     {
         ScriptContext *scriptContext = this->GetScriptContext();
+#ifdef ENABLE_TEST_HOOKS
+        Var intConst = scriptContext->GetConfig()->Force32BitByteCode() ?
+            JavascriptNumber::ToVarFor32BitBytecode((int32)val, scriptContext) :
+            JavascriptNumber::ToVar((int32)val, scriptContext);
+#else
         Var intConst = JavascriptNumber::ToVar((int32)val, scriptContext);
+#endif
         this->RecordConstant(location, intConst);
     }
 

lib/Runtime/Base/ThreadConfigFlagsList.h

@@ -54,6 +54,9 @@ FLAG_RELEASE(IsESExportNsAsEnabled, ESExportNsAs)
 FLAG_RELEASE(IsESSymbolDescriptionEnabled, ESSymbolDescription)
 FLAG_RELEASE(IsESGlobalThisEnabled, ESGlobalThis)
 FLAG_RELEASE(IsES2018AsyncIterationEnabled, ES2018AsyncIteration)
+#ifdef ENABLE_TEST_HOOKS
+FLAG_RELEASE(Force32BitByteCode, Force32BitByteCode)
+#endif
 #ifdef ENABLE_PROJECTION
 FLAG(AreWinRTDelegatesInterfaces, WinRTDelegateInterfaces)
 FLAG_RELEASE(IsWinRTAdaptiveAppsEnabled, WinRTAdaptiveApps)

lib/Runtime/ByteCode/ByteCodeEmitter.cpp

@@ -10140,7 +10140,11 @@ void EmitBinary(Js::OpCode opcode, ParseNode *pnode, ByteCodeGenerator *byteCode
     byteCodeGenerator->EndStatement(pnode);
 }
 
-bool CollectConcat(ParseNode *pnodeAdd, DListCounted<ParseNode *, ArenaAllocator>& concatOpnds, ArenaAllocator *arenaAllocator)
+bool CollectConcat(ParseNode *pnodeAdd, DListCounted<ParseNode *, ArenaAllocator>& concatOpnds, ArenaAllocator *arenaAllocator 
+#ifdef ENABLE_TEST_HOOKS
+    , bool Force32BitByteCode = false
+#endif
+)
 {
     Assert(pnodeAdd->nop == knopAdd);
     Assert(pnodeAdd->CanFlattenConcatExpr());
@@ -10161,7 +10165,18 @@ bool CollectConcat(ParseNode *pnodeAdd, DListCounted<ParseNode *, ArenaAllocator
 
             // Detect if there are any string larger then the append size limit.
             // If there are, we can do concat; otherwise, still use add so we will not lose the AddLeftDead opportunities.
+#ifdef ENABLE_TEST_HOOKS
+            if (Force32BitByteCode)
+            {
+                doConcatString = doConcatString || (pnode->AsParseNodeStr()->pid->Cch() > 4);
+            }
+            else
+            {
+                doConcatString = doConcatString || !Js::CompoundString::ShouldAppendChars(pnode->AsParseNodeStr()->pid->Cch());
+            }
+#else
             doConcatString = doConcatString || !Js::CompoundString::ShouldAppendChars(pnode->AsParseNodeStr()->pid->Cch());
+#endif
         }
         else
         {
@@ -10233,7 +10248,12 @@ void EmitAdd(ParseNode *pnode, ByteCodeGenerator *byteCodeGenerator, FuncInfo *f
         // We should only have a string concat if the feature is on.
         Assert(!PHASE_OFF1(Js::ByteCodeConcatExprOptPhase));
         DListCounted<ParseNode*, ArenaAllocator> concatOpnds(byteCodeGenerator->GetAllocator());
+#ifdef ENABLE_TEST_HOOKS
+        bool doConcatString = CollectConcat(pnode, concatOpnds, byteCodeGenerator->GetAllocator(),
+            byteCodeGenerator->GetScriptContext()->GetConfig()->Force32BitByteCode());
+#else
         bool doConcatString = CollectConcat(pnode, concatOpnds, byteCodeGenerator->GetAllocator());
+#endif
         if (doConcatString)
         {
             uint concatCount = concatOpnds.Count();

lib/Runtime/ByteCode/ByteCodeSerializer.cpp

@@ -488,6 +488,12 @@ class ByteCodeBufferBuilder
         {
             expectedFunctionBodySize.value = 0;
             expectedOpCodeCount.value = 0;
+#ifdef ENABLE_TEST_HOOKS
+            if (scriptContext->GetConfig()->Force32BitByteCode())
+            {
+                architecture.value = 32;
+            }
+#endif
         }
 
         // Library bytecode uses its own scheme

lib/Runtime/Library/JavascriptNumber.h

@@ -37,6 +37,9 @@ namespace Js
         static Var ToVarMaybeInPlace(double value, ScriptContext* scriptContext, JavascriptNumber *result);
         static Var ToVarIntCheck(double value, ScriptContext* scriptContext);
         static Var ToVar(int32 nValue, ScriptContext* scriptContext);
+#ifdef ENABLE_TEST_HOOKS
+        static Var ToVarFor32BitBytecode(int32 nValue, ScriptContext* scriptContext);
+#endif
 #if defined(__clang__) && defined(_M_IX86)
         static Var ToVar(intptr_t nValue, ScriptContext* scriptContext);
 #endif

lib/Runtime/Library/JavascriptNumber.inl

@@ -41,6 +41,17 @@ namespace Js
         }
     }
 
+#ifdef ENABLE_TEST_HOOKS
+    __forceinline Var JavascriptNumber::ToVarFor32BitBytecode(int32 nValue, ScriptContext* scriptContext)
+    {
+        if ((1073741824 > nValue) && (nValue > -1073741824))
+        {
+            return TaggedInt::ToVarUnchecked(nValue);
+        }
+        return JavascriptNumber::NewInlined((double) nValue, scriptContext);
+    }
+#endif
+
 #if defined(__clang__) && defined(_M_IX86)
     __forceinline Var JavascriptNumber::ToVar(intptr_t nValue, ScriptContext* scriptContext)
     {

tools/xplatRegenByteCode.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+#-------------------------------------------------------------------------------------------------------
+# Copyright (C) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+#-------------------------------------------------------------------------------------------------------
+
+# Regenerate embedded bytecode headers.
+# NOTEs:
+# 1. this script is for linux and macOS only, on windows please use RegenAllByteCode.cmd AND update_bytecode_version.ps1
+# 2. this script uses paths relative to the tools directory it's in so cd to it before running
+# 3. this script relies on forcing 64bit CC builds to produce 32bit bytecode - this could break due to future changes to CC.
+#    If this facility breaks the CI will fail AND either this will need fixing or bytecode will need to be regenerated using
+#    32 bit builds on windows
+# 4. Run with flag '--skip-build' if the necessary versions of ChakraCore are already built to skip the compilation step
+#    (this is useful if editing the .js files that the bytecode is produced from)
+# Two versions of CC and ch must be compiled for bytecode generation 
+# ch is used to generate bytecode.
+# ch (NoJIT variety) is used to generate NoJIT bytecodes.
+
+import subprocess
+import sys
+import uuid
+
+# Compile ChakraCore both noJit and Jit variants
+def run_sub(message, commands, error):
+    print(message)
+    sub = subprocess.Popen(commands)
+    sub.wait()
+    if sub.returncode != 0:
+        sys.exit(error)
+
+if len(sys.argv) == 1 or sys.argv[1] != '--skip-build':
+    run_sub('Compiling ChakraCore with no Jit',
+        ['../build.sh', '--no-jit', '--test-build', '--target-path=../out/noJit', '-j=2'], 
+        'No Jit build failed - aborting bytecode generation')
+    run_sub('Compiling ChakraCore with Jit',
+        ['../build.sh', '--test-build', '--target-path=../out', '-j=2'], 
+        'Jit build failed - aborting bytecode generation')
+
+# Regenerate the bytecode
+def bytecode_job(outPath, command, error):
+    header = open(outPath, 'w')
+    job = subprocess.Popen(command, stdout=header)
+    job.wait()
+    if job.returncode != 0:
+        sys.exit(error)
+
+# INTL
+print('Generating INTL bytecode')
+bytecode_job('../lib/Runtime/Library/InJavascript/Intl.js.nojit.bc.64b.h',
+    ['../out/noJit/test/ch', '-GenerateLibraryByteCodeHeader', '-Intl', '../lib/Runtime/Library/InJavascript/Intl.js'],
+    'Failed to generate INTL 64bit noJit bytecode')
+
+bytecode_job('../lib/Runtime/Library/InJavascript/Intl.js.nojit.bc.32b.h',
+    ['../out/noJit/test/ch', '-GenerateLibraryByteCodeHeader', '-Intl', '-Force32BitByteCode','../lib/Runtime/Library/InJavascript/Intl.js'],
+    'Failed to generate INTL 32bit noJit bytecode')
+
+bytecode_job('../lib/Runtime/Library/InJavascript/Intl.js.bc.64b.h',
+    ['../out/test/ch', '-GenerateLibraryByteCodeHeader', '-Intl', '../lib/Runtime/Library/InJavascript/Intl.js'],
+    'Failed to generate INTL 64bit bytecode')
+
+bytecode_job('../lib/Runtime/Library/InJavascript/Intl.js.bc.32b.h',
+    ['../out/test/ch', '-GenerateLibraryByteCodeHeader', '-Intl', '-Force32BitByteCode','../lib/Runtime/Library/InJavascript/Intl.js'],
+    'Failed to generate INTL 32bit bytecode')
+
+# JsBuiltin
+print('Generating JsBuiltin Bytecode')
+bytecode_job('../lib/Runtime/Library/JsBuiltin/JsBuiltin.js.nojit.bc.64b.h',
+    ['../out/noJit/test/ch', '-GenerateLibraryByteCodeHeader', '-JsBuiltIn', '-LdChakraLib', '../lib/Runtime/Library/JsBuiltin/JsBuiltin.js'],
+    'Failed to generate noJit 64bit JsBuiltin Bytecode')
+
+bytecode_job('../lib/Runtime/Library/JsBuiltin/JsBuiltin.js.nojit.bc.32b.h',
+    ['../out/noJit/test/ch', '-GenerateLibraryByteCodeHeader', '-JsBuiltIn', '-LdChakraLib', '-Force32BitByteCode', '../lib/Runtime/Library/JsBuiltin/JsBuiltin.js'],
+    'Failed to generate noJit 32bit JsBuiltin Bytecode')
+
+bytecode_job('../lib/Runtime/Library/JsBuiltin/JsBuiltin.js.bc.64b.h',
+    ['../out/test/ch', '-GenerateLibraryByteCodeHeader', '-JsBuiltIn', '-LdChakraLib', '../lib/Runtime/Library/JsBuiltin/JsBuiltin.js'],
+    'Failed to generate 64bit JsBuiltin Bytecode')
+
+bytecode_job('../lib/Runtime/Library/JsBuiltin/JsBuiltin.js.bc.32b.h',
+    ['../out/test/ch', '-GenerateLibraryByteCodeHeader', '-JsBuiltIn', '-LdChakraLib', '-Force32BitByteCode', '../lib/Runtime/Library/JsBuiltin/JsBuiltin.js'],
+    'Failed to generate 32bit JsBuiltin Bytecode')
+
+# Bytecode regeneration complete - create a new GUID for it
+print('Generating new GUID for new bytecode')
+guid_header = open('../lib/Runtime/Bytecode/ByteCodeCacheReleaseFileVersion.h', 'w')
+guid = str(uuid.uuid4())
+
+output_str = '''//-------------------------------------------------------------------------------------------------------
+// Copyright (C) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+//-------------------------------------------------------------------------------------------------------
+// NOTE: If there is a merge conflict the correct fix is to make a new GUID.
+// This file was generated with tools/xplatRegenByteCode.py
+
+// {%s}
+const GUID byteCodeCacheReleaseFileVersion =
+{ 0x%s, 0x%s, 0x%s, {0x%s, 0x%s, 0x%s, 0x%s, 0x%s, 0x%s, 0x%s, 0x%s } };
+''' % (guid,
+    guid[:8], guid[9:13], guid[14:18], guid[19:21], guid[21:23], guid[24:26],
+    guid[26:28], guid[28:30], guid[30:32], guid[32:34], guid[-2:])
+
+guid_header.write(output_str)
+
+print('Bytecode successfully regenerated. Please rebuild ChakraCore to incorporate it.')