Merge pull request #164 from jgravelle-google/waterfall_lkgr

Add upstream waterfall tools and sdk

Author: juj

.gitignore

@@ -24,3 +24,5 @@ llvm-nightlies-32bit.txt
 llvm-nightlies-64bit.txt
 llvm-tags-32bit.txt
 llvm-tags-64bit.txt
+upstream
+!upstream/lkgr.json

emsdk

@@ -1347,7 +1347,7 @@ class Tool:
         success = build_llvm_tool(self)
       elif hasattr(self, 'git_branch'):
         success = git_clone_checkout_and_pull(url, self.installation_path(), self.git_branch)
-      elif url.endswith('zip') or url.endswith('.tar') or url.endswith('.gz') or url.endswith('.xz'):
+      elif url.endswith('zip') or url.endswith('.tar') or url.endswith('.gz') or url.endswith('.xz') or url.endswith('.tbz2'):
         download_even_if_exists = (self.id == 'vs-tool')
         success = download_and_unzip(url, self.installation_path(), download_even_if_exists, filename_prefix=self.zipfile_prefix if hasattr(self, 'zipfile_prefix') else '')
       else:
@@ -1577,6 +1577,7 @@ def fetch_emscripten_tags():
   print('Fetching all precompiled tagged releases..')
   download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name_for_llvm_location() + '_32bit/index.txt', 'llvm-tags-32bit.txt', download_even_if_exists=True)
   download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name_for_llvm_location() + '_64bit/index.txt', 'llvm-tags-64bit.txt', download_even_if_exists=True)
+  download_waterfall_lkgr()
 
   if not git:
     print('Update complete, however skipped fetching the Emscripten tags, since git was not found.')
@@ -1659,6 +1660,18 @@ def load_llvm_precompiled_tags_32bit():
 def load_llvm_precompiled_tags_64bit():
   return load_file_index_list('llvm-tags-64bit.txt')
 
+def download_waterfall_lkgr():
+  lkgr_url = 'https://storage.googleapis.com/wasm-llvm/builds/linux/lkgr.json'
+  download_file(lkgr_url, 'upstream', download_even_if_exists=True)
+
+def load_waterfall_lkgr():
+  try:
+    data = json.loads(open(os.path.join('upstream', 'lkgr.json'), 'r').read())
+    lkgr = data['build']
+    return [lkgr]
+  except:
+    return []
+
 def is_string(s):
   if (sys.version_info[0] >= 3): return isinstance(s, str)
   return isinstance(s, basestring)
@@ -1680,6 +1693,7 @@ def load_sdk_manifest():
   llvm_32bit_nightlies = list(reversed(load_llvm_32bit_nightlies()))
   llvm_64bit_nightlies = list(reversed(load_llvm_64bit_nightlies()))
   emscripten_nightlies = list(reversed(load_emscripten_nightlies()))
+  waterfall_lkgr = load_waterfall_lkgr()
 
   def dependencies_exist(sdk):
     for tool_name in sdk.uses:
@@ -1750,6 +1764,7 @@ def load_sdk_manifest():
       elif '%nightly-llvm-64bit%' in t.version: expand_category_param('%nightly-llvm-64bit%', llvm_64bit_nightlies, t, is_sdk=False)
       elif '%nightly-llvm-32bit%' in t.version: expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, t, is_sdk=False)
       elif '%nightly-emscripten%' in t.version: expand_category_param('%nightly-emscripten%', emscripten_nightlies, t, is_sdk=False)
+      elif '%waterfall-lkgr%' in t.version: expand_category_param('%waterfall-lkgr%', waterfall_lkgr, t, is_sdk=False)
       else:
         add_tool(t)
 
@@ -1767,6 +1782,7 @@ def load_sdk_manifest():
       elif '%nightly-llvm-64bit%' in sdk.version: expand_category_param('%nightly-llvm-64bit%', llvm_64bit_nightlies, sdk, is_sdk=True)
       elif '%nightly-llvm-32bit%' in sdk.version: expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, sdk, is_sdk=True)
       elif '%nightly-emscripten%' in sdk.version: expand_category_param('%nightly-emscripten%', emscripten_nightlies, sdk, is_sdk=True)
+      elif '%waterfall-lkgr%' in sdk.version: expand_category_param('%waterfall-lkgr%', waterfall_lkgr, sdk, is_sdk=True)
       else:
         add_sdk(sdk)
 

emsdk_manifest.json

@@ -190,6 +190,34 @@
     "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%",
     "cmake_build_type": "Release"
   },
+  {
+    "id": "clang",
+    "version": "upstream-%waterfall-lkgr%",
+    "bitness": 64,
+    "linux_url": "https://storage.googleapis.com/wasm-llvm/builds/linux/%waterfall-lkgr%/wasm-binaries.tbz2",
+    "install_path": "upstream/%waterfall-lkgr%",
+    "activated_path": "%installation_dir%",
+    "activated_cfg": "LLVM_ROOT='%installation_dir%/bin'"
+  },
+  {
+    "id": "emscripten",
+    "version": "upstream-%waterfall-lkgr%",
+    "bitness": 64,
+    "linux_url": "https://storage.googleapis.com/wasm-llvm/builds/linux/%waterfall-lkgr%/wasm-binaries.tbz2",
+    "install_path": "upstream/%waterfall-lkgr%",
+    "activated_path": "%installation_dir%",
+    "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%/emscripten'"
+  },
+  {
+    "id": "binaryen",
+    "version": "upstream-%waterfall-lkgr%",
+    "bitness": 64,
+    "linux_url": "https://storage.googleapis.com/wasm-llvm/builds/linux/%waterfall-lkgr%/wasm-binaries.tbz2",
+    "install_path": "upstream/%waterfall-lkgr%",
+    "activated_path": "%installation_dir%",
+    "activated_cfg": "BINARYEN_ROOT='%installation_dir%'",
+    "activated_env": "BINARYEN_ROOT=%installation_dir%"
+  },
   {
     "id": "clang",
     "version": "e1.13.0",
@@ -1534,6 +1562,12 @@
     "uses": ["clang-e1.37.1-64bit", "node-4.1.1-64bit", "python-2.7.13.1-64bit", "emscripten-1.37.1"],
     "os": "win"
   },
+  {
+    "version": "upstream-%waterfall-lkgr%",
+    "bitness": 64,
+    "uses": ["clang-upstream-%waterfall-lkgr%-64bit", "emscripten-upstream-%waterfall-lkgr%-64bit", "binaryen-upstream-%waterfall-lkgr%-64bit"],
+    "os": "linux"
+  },
   {
     "version": "%precompiled_tag32%",
     "bitness": 32,

upstream/lkgr.json

@@ -0,0 +1,157 @@
+{
+  "repositories": {
+    "libcxxabi": {
+      "remote": "https://llvm.googlesource.com/libcxxabi.git", 
+      "subject": "[CMake] Set per-runtime library directory suffix in runtimes build", 
+      "hash": "5b1f0aa6f041edaad988a0359c5beb8cd5f1ebfb", 
+      "name": "Petr Hosek", 
+      "email": "[email protected]"
+    }, 
+    "binaryen": {
+      "remote": "https://chromium.googlesource.com/external/github.com/WebAssembly/binaryen.git", 
+      "subject": "Refactor stack writing code into a new StackWriter class (#1620)", 
+      "hash": "e601522b1e15e5a2a96578244faca1648021fb2d", 
+      "name": "Alon Zakai", 
+      "email": "[email protected]"
+    }, 
+    "gnuwin32": null, 
+    "lld": {
+      "remote": "https://llvm.googlesource.com/lld.git", 
+      "subject": "[WebAssemlby] Set IsUsedInRegularObj correctly for undefined data symbols", 
+      "hash": "5934900bdcbc82eb2c582b68909447f8de89a5df", 
+      "name": "Sam Clegg", 
+      "email": "[email protected]"
+    }, 
+    "cr-buildtools": {
+      "remote": "https://chromium.googlesource.com/chromium/src/build.git", 
+      "subject": "Finish removing refrences to exe_and_shlib_deps", 
+      "hash": "7315579e388589b62236ad933f09afd1e838d234", 
+      "name": "Tom Anderson", 
+      "email": "[email protected]"
+    }, 
+    "compiler-rt": {
+      "remote": "https://llvm.googlesource.com/compiler-rt.git", 
+      "subject": "[builtins] Implement the __chkstk function for ARM for MinGW", 
+      "hash": "a22026054ba1a9befadef306c3f56481b4397ab3", 
+      "name": "Martin Storsjo", 
+      "email": "[email protected]"
+    }, 
+    "gcc": {
+      "remote": "https://chromium.googlesource.com/chromiumos/third_party/gcc.git", 
+      "subject": "[GCC] Fix compile time error introduced by PR64111 (GCC FSF).", 
+      "hash": "b6125c702850488ac3bfb1079ae5c9db89989406", 
+      "name": "Caroline Tice", 
+      "email": "[email protected]"
+    }, 
+    "emscripten": {
+      "remote": "https://chromium.googlesource.com/external/github.com/kripken/emscripten.git", 
+      "subject": "Add 'ENV' to the list of runtime elements that may be exported. (#6855)", 
+      "hash": "addf0ae7697f01a1bd36889f0b7a6c30e5cf642f", 
+      "name": "Ian Henderson", 
+      "email": "[email protected]"
+    }, 
+    "libcxx": {
+      "remote": "https://llvm.googlesource.com/libcxx.git", 
+      "subject": "Address \"always inline function is not always inlinable\" warning with GCC.", 
+      "hash": "ffbb91bb640b1b0425a91aa70e2a6a2e0f7244e0", 
+      "name": "Eric Fiselier", 
+      "email": "[email protected]"
+    }, 
+    "nodejs": null, 
+    "host-toolchain": {
+      "remote": "/b/build/slave/cache_dir/chromium.googlesource.com-external-github.com-webassembly-waterfall", 
+      "subject": "Pin chromium clang to a previous revision (#386)", 
+      "hash": "555c690d4cb7cefba750385eeb71cce050bd1e82", 
+      "name": "Sam Clegg", 
+      "email": "[email protected]"
+    }, 
+    "fastcomp": {
+      "remote": "https://chromium.googlesource.com/external/github.com/kripken/emscripten-fastcomp.git", 
+      "subject": "1.38.8", 
+      "hash": "6c4569077d771cf318ff5089a94d82ccef56a038", 
+      "name": "Alon Zakai (kripken)", 
+      "email": "[email protected]"
+    }, 
+    "spec": {
+      "remote": "https://chromium.googlesource.com/external/github.com/WebAssembly/spec.git", 
+      "subject": "[test] Add test to cover one checkpoint in Todo.md (#834)", 
+      "hash": "c8338e9b4a60edb431edba51a4e1a1617060c2e9", 
+      "name": "Wanming Lin", 
+      "email": "[email protected]"
+    }, 
+    "ocamlbuild": {
+      "remote": "https://github.com/ocaml/ocamlbuild.git", 
+      "subject": "prepare for 0.11.0 release", 
+      "hash": "340c02089003dffd10b654a7eb203ffd01c37799", 
+      "name": "Gabriel Scherer", 
+      "email": "[email protected]"
+    }, 
+    "llvm": {
+      "remote": "https://llvm.googlesource.com/llvm.git", 
+      "subject": "[LangRef] Clarify which fast-math flags affect fcmp.", 
+      "hash": "1ea502b3c158159aa2d7827acbd81c7930165a13", 
+      "name": "Eli Friedman", 
+      "email": "[email protected]"
+    }, 
+    "musl": {
+      "remote": "https://github.com/jfbastien/musl.git", 
+      "subject": "Merge pull request #48 from jfbastien/fix_signatures", 
+      "hash": "0ed10f0b97192e9798fbb23dcd284cb5fe4df47a", 
+      "name": "JF Bastien", 
+      "email": "[email protected]"
+    }, 
+    "clang": {
+      "remote": "https://llvm.googlesource.com/clang.git", 
+      "subject": "Remove unnecessary trailing ; in macro intrinsic definition.", 
+      "hash": "726843da082dc448eec5048f24bea1bfebbc969c", 
+      "name": "Eric Christopher", 
+      "email": "[email protected]"
+    }, 
+    "waterfall": {
+      "remote": "/b/build/slave/cache_dir/chromium.googlesource.com-external-github.com-webassembly-waterfall", 
+      "subject": "Pin chromium clang to a previous revision (#386)", 
+      "hash": "555c690d4cb7cefba750385eeb71cce050bd1e82", 
+      "name": "Sam Clegg", 
+      "email": "[email protected]"
+    }, 
+    "v8": {
+      "remote": "https://chromium.googlesource.com/v8/v8.git", 
+      "subject": "[Intl] Use correct fallback values for options in Locale constructor", 
+      "hash": "b7e108d6016bf6b7de3a34e6d61cb522f5193460", 
+      "name": "Frank Tang", 
+      "email": "[email protected]"
+    }, 
+    "cmake": null, 
+    "java": null, 
+    "llvm-test-suite": {
+      "remote": "https://llvm.googlesource.com/test-suite.git", 
+      "subject": "Fixed test to conform to FileCceck change in r336830.", 
+      "hash": "121047f033754e642217dd45b7d49d1947cff3ab", 
+      "name": "Sunil Srivastava", 
+      "email": "[email protected]"
+    }, 
+    "ocaml": {
+      "remote": "https://github.com/ocaml/ocaml.git", 
+      "subject": "change VERSION for 4.05.0", 
+      "hash": "36750d1301ec197e52a1e3d77f20b33dfdf14803", 
+      "name": "Damien Doligez", 
+      "email": "[email protected]"
+    }, 
+    "fastcomp-clang": {
+      "remote": "https://chromium.googlesource.com/external/github.com/kripken/emscripten-fastcomp-clang.git", 
+      "subject": "1.38.8", 
+      "hash": "7c43b8c67d35a870986fb92b5a04e615477ab9cf", 
+      "name": "Alon Zakai (kripken)", 
+      "email": "[email protected]"
+    }, 
+    "wabt": {
+      "remote": "https://chromium.googlesource.com/external/github.com/WebAssembly/wabt.git", 
+      "subject": "[wasm2c] Fix bad realloc in wasm-rt-impl (#869)", 
+      "hash": "e5b3830f4c992c5a0b407dae3e9c8c4ee9a626b2", 
+      "name": "Ben Smith", 
+      "email": "[email protected]"
+    }
+  }, 
+  "build": "33957", 
+  "scheduler": "llvm"
+}