Revert "revert tg calib and opencl cleanup (#37113)"

This reverts commit 51312af.

Author: adeebshihadeh

Dockerfile.openpilot_base

@@ -18,43 +18,6 @@ RUN /tmp/tools/install_ubuntu_dependencies.sh && \
     cd /usr/lib/gcc/arm-none-eabi/* && \
     rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
 
-# Add OpenCL
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    apt-utils \
-    alien \
-    unzip \
-    tar \
-    curl \
-    xz-utils \
-    dbus \
-    gcc-arm-none-eabi \
-    tmux \
-    vim \
-    libx11-6 \
-    wget \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /tmp/opencl-driver-intel && \
-    cd /tmp/opencl-driver-intel && \
-    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
-    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    mkdir -p /etc/OpenCL/vendors && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
-    cd /opt/intel && \
-    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    mkdir -p /etc/ld.so.conf.d && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    cd / && \
-    rm -rf /tmp/opencl-driver-intel
-
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
 ENV QTWEBENGINE_DISABLE_SANDBOX=1

SConstruct

@@ -94,7 +94,6 @@ env = Environment(
 
 # Arch-specific flags and paths
 if arch == "larch64":
-  env.Append(CPPPATH=["#third_party/opencl/include"])
   env.Append(LIBPATH=[
     "/usr/local/lib",
     "/system/vendor/lib64",

common/SConscript

@@ -5,7 +5,6 @@ common_libs = [
   'swaglog.cc',
   'util.cc',
   'ratekeeper.cc',
-  'clutil.cc',
 ]
 
 _common = env.Library('common', common_libs, LIBS="json11")

common/clutil.cc

@@ -1,35 +1,10 @@
 import os
 import glob
 
-Import('env', 'envCython', 'arch', 'cereal', 'messaging', 'common', 'visionipc')
+Import('env', 'arch')
 lenv = env.Clone()
-lenvCython = envCython.Clone()
 
-libs = [cereal, messaging, visionipc, common, 'capnp', 'kj', 'pthread']
-frameworks = []
-
-common_src = [
-  "models/commonmodel.cc",
-  "transforms/loadyuv.cc",
-  "transforms/transform.cc",
-]
-
-# OpenCL is a framework on Mac
-if arch == "Darwin":
-  frameworks += ['OpenCL']
-else:
-  libs += ['OpenCL']
-
-# Set path definitions
-for pathdef, fn in {'TRANSFORM': 'transforms/transform.cl', 'LOADYUV': 'transforms/loadyuv.cl'}.items():
-  for xenv in (lenv, lenvCython):
-    xenv['CXXFLAGS'].append(f'-D{pathdef}_PATH=\\"{File(fn).abspath}\\"')
-
-# Compile cython
-cython_libs = envCython["LIBS"] + libs
-commonmodel_lib = lenv.Library('commonmodel', common_src)
-lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks)
-tinygrad_files = sorted(["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath) if 'pycache' not in x])
+tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath) if 'pycache' not in x]
 
 # Get model metadata
 for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']:
@@ -38,22 +13,30 @@ for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']:
   cmd = f'python3 {Dir("#selfdrive/modeld").abspath}/get_model_metadata.py {fn}.onnx'
   lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"] + tinygrad_files + script_files, cmd)
 
+# compile warp
+tg_flags = {
+    'larch64': 'DEV=QCOM FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0',
+    'Darwin': f'DEV=CPU HOME={os.path.expanduser("~")}', # tinygrad calls brew which needs a $HOME in the env
+}.get(arch, 'DEV=CPU CPU_LLVM=1')
+image_flag = {
+     'larch64': 'IMAGE=2',
+}.get(arch, 'IMAGE=0')
+script_files = [File(Dir("#selfdrive/modeld").File("compile_warp.py").abspath)]
+cmd = f'{tg_flags} python3 {Dir("#selfdrive/modeld").abspath}/compile_warp.py '
+lenv.Command(fn + "warp_tinygrad.pkl", tinygrad_files + script_files, cmd)
+
 def tg_compile(flags, model_name):
   pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#tinygrad_repo").abspath + '"'
   fn = File(f"models/{model_name}").abspath
   return lenv.Command(
     fn + "_tinygrad.pkl",
     [fn + ".onnx"] + tinygrad_files,
-    f'{pythonpath_string} {flags} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl'
+    f'{pythonpath_string} {flags} {image_flag} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl'
   )
 
 # Compile small models
 for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']:
-  flags = {
-    'larch64': 'DEV=QCOM FLOAT16=1 NOLOCALS=1 IMAGE=2 JIT_BATCH_SIZE=0',
-    'Darwin': f'DEV=CPU HOME={os.path.expanduser("~")}', # tinygrad calls brew which needs a $HOME in the env
-  }.get(arch, 'DEV=CPU CPU_LLVM=1')
-  tg_compile(flags, model_name)
+  tg_compile(tg_flags, model_name)
 
 # Compile BIG model if USB GPU is available
 if "USBGPU" in os.environ: