Car diff

.github/workflows/tests.yml

@@ -60,7 +60,53 @@ jobs:
           scons -j8
           cd opendbc/safety/tests && ./mutation.sh
 
-  # TODO: this test needs to move to opendbc
+  car_diff:
+    name: car diff
+    runs-on: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
+    env:
+      GIT_REF: ${{ github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.event.before || format('origin/{0}', github.event.repository.default_branch) }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - uses: ./.github/workflows/cache
+    - name: Build opendbc
+      run: |
+        source setup.sh
+        scons -j8
+    - name: Test car diff
+      if: github.event_name == 'pull_request'
+      run: source setup.sh && python opendbc/car/tests/car_diff.py | tee diff.txt
+    - name: Comment PR
+      if: always() && github.event_name == 'pull_request'
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: '[ -s diff.txt ] && gh pr comment ${{ github.event.pull_request.number }} --repo ${{ github.repository }} -F diff.txt || true'
+    - name: Update refs
+      if: github.repository == 'commaai/opendbc' && github.ref == 'refs/heads/master'
+      run: source setup.sh && python opendbc/car/tests/car_diff.py --update-refs
+    - name: Checkout ci-artifacts
+      if: github.repository == 'commaai/opendbc' && github.ref == 'refs/heads/master'
+      uses: actions/checkout@v4
+      with:
+        repository: commaai/ci-artifacts
+        ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+        path: ${{ github.workspace }}/ci-artifacts
+    - name: Push refs
+      if: github.repository == 'commaai/opendbc' && github.ref == 'refs/heads/master'
+      working-directory: ${{ github.workspace }}/ci-artifacts
+      run: |
+        ls ${{ github.workspace }}/car_diff/*.zst 2>/dev/null || exit 0
+        git config user.name "GitHub Actions Bot"
+        git config user.email "<>"
+        git fetch origin car_diff || true
+        git checkout car_diff 2>/dev/null || git checkout --orphan car_diff
+        cp ${{ github.workspace }}/car_diff/*.zst .
+        git add *.zst
+        git commit -m "car_diff refs for ${{ github.sha }}" || echo "No changes to commit"
+        git push origin car_diff
+
+  # TODO: this needs to move to opendbc
   test_models:
     name: test models
     runs-on: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}

opendbc/car/logreader.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+import os
+import capnp
+import urllib.parse
+import warnings
+from urllib.request import urlopen
+import zstandard as zstd
+
+from opendbc.car.common.basedir import BASEDIR
+
+capnp_log = capnp.load(os.path.join(BASEDIR, "rlog.capnp"))
+
+
+def decompress_stream(data: bytes):
+  dctx = zstd.ZstdDecompressor()
+  decompressed_data = b""
+
+  with dctx.stream_reader(data) as reader:
+    decompressed_data = reader.read()
+
+  return decompressed_data
+
+
+class LogReader:
+  def __init__(self, fn, only_union_types=False, sort_by_time=False):
+    self._only_union_types = only_union_types
+    _, ext = os.path.splitext(urllib.parse.urlparse(fn).path)
+
+    if fn.startswith("http"):
+      with urlopen(fn) as f:
+        dat = f.read()
+    else:
+      with open(fn, "rb") as f:
+        dat = f.read()
+
+    if ext == ".zst" or dat.startswith(b'\x28\xB5\x2F\xFD'):
+      # https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
+      dat = decompress_stream(dat)
+
+    ents = capnp_log.Event.read_multiple_bytes(dat)
+
+    self._ents = []
+    try:
+      for e in ents:
+        self._ents.append(e)
+    except capnp.KjException:
+      warnings.warn("Corrupted events detected", RuntimeWarning, stacklevel=1)
+
+    if sort_by_time:
+      self._ents.sort(key=lambda x: x.logMonoTime)
+
+  def __iter__(self):
+    for ent in self._ents:
+      if self._only_union_types:
+        try:
+          ent.which()
+          yield ent
+        except capnp.lib.capnp.KjException:
+          pass
+      else:
+        yield ent
+
+  def filter(self, msg_type: str):
+    return (getattr(m, m.which()) for m in filter(lambda m: m.which() == msg_type, self))
+
+  def first(self, msg_type: str):
+    return next(self.filter(msg_type), None)

opendbc/car/rlog.capnp

@@ -0,0 +1,23 @@
+@0xce500edaaae36b0e;
+
+# Minimal schema for parsing rlog CAN messages
+# Subset of cereal/log.capnp
+
+struct CanData {
+  address @0 :UInt32;
+  busTimeDEPRECATED @1 :UInt16;
+  dat @2 :Data;
+  src @3 :UInt8;
+}
+
+struct Event {
+  logMonoTime @0 :UInt64;
+
+  union {
+    initData @1 :Void;
+    frame @2 :Void;
+    gpsNMEA @3 :Void;
+    sensorEventDEPRECATED @4 :Void;
+    can @5 :List(CanData);
+  }
+}