tests: drop pandas dependency

Rewrite the only test that needs pandas.

pandas is a heavy dependency that weighs in around 50MB. It also
recently started printing a warning that in the future it will require
pyarrow, which is another 40 MB.

Pandas is great, however our usage of it is minimal.

Size of devctr does not change much, but we avoid the warning, and not
having to deal with pyarrow in the future.

Example output:
```
MSR removed 0x13 before=0x0
MSR changed 0x179 before=0x20ffff after=0x20
MSR added 0x17 after=0x0
MSR added 0x11 after=0x25ba008
```

Signed-off-by: Pablo Barbáchano <[email protected]>

Author: pb8o

tests/integration_tests/functional/test_cpu_features.py

@@ -4,6 +4,7 @@
 
 # pylint: disable=too-many-lines
 
+import csv
 import io
 import os
 import platform
@@ -14,7 +15,6 @@
 from difflib import unified_diff
 from pathlib import Path
 
-import pandas as pd
 import pytest
 
 import framework.utils_cpuid as cpuid_utils
@@ -30,6 +30,12 @@
 DATA_FILES = Path("./data/msr")
 
 
+def read_msr_csv(fd):
+    """Read a CSV of MSRs"""
+    csvin = csv.DictReader(fd)
+    return list(csvin)
+
+
 def clean_and_mkdir(dir_path):
     """
     Create a clean directory
@@ -313,7 +319,7 @@ def test_cpu_rdmsr(
     assert stderr == ""
 
     # Load results read from the microvm
-    microvm_df = pd.read_csv(io.StringIO(stdout))
+    guest_recs = read_msr_csv(io.StringIO(stdout))
 
     # Load baseline
     host_cpu = global_props.cpu_codename
@@ -329,11 +335,9 @@ def test_cpu_rdmsr(
 
     # Load baseline
     baseline_file_path = DATA_FILES / baseline_file_name
-    # We can use the following line when regathering baselines.
-    # microvm_df.to_csv(baseline_file_path, index=False, encoding="utf-8")
-    baseline_df = pd.read_csv(baseline_file_path)
+    baseline_recs = read_msr_csv(baseline_file_path.open())
 
-    check_msrs_are_equal(baseline_df, microvm_df)
+    check_msrs_are_equal(baseline_recs, guest_recs)
 
 
 # These names need to be consistent across the two parts of the snapshot-restore test
@@ -441,29 +445,31 @@ def test_cpu_wrmsr_snapshot(
     snapshot.save_to(snapshot_artifacts_dir)
 
 
-def check_msrs_are_equal(before_df, after_df):
+def check_msrs_are_equal(before_recs, after_recs):
     """
     Checks that reported MSRs and their values in the files are equal.
     """
 
+    before = {x["MSR_ADDR"]: x["VALUE"] for x in before_recs}
+    after = {x["MSR_ADDR"]: x["VALUE"] for x in after_recs}
     # We first want to see if the same set of MSRs are exposed in the microvm.
-    # Drop the VALUE columns and compare the 2 dataframes.
-    join = pd.merge(before_df, after_df, on="MSR_ADDR", how="outer", indicator=True)
-    removed = join[join["_merge"] == "left_only"]
-    added = join[join["_merge"] == "right_only"]
-
-    assert removed.empty, f"MSRs removed:\n{removed[['MSR_ADDR', 'VALUE_x']]}"
-    assert added.empty, f"MSRs added:\n{added[['MSR_ADDR', 'VALUE_y']]}"
-
-    # Remove MSR that can change at runtime.
-    before_df = before_df[~before_df["MSR_ADDR"].isin(MSR_EXCEPTION_LIST)]
-    after_df = after_df[~after_df["MSR_ADDR"].isin(MSR_EXCEPTION_LIST)]
-
-    # Compare values
-    val_diff = pd.concat(
-        [before_df, after_df], keys=["before", "after"]
-    ).drop_duplicates(keep=False)
-    assert val_diff.empty, f"\n {val_diff.to_string()}"
+    all_msrs = set(before.keys()) | set(after.keys())
+
+    changes = 0
+    for msr in all_msrs:
+        if msr in before and msr not in after:
+            print(f"MSR removed {msr} before={before[msr]}")
+            changes += 1
+        elif msr not in before and msr in after:
+            print(f"MSR added {msr} after={after[msr]}")
+            changes += 1
+        elif msr in MSR_EXCEPTION_LIST:
+            continue
+        elif before[msr] != after[msr]:
+            # Compare values
+            print(f"MSR changed {msr} before={before[msr]} after={after[msr]}")
+            changes += 1
+    assert changes == 0
 
 
 @pytest.mark.skipif(
@@ -504,11 +510,12 @@ def test_cpu_wrmsr_restore(microvm_factory, msr_cpu_template, guest_kernel):
     # Dump MSR state to a file for further comparison
     msrs_after_fname = snapshot_artifacts_dir / shared_names["msrs_after_fname"]
     dump_msr_state_to_file(msrs_after_fname, vm.ssh, shared_names)
+    msrs_before_fname = snapshot_artifacts_dir / shared_names["msrs_before_fname"]
 
     # Compare the two lists of MSR values and assert they are equal
-    before_df = pd.read_csv(snapshot_artifacts_dir / shared_names["msrs_before_fname"])
-    after_df = pd.read_csv(snapshot_artifacts_dir / shared_names["msrs_after_fname"])
-    check_msrs_are_equal(before_df, after_df)
+    before_recs = read_msr_csv(msrs_before_fname.open())
+    after_recs = read_msr_csv(msrs_after_fname.open())
+    check_msrs_are_equal(before_recs, after_recs)
 
 
 def dump_cpuid_to_file(dump_fname, ssh_conn):

tools/devctr/Dockerfile

@@ -57,8 +57,6 @@ RUN apt-get update \
         jq \
         less \
         libbfd-dev \
-        # for pandas
-        libbz2-dev \
         libdw-dev \
         # for aarch64, but can install in x86_64
         libfdt-dev \

tools/devctr/poetry.lock

@@ -17,7 +17,6 @@ mdformat-gfm = "^0.3.5"
 mdformat-footnote = "^0.1.1"
 mdformat-frontmatter = "^2.0.8"
 openapi-spec-validator = "^0.5.6"
-pandas = "^2.0.1"
 psutil = "^5.9.5"
 pylint = "^2.17.4"
 pytest = "^7.3.1"