renderdiff: add viewer for image differences

 - Modify the compare script to output more details of a
   comparison. This will include the source/golden directory,
   the comparison directory (the new renderings), and a file
   path to difference images if the golden does not match
   the rendered image.
 - The image_diff script can now output a TIFF that is the
   difference of two input TIFFs.
 - Add a viewer for examining the differences between rendered
   output and golden images.
 - The viewer consists of a simple server of web API endpoints
   for querying difference results (along with rendered images
   in TIFF).
 - And a web-based (html + lit-element) UI for looking at the
   rendered images and differences.

Author: poweifeng

test/renderdiff/generate.sh

@@ -34,7 +34,6 @@ function start_render_() {
             fi
         done
     fi
-    mkdir -p ${OUTPUT_DIR}
     CXX=`which clang++` CC=`which clang` ./build.sh -f -X ${MESA_DIR} -p desktop debug gltf_viewer
 }
 
@@ -54,6 +53,6 @@ start_render_ && \
     python3 ${RENDERDIFF_TEST_DIR}/src/render.py \
             --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \
             --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \
-            --output_dir=${OUTPUT_DIR} \
+            --output_dir=${RENDER_OUTPUT_DIR} \
             --opengl_lib=${MESA_LIB_DIR} && \
     end_render_

test/renderdiff/src/compare.py

@@ -4,26 +4,54 @@
 import pprint
 import json
 
-from utils import execute, ArgParseImpl, important_print
-from image_diff import same_image
+from utils import execute, ArgParseImpl, important_print, mkdir_p
+from image_diff import same_image, output_image_diff
 from results import RESULT_OK, RESULT_FAILED, RESULT_MISSING
 
-def _compare_goldens(base_dir, comparison_dir):
-  render_results = {}
-  base_files = glob.glob(os.path.join(base_dir, "./**/*.tif"))
-  for golden_file in base_files:
-    base_fname = os.path.abspath(golden_file)
-    test_case = base_fname.replace(f'{os.path.abspath(base_dir)}/', '')
-    comp_fname = os.path.abspath(os.path.join(comparison_dir, test_case))
-    if not os.path.exists(comp_fname):
-      print(f'file name not found: {comp_fname}')
-      render_results[test_case] = RESULT_MISSING
-      continue
-    if not same_image(base_fname, comp_fname):
-      render_results[test_case] = RESULT_FAILED
-    else:
-      render_results[test_case] = RESULT_OK
-  return render_results
+def _compare_goldens(base_dir, comparison_dir, out_dir=None):
+  all_files = glob.glob(os.path.join(base_dir, "./**/*.tif"), recursive=True)
+  test_dirs = set(os.path.abspath(os.path.dirname(f)).replace(os.path.abspath(base_dir) + '/', '') \
+                  for f in all_files)
+  all_results = []
+  for test_dir in test_dirs:
+    results_meta = {}
+    results = []
+    output_test_dir = None if not out_dir else os.path.join(out_dir, test_dir)
+    if output_test_dir:
+      mkdir_p(output_test_dir)
+    base_test_dir = os.path.abspath(os.path.join(base_dir, test_dir))
+    comp_test_dir = os.path.abspath(os.path.join(comparison_dir, test_dir))
+    results_meta['base_dir'] = base_test_dir
+    results_meta['comparison_dir'] = comp_test_dir
+    for golden_file in \
+        glob.glob(os.path.join(base_test_dir, "*.tif")):
+      base_fname = os.path.abspath(golden_file)
+      test_case = base_fname.replace(f'{base_test_dir}/', '')
+      comp_fname = os.path.join(comp_test_dir, test_case)
+      result = {
+        'name': test_case,
+      }
+      if not os.path.exists(comp_fname):
+        print(f'file name not found: {comp_fname}')
+        result['result'] = RESULT_MISSING
+      elif not same_image(base_fname, comp_fname):
+        result['result'] = RESULT_FAILED
+        if output_test_dir:
+          # just the file name
+          diff_fname = f"{test_case.replace('.tif', '_diff.tif')}"
+          output_image_diff(base_fname, comp_fname, os.path.join(output_test_dir, diff_fname))
+          result['diff'] = diff_fname
+      else:
+        result['result'] = RESULT_OK
+      results.append(result)
+    if output_test_dir:
+      results_meta['results'] = results
+      output_fname = os.path.join(output_test_dir, "compare_results.json")
+      with open(output_fname, 'w') as f:
+        f.write(json.dumps(results_meta, indent=2))
+      important_print(f'Written comparison results for {test_dir} to \n    {output_fname}')
+    all_results += results
+  return all_results
 
 if __name__ == '__main__':
   parser = ArgParseImpl()
@@ -39,14 +67,9 @@ def _compare_goldens(base_dir, comparison_dir):
     dest = os.path.join(os.getcwd(), './out/renderdiff_tests')
   assert os.path.exists(dest), f"Destination folder={dest} does not exist."
 
-  results = _compare_goldens(args.src, dest)
+  results = _compare_goldens(args.src, dest, out_dir=args.out)
 
-  if args.out:
-    assert os.path.exists(arg.out), f"Output folder={dest} does not exist."
-    with open(os.path.join(args.out, "compare_results.json", 'w')) as f:
-      f.write(json.dumps(results))
-
-  failed = [f"   {k}" for k in results.keys() if results[k] != RESULT_OK]
+  failed = [f"   {k['name']}" for k in results if k['result'] != RESULT_OK]
   success_count = len(results) - len(failed)
   important_print(f'Successfully compared {success_count} / {len(results)} images' +
                     ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))

test/renderdiff/src/image_diff.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import tifffile
-import numpy
+import numpy as np
 
 def same_image(tiff_file_a, tiff_file_b):
   try:
@@ -27,7 +27,7 @@ def same_image(tiff_file_a, tiff_file_b):
       return False
 
     # numpy.array_equal() checks if two arrays have the same shape and elements.
-    if numpy.array_equal(img1_data, img2_data):
+    if np.array_equal(img1_data, img2_data):
       return True
     else:
       return False
@@ -41,3 +41,27 @@ def same_image(tiff_file_a, tiff_file_b):
   except Exception as e:
     print(f"An unexpected error occurred: {e}")
     return False
+
+def output_image_diff(tiff_file_a, tiff_file_b, output_path):
+  try:
+    img1_data = tifffile.imread(tiff_file_a)
+    img2_data = tifffile.imread(tiff_file_b)
+
+    # If the dimensions (height, width, number of channels, number of pages/frames)
+    # are different, the images are not the same.
+    if img1_data.shape != img2_data.shape:
+      raise RuntimeError(f"Images {tiff_file_a} and {tiff_file_b} have different shapes: {img1_data.shape} vs {img2_data.shape}")
+
+    diff_img = np.abs(img1_data.astype(np.int16) - img2_data.astype(np.int16))
+    diff_img = diff_img.astype(np.uint8)
+    tifffile.imwrite(output_path, diff_img)
+
+  except FileNotFoundError:
+    print(f"Error: One or both files not found ('{file_path1}', '{file_path2}').")
+    return False
+  except tifffile.TiffFileError as e:
+    print(f"Error: One or both files are not valid TIFF files or could not be read. Details: {e}")
+    return False
+  except Exception as e:
+    print(f"An unexpected error occurred: {e}")
+    return False

test/renderdiff/src/preamble.sh

@@ -16,7 +16,9 @@
 
 # Sets up the environment for scripts in test/renderdiff/
 
-OUTPUT_DIR="$(pwd)/out/renderdiff_tests"
+RENDER_OUTPUT_DIR="$(pwd)/out/renderdiff/renders"
+DIFF_OUTPUT_DIR="$(pwd)/out/renderdiff/diffs"
+GOLDEN_OUTPUT_DIR="$(pwd)/out/renderdiff/goldens"
 RENDERDIFF_TEST_DIR="$(pwd)/test/renderdiff"
 MESA_DIR="$(pwd)/mesa/out/"
 VENV_DIR="$(pwd)/venv"
@@ -33,6 +35,7 @@ else
 fi
 
 function start_() {
+    mkdir -p ${RENDER_OUTPUT_DIR} ${DIFF_OUTPUT_DIR} ${GOLDEN_OUTPUT_DIR}
     if [[ "$GITHUB_WORKFLOW" ]]; then
         set -ex
     fi

test/renderdiff/src/render.py

@@ -102,7 +102,7 @@ def _render_test_config(gltf_viewer,
                         vk_icd=args.vk_icd)
 
   with open(f'{output_dir}/render_results.json', 'w') as f:
-    f.write(json.dumps(results))
+    f.write(json.dumps(results, indent=2))
 
   shutil.copy2(args.test, f'{output_dir}/test.json')
 

test/renderdiff/src/viewer.py

@@ -0,0 +1,81 @@
+# Copyright (C) 2025 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import flask
+import pathlib
+import json
+
+from utils import ArgParseImpl
+
+from flask import Flask, request, make_response, send_from_directory
+
+DIR = pathlib.Path(__file__).parent.absolute()
+HTML_DIR = os.path.join(DIR, "viewer_html")
+
+def _create_app(config):
+  app = Flask(__name__)
+
+  client_config = config.copy()
+  base_dir = client_config['base_dir']
+  comparison_dir = client_config['comparison_dir']
+  diff_dir = client_config['diff_dir']
+
+  del client_config['base_dir']
+  del client_config['comparison_dir']
+  del client_config['diff_dir']
+
+  @app.route('/r/', methods=['GET'])
+  def get_r():
+    return json.dumps(client_config)
+
+  @app.route('/g/<path:filepath>', methods=['GET'])
+  def get_g(filepath):
+    return send_from_directory(base_dir, filepath)
+
+  @app.route('/d/<path:filepath>', methods=['GET'])
+  def get_d(filepath):
+    return send_from_directory(diff_dir, filepath)
+
+  @app.route('/c/<path:filepath>', methods=['GET'])
+  def get_c(filepath):
+    return send_from_directory(comparison_dir, filepath)
+
+  @app.route('/<path:filepath>')
+  def get_static_file(filepath):
+    return send_from_directory(HTML_DIR, filepath)
+
+  @app.route('/')
+  def get_index():
+    return send_from_directory(HTML_DIR, 'index.html')
+
+  app.url_map.strict_slashes = False
+  return app
+
+if __name__ == '__main__':
+  PORT = 8901
+  parser = ArgParseImpl()
+  parser.add_argument('--diff', help='Diff directory', required=True)
+  args, _ = parser.parse_known_args(sys.argv[1:])
+
+  with open(os.path.join(args.diff, 'compare_results.json'), 'r') as f:
+    config = json.loads(f.read())
+  config['diff_dir'] = os.path.abspath(args.diff)
+
+  app = _create_app(config)
+  from waitress import serve
+
+  print(f'Point your browser to http://localhost:{PORT} to see the diff results')
+  serve(app, host="127.0.0.1", port=PORT)