Skip to content

Commit 361ba2a

Browse files
poweifengpoweifeng
authored
renderdiff: separate comparison from rendering (#8733)
- Move the comparison logic into its own script - Add entry point bash script to generate the renderings - Separate the preamble bash logic into its own file
1 parent 78419cd commit 361ba2a

File tree

10 files changed

+219
-143
lines changed

10 files changed

+219
-143
lines changed

‎test/renderdiff/generate.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Copyright (C) 2025 The Android Open Source Project
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
#!/usr/bin/bash
16+
17+
source `dirname $0`/src/preamble.sh
18+
19+
function start_render_() {
20+
start_
21+
if [[ ! "$GITHUB_WORKFLOW" ]]; then
22+
if [ ! -d ${MESA_LIB_DIR} ]; then
23+
bash ${BUILD_COMMON_DIR}/get-mesa.sh
24+
fi
25+
26+
# Install python deps
27+
python3 -m venv ${VENV_DIR}
28+
source ${VENV_DIR}/bin/activate
29+
30+
NEEDED_PYTHON_DEPS=("numpy" "tifffile")
31+
for cmd in "${NEEDED_PYTHON_DEPS[@]}"; do
32+
if ! python3 -m pip show -q "${cmd}"; then
33+
python3 -m pip install ${cmd}
34+
fi
35+
done
36+
fi
37+
mkdir -p ${OUTPUT_DIR}
38+
CXX=`which clang++` CC=`which clang` ./build.sh -f -X ${MESA_DIR} -p desktop debug gltf_viewer
39+
}
40+
41+
function end_render_() {
42+
if [[ ! "$GITHUB_WORKFLOW" ]]; then
43+
deactivate # End python virtual env
44+
fi
45+
end_
46+
}
47+
48+
# Following steps are taken:
49+
# - Get and build mesa
50+
# - Build gltf_viewer
51+
# - Run a test
52+
53+
start_render_ && \
54+
python3 ${RENDERDIFF_TEST_DIR}/src/render.py \
55+
--gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \
56+
--test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \
57+
--output_dir=${OUTPUT_DIR} \
58+
--opengl_lib=${MESA_LIB_DIR} && \
59+
end_render_

‎test/renderdiff/src/compare.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import glob
2+
import os
3+
import sys
4+
import pprint
5+
import json
6+
7+
from utils import execute, ArgParseImpl, important_print
8+
from image_diff import same_image
9+
from results import RESULT_OK, RESULT_FAILED, RESULT_MISSING
10+
11+
def _compare_goldens(base_dir, comparison_dir):
12+
render_results = {}
13+
base_files = glob.glob(os.path.join(base_dir, "./**/*.tif"))
14+
for golden_file in base_files:
15+
base_fname = os.path.abspath(golden_file)
16+
test_case = base_fname.replace(f'{os.path.abspath(base_dir)}/', '')
17+
comp_fname = os.path.abspath(os.path.join(comparison_dir, test_case))
18+
if not os.path.exists(comp_fname):
19+
print(f'file name not found: {comp_fname}')
20+
render_results[test_case] = RESULT_MISSING
21+
continue
22+
if not same_image(base_fname, comp_fname):
23+
render_results[test_case] = RESULT_FAILED
24+
else:
25+
render_results[test_case] = RESULT_OK
26+
return render_results
27+
28+
if __name__ == '__main__':
29+
parser = ArgParseImpl()
30+
parser.add_argument('--src', help='Directory of the base of the diff.', required=True)
31+
parser.add_argument('--dest', help='Directory of the comparison of the diff.')
32+
parser.add_argument('--out', help='Directory of output for the result of the diff.')
33+
34+
args, _ = parser.parse_known_args(sys.argv[1:])
35+
36+
dest = args.dest
37+
if not dest:
38+
print('Assume the default renderdiff output folder')
39+
dest = os.path.join(os.getcwd(), './out/renderdiff_tests')
40+
assert os.path.exists(dest), f"Destination folder={dest} does not exist."
41+
42+
results = _compare_goldens(args.src, dest)
43+
44+
if args.out:
45+
assert os.path.exists(arg.out), f"Output folder={dest} does not exist."
46+
with open(os.path.join(args.out, "compare_results.json", 'w')) as f:
47+
f.write(json.dumps(results))
48+
49+
failed = [f" {k}" for k in results.keys() if results[k] != RESULT_OK]
50+
success_count = len(results) - len(failed)
51+
important_print(f'Successfully compared {success_count} / {len(results)} images' +
52+
('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))
53+
if len(failed) > 0:
54+
exit(1)

‎test/renderdiff/src/golden_manager.py‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import os
1616
import shutil
1717
import re
18+
import sys
1819

1920
from utils import execute, ArgParseImpl, mkdir_p
2021

@@ -132,12 +133,19 @@ def download_to(self, dest_dir, branch='main'):
132133
rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR)
133134
shutil.copytree(rdiff_dir, dest_dir, dirs_exist_ok=True)
134135

135-
# For testing only
136+
# The main entry point will enable download content of a branch to a directory
136137
if __name__ == "__main__":
138+
parser = ArgParseImpl()
139+
parser.add_argument('--branch', type=str, help='Branch of the golden repo', default='main')
140+
parser.add_argument('--output', type=str, help='Directory to download to', required=True)
141+
142+
args, _ = parser.parse_known_args(sys.argv[1:])
143+
144+
# prepare goldens working directory
145+
golden_dir = args.output
146+
assert os.path.isdir(golden_dir),\
147+
f"Output directory {golden_dir} does not exist"
148+
149+
# Download the golden repo into the current working directory
137150
golden_manager = GoldenManager(os.getcwd())
138-
# golden_manager.source_from_and_commit(
139-
# os.path.join(os.getcwd(), 'out/renderdiff_tests'),
140-
# 'First commit (local)',
141-
# branch='branch-test')
142-
# golden_manager.merge_to_main('branch-test', push_to_remote=True)
143-
# golden_manager.download_to(os.path.join(os.getcwd(), 'tmp/goldens'))
151+
golden_manager.download_to(golden_dir, branch=args.branch)

‎test/renderdiff/src/preamble.sh‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Copyright (C) 2025 The Android Open Source Project
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
#!/usr/bin/bash
16+
17+
# Sets up the environment for scripts in test/renderdiff/
18+
19+
OUTPUT_DIR="$(pwd)/out/renderdiff_tests"
20+
RENDERDIFF_TEST_DIR="$(pwd)/test/renderdiff"
21+
MESA_DIR="$(pwd)/mesa/out/"
22+
VENV_DIR="$(pwd)/venv"
23+
BUILD_COMMON_DIR="$(pwd)/build/common"
24+
25+
os_name=$(uname -s)
26+
if [[ "$os_name" == "Linux" ]]; then
27+
MESA_LIB_DIR="${MESA_DIR}lib/x86_64-linux-gnu"
28+
elif [[ "$os_name" == "Darwin" ]]; then
29+
MESA_LIB_DIR="${MESA_DIR}lib"
30+
else
31+
echo "Unsupported platform for renderdiff tests"
32+
exit 1
33+
fi
34+
35+
function start_() {
36+
if [[ "$GITHUB_WORKFLOW" ]]; then
37+
set -ex
38+
fi
39+
}
40+
41+
function end_() {
42+
if [[ "$GITHUB_WORKFLOW" ]]; then
43+
set +ex
44+
fi
45+
}

test/renderdiff/src/run.py renamed to test/renderdiff/src/render.py

Lines changed: 23 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -18,31 +18,18 @@
1818
import glob
1919
import shutil
2020

21-
from utils import execute, ArgParseImpl, mkdir_p, mv_f
22-
from parse_test_json import parse_test_config_from_path
21+
from utils import execute, ArgParseImpl, mkdir_p, mv_f, important_print
22+
23+
import test_config
2324
from golden_manager import GoldenManager
2425
from image_diff import same_image
26+
from results import RESULT_OK, RESULT_FAILED
2527

26-
def important_print(msg):
27-
lines = msg.split('\n')
28-
max_len = max([len(l) for l in lines])
29-
print('-' * (max_len + 8))
30-
for line in lines:
31-
diff = max_len - len(line)
32-
information = f'--- {line} ' + (' ' * diff) + '---'
33-
print(information)
34-
print('-' * (max_len + 8))
35-
36-
RESULT_OK = 'ok'
37-
RESULT_FAILED_TO_RENDER = 'failed-to-render'
38-
RESULT_FAILED_IMAGE_DIFF = 'failed-image-diff'
39-
RESULT_FAILED_NO_GOLDEN = 'failed-no-golden'
40-
41-
def run_test(gltf_viewer,
42-
test_config,
43-
output_dir,
44-
opengl_lib=None,
45-
vk_icd=None):
28+
def _render_test_config(gltf_viewer,
29+
test_config,
30+
output_dir,
31+
opengl_lib=None,
32+
vk_icd=None):
4633
assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist"
4734
assert os.access(gltf_viewer, os.X_OK)
4835

@@ -86,7 +73,7 @@ def run_test(gltf_viewer,
8673
mv_f(f'{test.name}0.tif', out_tif_name)
8774
mv_f(f'{test.name}0.json', f'{named_output_dir}/{test.name}.json')
8875
else:
89-
result = RESULT_FAILED_TO_RENDER
76+
result = RESULT_FAILED
9077
important_print(f'{test_desc} rendering failed with error={out_code}')
9178

9279
results.append({
@@ -96,67 +83,33 @@ def run_test(gltf_viewer,
9683
})
9784
return named_output_dir, results
9885

99-
def compare_goldens(render_results, output_dir, goldens):
100-
for result in render_results:
101-
if result['result'] != RESULT_OK:
102-
continue
103-
104-
out_tif_basename = f"{result['name']}.tif"
105-
out_tif_name = f'{output_dir}/{out_tif_basename}'
106-
golden_path = goldens.get(out_tif_basename)
107-
if not golden_path:
108-
result['result'] = RESULT_FAILED_NO_GOLDEN
109-
result['result_code'] = 1
110-
elif not same_image(golden_path, out_tif_name):
111-
result['result'] = RESULT_FAILED_IMAGE_DIFF
112-
result['result_code'] = 1
113-
114-
return render_results
115-
11686
if __name__ == "__main__":
11787
parser = ArgParseImpl()
11888
parser.add_argument('--test', help='Configuration of the test', required=True)
11989
parser.add_argument('--gltf_viewer', help='Path to the gltf_viewer', required=True)
12090
parser.add_argument('--output_dir', help='Output Directory', required=True)
12191
parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)')
12292
parser.add_argument('--vk_icd', help='Path to VK ICD file')
123-
parser.add_argument('--golden_branch', help='Branch of the golden repo to compare against')
12493

12594
args, _ = parser.parse_known_args(sys.argv[1:])
126-
test = parse_test_config_from_path(args.test)
95+
test = test_config.parse_from_path(args.test)
12796

12897
output_dir, results = \
129-
run_test(args.gltf_viewer,
130-
test,
131-
args.output_dir,
132-
opengl_lib=args.opengl_lib,
133-
vk_icd=args.vk_icd)
134-
135-
do_compare = False
136-
# The presence of this argument indicates comparison against a set of goldens.
137-
if args.golden_branch:
138-
# prepare goldens working directory
139-
tmp_golden_dir = '/tmp/renderdiff-goldens'
140-
mkdir_p(tmp_golden_dir)
141-
142-
# Download the golden repo into the current working directory
143-
golden_manager = GoldenManager(os.getcwd())
144-
golden_manager.download_to(tmp_golden_dir, branch=args.golden_branch)
145-
146-
goldens = {
147-
os.path.basename(fpath) : fpath for fpath in \
148-
glob.glob(f'{os.path.join(tmp_golden_dir, test.name)}/**/*.tif', recursive=True)
149-
}
150-
results = compare_goldens(results, output_dir, goldens)
151-
do_compare = True
152-
153-
with open(f'{output_dir}/results.json', 'w') as f:
98+
_render_test_config(args.gltf_viewer,
99+
test,
100+
args.output_dir,
101+
opengl_lib=args.opengl_lib,
102+
vk_icd=args.vk_icd)
103+
104+
with open(f'{output_dir}/render_results.json', 'w') as f:
154105
f.write(json.dumps(results))
155106

156107
shutil.copy2(args.test, f'{output_dir}/test.json')
157108

158109
failed = [f" {k['name']}" for k in results if k['result'] != RESULT_OK]
159110
success_count = len(results) - len(failed)
160-
op = 'tested' if do_compare else 'rendered'
161-
important_print(f'Successfully {op} {success_count} / {len(results)}' +
162-
('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))
111+
important_print(f'Successfully rendered {success_count} / {len(results)} tests' +
112+
('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))
113+
114+
if len(failed) > 0:
115+
exit(1)

‎test/renderdiff/src/results.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
RESULT_OK = 'ok'
2+
RESULT_FAILED = 'failed'
3+
RESULT_MISSING = 'missing'

test/renderdiff/src/parse_test_json.py renamed to test/renderdiff/src/test_config.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ def _remove_comments_from_json_txt(json_txt):
129129
res.append(line)
130130
return '\n'.join(res)
131131

132-
def parse_test_config_from_path(config_path):
132+
def parse_from_path(config_path):
133133
with open(config_path, 'r') as f:
134134
json_txt = json.loads(_remove_comments_from_json_txt(f.read()))
135135
return RenderTestConfig(json_txt)

‎test/renderdiff/src/update_golden.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ def _interactive_mode(base_golden_dir):
9090
if prompt_helper(
9191
f'Generate the new goldens from your local ' \
9292
f'Filament branch? (branch={cur_branch})') == PROMPT_YES:
93-
code, res = execute('bash ./test/renderdiff/test.sh generate',
93+
code, res = execute('bash ./test/renderdiff/generate.sh',
9494
capture_output=False)
9595
if code != 0:
9696
print('Failed to generate new goldens')

‎test/renderdiff/src/utils.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,3 +106,13 @@ def mkdir_p(path_str):
106106
def mv_f(src_str, dst_str):
107107
src = pathlib.Path(src_str)
108108
src.replace(dst_str)
109+
110+
def important_print(msg):
111+
lines = msg.split('\n')
112+
max_len = max([len(l) for l in lines])
113+
print('-' * (max_len + 8))
114+
for line in lines:
115+
diff = max_len - len(line)
116+
information = f'--- {line} ' + (' ' * diff) + '---'
117+
print(information)
118+
print('-' * (max_len + 8))

0 commit comments

Comments
 (0)