Skip to content
This repository was archived by the owner on Feb 3, 2025. It is now read-only.

Commit 4f62a4d

Browse files
author
DEKHTIARJonathan
committed
[Benchmarking-Py] Adding TF-Hub Movinet
1 parent 0f999c4 commit 4f62a4d

File tree

15 files changed

+1105
-0
lines changed

15 files changed

+1105
-0
lines changed

‎tftrt/benchmarking-python/tf_hub/movinet/base_run_inference.sh‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/bin/#!/usr/bin/env bash
2+
3+
nvidia-smi
4+
5+
BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
6+
7+
# Runtime Parameters
8+
MODEL_NAME=""
9+
10+
# Default Argument Values
11+
BATCH_SIZE=32
12+
NUM_ITERATIONS=1000
13+
OUTPUT_TENSOR_NAMES="classifier_head"
14+
15+
BYPASS_ARGUMENTS=""
16+
17+
# Loop through arguments and process them
18+
for arg in "$@"
19+
do
20+
case $arg in
21+
--model_name=*)
22+
MODEL_NAME="${arg#*=}"
23+
shift # Remove --model_name from processing
24+
;;
25+
--batch_size=*)
26+
BATCH_SIZE="${arg#*=}"
27+
shift # Remove --batch_size= from processing
28+
;;
29+
--num_iterations=*)
30+
NUM_ITERATIONS="${arg#*=}"
31+
shift # Remove --num_iterations= from processing
32+
;;
33+
--input_size=*)
34+
INPUT_SIZE="${arg#*=}"
35+
shift # Remove --model_name from processing
36+
;;
37+
--num_frames=*)
38+
NUM_FRAMES="${arg#*=}"
39+
shift # Remove --num_frames= from processing
40+
;;
41+
--output_tensors_name=*)
42+
OUTPUT_TENSOR_NAMES="${arg#*=}"
43+
shift # Remove --output_tensors_name= from processing
44+
;;
45+
--data_dir=*)
46+
DATA_DIR="${arg#*=}"
47+
shift # Remove --data_dir= from processing
48+
;;
49+
--input_saved_model_dir=*)
50+
MODEL_DIR="${arg#*=}"
51+
shift # Remove --input_saved_model_dir= from processing
52+
;;
53+
*)
54+
BYPASS_ARGUMENTS="${BYPASS_ARGUMENTS} ${arg}"
55+
;;
56+
esac
57+
done
58+
59+
# Trimming front and back whitespaces
60+
BYPASS_ARGUMENTS=$(echo ${BYPASS_ARGUMENTS} | tr -s " ")
61+
62+
echo -e "\n********************************************************************"
63+
echo "[*] MODEL_NAME: ${MODEL_NAME}"
64+
echo ""
65+
echo "[*] DATA_DIR: ${DATA_DIR}"
66+
echo "[*] MODEL_DIR: ${MODEL_DIR}"
67+
echo ""
68+
echo "[*] BATCH_SIZE: ${BATCH_SIZE}"
69+
echo ""
70+
# Custom MoViNets flags
71+
echo "[*] INPUT_SIZE: (${INPUT_SIZE}, ${INPUT_SIZE})"
72+
echo "[*] NUM_FRAMES: ${NUM_FRAMES}"
73+
echo "[*] OUTPUT_TENSOR_NAMES: ${OUTPUT_TENSOR_NAMES}"
74+
echo ""
75+
echo "[*] BYPASS_ARGUMENTS: ${BYPASS_ARGUMENTS}"
76+
77+
echo -e "********************************************************************\n"
78+
79+
MODEL_DIR="${MODEL_DIR}/${MODEL_NAME}/"
80+
81+
if [[ ! -d ${DATA_DIR} ]]; then
82+
echo "ERROR: \`--data_dir=/path/to/directory\` does not exist. [Received: \`${DATA_DIR}\`]"
83+
exit 1
84+
fi
85+
86+
if [[ ! -d ${MODEL_DIR} ]]; then
87+
echo "ERROR: \`--input_saved_model_dir=/path/to/directory\` does not exist. [Received: \`${MODEL_DIR}\`]"
88+
exit 1
89+
fi
90+
91+
set -x
92+
93+
python ${BASE_DIR}/infer.py \
94+
--data_dir=${DATA_DIR} \
95+
--calib_data_dir=${DATA_DIR} \
96+
--input_saved_model_dir=${MODEL_DIR} \
97+
--output_tensors_name=${OUTPUT_TENSOR_NAMES} \
98+
`# The following is set because we will be running synthetic benchmarks` \
99+
--total_max_samples=1 \
100+
--use_synthetic_data \
101+
--num_iterations=${NUM_ITERATIONS} \
102+
${BYPASS_ARGUMENTS}

‎tftrt/benchmarking-python/tf_hub/movinet/infer.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
#!# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
# =============================================================================
17+
18+
import os
19+
import sys
20+
21+
import numpy as np
22+
import tensorflow as tf
23+
24+
# Allow import of top level python files
25+
import inspect
26+
27+
currentdir = os.path.dirname(
28+
os.path.abspath(inspect.getfile(inspect.currentframe()))
29+
)
30+
parentdir = os.path.dirname(currentdir)
31+
parentdir = os.path.dirname(parentdir)
32+
33+
sys.path.insert(0, parentdir)
34+
35+
from benchmark_args import BaseCommandLineAPI
36+
from benchmark_runner import BaseBenchmarkRunner
37+
38+
39+
class CommandLineAPI(BaseCommandLineAPI):
40+
41+
def __init__(self):
42+
super(CommandLineAPI, self).__init__()
43+
44+
self._parser.add_argument(
45+
"--input_size",
46+
type=int,
47+
default=172,
48+
help="Height and width of each frame used by the model"
49+
)
50+
51+
self._parser.add_argument(
52+
"--num_frames",
53+
type=int,
54+
default=5,
55+
help="Number of frames per input video sequence"
56+
)
57+
58+
def _validate_args(self, args):
59+
super(CommandLineAPI, self)._validate_args(args)
60+
61+
# TODO: Remove when proper dataloading is implemented
62+
if not args.use_synthetic_data:
63+
raise NotImplementedError()
64+
65+
66+
class BenchmarkRunner(BaseBenchmarkRunner):
67+
68+
def get_dataset_batches(self):
69+
"""Returns a list of batches of input samples.
70+
71+
Each batch should be in the form [x, y], where
72+
x is a numpy array of the input samples for the batch, and
73+
y is a numpy array of the expected model outputs for the batch
74+
75+
Returns:
76+
- dataset: a TF Dataset object
77+
- bypass_data_to_eval: any object type that will be passed unmodified to
78+
`evaluate_result()`. If not necessary: `None`
79+
80+
Note: script arguments can be accessed using `self._args.attr`
81+
"""
82+
tf.random.set_seed(10)
83+
# The input is video data -- [batch_size, num_frames, height, width, channels]
84+
input_data = tf.random.uniform(
85+
shape=(
86+
1, self._args.num_frames, self._args.input_size,
87+
self._args.input_size, 3
88+
),
89+
dtype=tf.float32
90+
)
91+
92+
dataset = tf.data.Dataset.from_tensor_slices(input_data)
93+
dataset = dataset.repeat()
94+
dataset = dataset.batch(self._args.batch_size)
95+
dataset = dataset.prefetch(tf.data.AUTOTUNE)
96+
97+
return dataset, None
98+
99+
def preprocess_model_inputs(self, data_batch):
100+
"""This function prepare the `data_batch` generated from the dataset.
101+
Returns:
102+
x: input of the model
103+
y: data to be used for model evaluation
104+
105+
Note: script arguments can be accessed using `self._args.attr`
106+
"""
107+
108+
x = data_batch
109+
return x, None
110+
111+
def postprocess_model_outputs(self, predictions, expected):
112+
"""Post process if needed the predictions and expected tensors. At the
113+
minimum, this function transforms all TF Tensors into a numpy arrays.
114+
Most models will not need to modify this function.
115+
116+
Note: script arguments can be accessed using `self._args.attr`
117+
"""
118+
119+
return predictions.numpy(), expected.numpy()
120+
121+
def evaluate_model(self, predictions, expected, bypass_data_to_eval):
122+
"""Evaluate result predictions for entire dataset.
123+
124+
This computes overall accuracy, mAP, etc. Returns the
125+
metric value and a metric_units string naming the metric.
126+
127+
Note: script arguments can be accessed using `self._args.attr`
128+
"""
129+
130+
return None, "Top-1 Accuracy %"
131+
132+
133+
if __name__ == '__main__':
134+
135+
cmdline_api = CommandLineAPI()
136+
args = cmdline_api.parse_args()
137+
138+
runner = BenchmarkRunner(args)
139+
140+
runner.execute_benchmark()

‎tftrt/benchmarking-python/tf_hub/movinet/models/movinet_a0/analysis.txt‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
2+
MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
3+
4+
signature_def['__saved_model_init_op']:
5+
The given SavedModel SignatureDef contains the following input(s):
6+
The given SavedModel SignatureDef contains the following output(s):
7+
outputs['__saved_model_init_op'] tensor_info:
8+
dtype: DT_INVALID
9+
shape: unknown_rank
10+
name: NoOp
11+
Method name is:
12+
13+
signature_def['serving_default']:
14+
The given SavedModel SignatureDef contains the following input(s):
15+
inputs['image'] tensor_info:
16+
dtype: DT_FLOAT
17+
shape: (-1, -1, -1, -1, 3)
18+
name: serving_default_image:0
19+
The given SavedModel SignatureDef contains the following output(s):
20+
outputs['classifier_head'] tensor_info:
21+
dtype: DT_FLOAT
22+
shape: (-1, 600)
23+
name: StatefulPartitionedCall:0
24+
Method name is: tensorflow/serving/predict
25+
26+
Concrete Functions:
27+
Function Name: '__call__'
28+
Option #1
29+
Callable with:
30+
Argument #1
31+
DType: dict
32+
Value: {'image': TensorSpec(shape=(None, None, None, None, 3), dtype=tf.float32, name='inputs/image'), }
33+
Argument #2
34+
DType: bool
35+
Value: False
36+
Argument #3
37+
DType: NoneType
38+
Value: None
39+
Option #2
40+
Callable with:
41+
Argument #1
42+
DType: dict
43+
Value: {'image': TensorSpec(shape=(None, None, None, None, 3), dtype=tf.float32, name='image'), }
44+
Argument #2
45+
DType: bool
46+
Value: False
47+
Argument #3
48+
DType: NoneType
49+
Value: None
50+
Option #3
51+
Callable with:
52+
Argument #1
53+
DType: dict
54+
Value: {'image': TensorSpec(shape=(None, None, None, None, 3), dtype=tf.float32, name='image'), }
55+
Argument #2
56+
DType: bool
57+
Value: True
58+
Argument #3
59+
DType: NoneType
60+
Value: None
61+
Option #4
62+
Callable with:
63+
Argument #1
64+
DType: dict
65+
Value: {'image': TensorSpec(shape=(None, None, None, None, 3), dtype=tf.float32, name='inputs/image'), }
66+
Argument #2
67+
DType: bool
68+
Value: True
69+
Argument #3
70+
DType: NoneType
71+
Value: None
72+
73+
Function Name: '_default_save_signature'
74+
Option #1
75+
Callable with:
76+
Argument #1
77+
DType: dict
78+
Value: {'image': TensorSpec(shape=(None, None, None, None, 3), dtype=tf.float32, name='image'), }
79+
80+
Function Name: 'call_and_return_all_conditional_losses'
81+
Option #1
82+
Callable with:
83+
Argument #1
84+
DType: dict
85+
Value: {'image': TensorSpec(shape=(None, None, None, None, 3), dtype=tf.float32, name='image'), }
86+
Argument #2
87+
DType: bool
88+
Value: False
89+
Argument #3
90+
DType: NoneType
91+
Value: None
92+
Option #2
93+
Callable with:
94+
Argument #1
95+
DType: dict
96+
Value: {'image': TensorSpec(shape=(None, None, None, None, 3), dtype=tf.float32, name='image'), }
97+
Argument #2
98+
DType: bool
99+
Value: True
100+
Argument #3
101+
DType: NoneType
102+
Value: None
103+
Option #3
104+
Callable with:
105+
Argument #1
106+
DType: dict
107+
Value: {'image': TensorSpec(shape=(None, None, None, None, 3), dtype=tf.float32, name='inputs/image'), }
108+
Argument #2
109+
DType: bool
110+
Value: False
111+
Argument #3
112+
DType: NoneType
113+
Value: None
114+
Option #4
115+
Callable with:
116+
Argument #1
117+
DType: dict
118+
Value: {'image': TensorSpec(shape=(None, None, None, None, 3), dtype=tf.float32, name='inputs/image'), }
119+
Argument #2
120+
DType: bool
121+
Value: True
122+
Argument #3
123+
DType: NoneType
124+
Value: None
125+
126+
Function Name: 'init_states'

‎tftrt/benchmarking-python/tf_hub/movinet/models/movinet_a0/run_inference.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
6+
7+
bash ${BASE_DIR}/base_run_inference.sh --model_name="a0" --num_frames=5 --input_size=172 ${@}

0 commit comments

Comments
 (0)