[BERT/TF] TRT int8 and Triton

Author: nv-kkudrynski

TensorFlow/LanguageModeling/BERT/.gitignore

@@ -4,6 +4,12 @@
 __pycache__/
 *.py[cod]
 *$py.class
+*.png
+.idea/
+*swp
+data/
+checkpoints/
+data_dl/
 
 # C extensions
 *.so

TensorFlow/LanguageModeling/BERT/Dockerfile

@@ -17,7 +17,7 @@ RUN git clone https://github.com/titipata/pubmed_parser
 RUN pip3 install /workspace/pubmed_parser
 
 #Copy the perf_client over
-ARG TRTIS_CLIENTS_URL=https://github.com/NVIDIA/triton-inference-server/releases/download/v2.0.0/v2.0.0_ubuntu1804.clients.tar.gz
+ARG TRTIS_CLIENTS_URL=https://github.com/NVIDIA/triton-inference-server/releases/download/v2.2.0/v2.2.0_ubuntu1804.clients.tar.gz
 RUN mkdir -p /workspace/install \
     && curl -L ${TRTIS_CLIENTS_URL} | tar xvz -C /workspace/install
 

TensorFlow/LanguageModeling/BERT/README.md

@@ -273,7 +273,7 @@ Note: Not using BookCorpus can potentially change final accuracy on a few downst
 
 4. Download the pretrained models from NGC.
 
-We have uploaded checkpoints that have been [fine tuned](https://ngc.nvidia.com/catalog/models/nvidia:bert_tf_v1_1_large_fp32_384) and [pre-trained](https://ngc.nvidia.com/catalog/models/nvidia:bert_tf_pretraining_lamb_16n) for various configurations on the NGC Model Registry. You can browse and download the relevant checkpoints directly from the [NGC model catalog](https://ngc.nvidia.com/catalog/models). Download them to the `results/models/` to easily access them in your scripts. 
+We have uploaded checkpoints that have been [fine tuned](https://ngc.nvidia.com/catalog/models/nvidia:bert_tf_v1_1_large_fp16_384) and [pre-trained](https://ngc.nvidia.com/catalog/models/nvidia:bert_tf_pretraining_lamb_16n) for various configurations on the NGC Model Registry. Our data download scripts, by default download some of them but you can browse and download the relevant checkpoints directly from the [NGC model catalog](https://ngc.nvidia.com/catalog/models). Download them to the `data/download/nvidia_pretrained/` to easily access them in your scripts. 
 
 5. Start an interactive session in the NGC container to run training/inference.
 

TensorFlow/LanguageModeling/BERT/biobert/conlleval.py

@@ -22,7 +22,7 @@
 class FormatError(Exception):
     pass
 
-Metrics = namedtuple('Metrics', 'tp fp fn precision recall f1')
+Metrics = namedtuple('Metrics', 'tp fp fn prec rec fscore')
 
 
 class EvalCounts(object):

TensorFlow/LanguageModeling/BERT/biobert/scripts/ner_bc5cdr-chem.sh

@@ -2,7 +2,7 @@
 
 echo "Container nvidia build = " $NVIDIA_BUILD_ID
 
-init_checkpoint=${1:-"/results/biobert_tf_uncased_base/model.ckpt-4340"}
+init_checkpoint=${1:-"/results/biobert_tf_uncased_base/model.ckpt"}
 train_batch_size=${2:-8}
 learning_rate=${3:-3.125e-6}
 cased=${4:-false}
@@ -30,7 +30,6 @@ else
     export BERT_DIR=/workspace/bert/data/download/google_pretrained_weights/${CASING_DIR_PREFIX}_L-12_H-768_A-12
 fi
 
-
 export GBS=$(expr $train_batch_size \* $num_gpu)
 printf -v TAG "tf_bert_biobert_ner_bc5cdr_chem_%s_%s_gbs%d" "$bert_model" "$precision" $GBS
 DATESTAMP=`date +'%y%m%d%H%M%S'`

TensorFlow/LanguageModeling/BERT/biobert/scripts/ner_bc5cdr-disease.sh

@@ -2,7 +2,7 @@
 
 echo "Container nvidia build = " $NVIDIA_BUILD_ID
 
-init_checkpoint=${1:-"/results/biobert_tf_uncased_base/model.ckpt-4340"}
+init_checkpoint=${1:-"/results/biobert_tf_uncased_base/model.ckpt"}
 train_batch_size=${2:-8}
 learning_rate=${3:-3.125e-6}
 cased=${4:-false}

TensorFlow/LanguageModeling/BERT/biobert/scripts/rel_chemprot.sh

@@ -2,7 +2,7 @@
 
 echo "Container nvidia build = " $NVIDIA_BUILD_ID
 
-init_checkpoint=${1:-"/results/biobert_tf_uncased_base/model.ckpt-4340"}
+init_checkpoint=${1:-"/results/biobert_tf_uncased_base/model.ckpt"}
 train_batch_size=${2:-8}
 learning_rate=${3:-1.5e-6}
 cased=${4:-false}

TensorFlow/LanguageModeling/BERT/biobert/scripts/run_biobert_finetuning_inference.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 task=${1:-"ner_bc5cdr-chem"}
-init_checkpoint=${2:-"/results/biobert_tf_uncased_base/model.ckpt-4340"}
+init_checkpoint=${2:-"/results/biobert_tf_uncased_base/model.ckpt"}
 bert_model=${3:-"base"}
 cased=${4:-"false"}
 precision=${5:-"fp16"}

TensorFlow/LanguageModeling/BERT/data/create_datasets_from_start.sh

@@ -23,10 +23,30 @@ if [ "$to_download" = "wiki_books" ] ; then
 fi
 
 python3 /workspace/bert/data/bertPrep.py --action download --dataset wikicorpus_en
-python3 /workspace/bert/data/bertPrep.py --action download --dataset google_pretrained_weights  # Includes vocab
 python3 /workspace/bert/data/bertPrep.py --action download --dataset squad
 python3 /workspace/bert/data/bertPrep.py --action download --dataset mrpc
 python3 /workspace/bert/data/bertPrep.py --action download --dataset sst-2
+python3 ${BERT_PREP_WORKING_DIR}/bertPrep.py --action download --dataset google_pretrained_weights
+
+mkdir -p /workspace/bert/data/download/nvidia_pretrained
+#SQuAD Large Checkpoint
+	echo "Downloading SQuAD Large Checkpoint"
+	cd /workspace/bert/data/download/nvidia_pretrained && \
+		wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/bert_tf_ckpt_large_qa_squad11_amp_384/versions/19.03.1/zip -O bert_tf_ckpt_large_qa_squad11_amp_384_19.03.1.zip \
+		 && unzip bert_tf_ckpt_large_qa_squad11_amp_384_19.03.1.zip -d bert_tf_squad11_large_384 && rm bert_tf_ckpt_large_qa_squad11_amp_384_19.03.1.zip
+
+#SQuAD Base Checkpoint
+cd /workspace/bert/data/download/nvidia_pretrained && \
+	wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/bert_tf_ckpt_base_qa_squad11_amp_128/versions/19.03.1/zip -O bert_tf_ckpt_base_qa_squad11_amp_128_19.03.1.zip \
+	 && unzip bert_tf_ckpt_base_qa_squad11_amp_128_19.03.1.zip -d bert_tf_squad11_base_128 && rm bert_tf_ckpt_base_qa_squad11_amp_128_19.03.1.zip
+
+#Pretraining Large checkpoint
+cd /workspace/bert/data/download/nvidia_pretrained && \
+	wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/bert_tf_ckpt_large_pretraining_amp_lamb/versions/19.03.1/zip -O bert_tf_ckpt_large_pretraining_amp_lamb_19.03.1.zip \
+	&& unzip bert_tf_ckpt_large_pretraining_amp_lamb_19.03.1.zip -d bert_tf_pretraining_large_lamb && rm bert_tf_ckpt_large_pretraining_amp_lamb_19.03.1.zip
+
+python3 /workspace/bert/data/bertPrep.py --action download --dataset google_pretrained_weights  # Redundant, to verify and remove
+
 
 DATASET="wikicorpus_en"
 # Properly format the text files