[build,src] Upgrade TensorFlow RNN to 2.0 (#3771)

* Improve CMake build script

* Builds or ignore CUDA objects based on availibility of CUDA tools.
* Builds C++ binaries of tfrnnlm if `-DTENSORFLOW_DIR=...` is provided.
* Select Apple BLAS by default when building on MacOS.
* Enable CTest for tests (run `ctest` or `make test`).

Typical build on Linux with CUDA+MKL:

```
export MKLROOT
cmake -DMATHLIB=MKL <cmakefile>
make
ctest
```

* Update model training script to TensorFlow v2 flavor

* Enable rescore tool to load from SavedModel format

* Update the training script which uses custom loss function

Still tried to mimic original behavior without whole pipeline ready.

* Fix numerical stability of the fast softmax model

Author: scottcjt

CMakeLists.txt

@@ -6,8 +6,16 @@ include(GNUInstallDirs)
 include(Utils)
 include(third_party/get_third_party)
 
+# Should update cmake to a more recent version which supports FindPython3.
+find_package(PythonInterp)
+if(NOT PYTHON_EXECUTABLE OR PYTHON_VERSION_MAJOR LESS 3)
+    message(WARNING "Needs python3 to auto-generate most CMake files, but not found. "
+                    "Will try `python3` directly...")
+    set(PYTHON_EXECUTABLE "python3")
+endif()
+
 message(STATUS "Running gen_cmake_skeleton.py")
-execute_process(COMMAND python
+execute_process(COMMAND ${PYTHON_EXECUTABLE}
     "${CMAKE_CURRENT_SOURCE_DIR}/cmake/gen_cmake_skeleton.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/src"
     "--quiet"
@@ -28,11 +36,21 @@ if(BUILD_SHARED_LIBS)
     endif()
 endif()
 
-set(MATHLIB "OpenBLAS" CACHE STRING "OpenBLAS|MKL|Accelerate")
+if(APPLE)
+    # Use built-in BLAS on MacOS by default.
+    set(MATHLIB "Accelerate" CACHE STRING "OpenBLAS|MKL|Accelerate")
+else()
+    set(MATHLIB "OpenBLAS" CACHE STRING "OpenBLAS|MKL|Accelerate")
+endif()
 option(KALDI_BUILD_EXE "If disabled, will make add_kaldi_executable a no-op" ON)
 option(KALDI_BUILD_TEST "If disabled, will make add_kaldi_test_executable a no-op" ON)
 option(KALDI_USE_PATCH_NUMBER "Use MAJOR.MINOR.PATCH format, otherwise MAJOR.MINOR" OFF)
 
+if (KALDI_BUILD_TEST)
+    include(CTest)
+    enable_testing()
+endif()
+
 link_libraries(${CMAKE_DL_LIBS})
 
 find_package(Threads)
@@ -53,6 +71,19 @@ elseif(MATHLIB STREQUAL "MKL")
     include_directories($ENV{MKLROOT}/include) # TODO: maybe not use env, idk, find_package doesnt handle includes...
     link_libraries(${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
 elseif(MATHLIB STREQUAL "Accelerate")
+    execute_process(COMMAND sw_vers -productVersion
+        OUTPUT_VARIABLE MACOS_VERSION)
+    if(MACOS_VERSION VERSION_LESS "10.12" AND MACOS_VERSION VERSION_GREATER_EQUAL "10.11")
+        message(WARNING
+            "**BAD WARNING**: You are using OS X El Capitan.  Some versions of this OS"
+            " have a bug in the BLAS implementation that affects Kaldi."
+            " After compiling, cd to matrix/ and type 'make test'.  The"
+            " test will fail if the problem exists in your version."
+            " Eventually this issue will be fixed by system updates from"
+            " Apple.  Unexplained crashes with reports of NaNs will"
+            " be caused by this bug, but some recipes will (sometimes) work."
+        )
+    endif()
     set(BLA_VENDOR "Apple")
     find_package(BLAS REQUIRED)
     find_package(LAPACK REQUIRED)
@@ -160,6 +191,11 @@ add_subdirectory(src/kws)
 
 add_subdirectory(src/itf)
 
+if(TENSORFLOW_DIR)
+    add_subdirectory(src/tfrnnlm)
+    add_subdirectory(src/tfrnnlmbin)
+endif()
+
 # add all cuda libraries
 if(CUDA_FOUND)
     add_subdirectory(src/cudafeat)

cmake/Utils.cmake

@@ -36,6 +36,10 @@ macro(add_kaldi_test_executable)
         cmake_parse_arguments(kaldi_test_exe "" "NAME" "SOURCES;DEPENDS" ${ARGN})
         add_executable(${kaldi_test_exe_NAME} ${kaldi_test_exe_SOURCES})
         target_link_libraries(${kaldi_test_exe_NAME} PRIVATE ${kaldi_test_exe_DEPENDS})
+        add_test(
+            NAME ${kaldi_test_exe_NAME}
+            COMMAND ${kaldi_test_exe_NAME}
+            WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
         # list(APPEND KALDI_TEST_EXECUTABLES ${kaldi_test_exe_NAME})
         install(TARGETS ${kaldi_test_exe_NAME} RUNTIME DESTINATION testbin)
 

cmake/gen_cmake_skeleton.py

@@ -180,11 +180,13 @@ def gen_code(self):
 
         if len(self.cuda_source_list) > 0:
             self.source_list.append("${CUDA_OBJS}")
-            ret.append("cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..)")
-            ret.append("cuda_compile(CUDA_OBJS")
+            ret.append("if(CUDA_FOUND)")
+            ret.append("    cuda_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..)")
+            ret.append("    cuda_compile(CUDA_OBJS")
             for f in self.cuda_source_list:
-                ret.append("    " + f)
-            ret.append(")\n")
+                ret.append("        " + f)
+            ret.append("    )")
+            ret.append("endif()\n")
 
         ret.append("add_library(" + self.target_name)
         for f in self.source_list:
@@ -278,6 +280,8 @@ def write_file(self):
 
     subdirs = get_subdirectories(".")
     for d in subdirs:
+        if d.startswith('tfrnnlm'):
+            continue
         cmakelists = CMakeListsFile(d)
         if is_bin_dir(d):
             for f in get_files(d):