Tensorflow C handler Example

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "/Users/demo/git/multi-model-server/examples/tesnorflow_c_handler/bindings/pybind11"]
+	path = /Users/demo/git/multi-model-server/examples/tesnorflow_c_handler/bindings/pybind11
+	url = https://github.com/pybind/pybind11.git

examples/tesnorflow_c_handler/README.md

@@ -0,0 +1,53 @@
+#  TensorFlow Model inference using C API
+
+This example demonstrates adding custom handler for TensorFlow Model inference using C API. Since custom handlers
+need to be written in Python. We use [pybind11](https://github.com/pybind/pybind11) module for Python to C++ binding.
+
+
+## Prerequisites
+
+* TensorFlow C API
+* A compiler with C++11 support
+* CMake >= 2.8.12
+
+
+## Generate Python Module
+
+We are using [CppFlow](https://github.com/serizba/cppflow) source code, to invoke TensorFlow C API. The CppFlow
+uses TensorFlow C API to run the models. Download it from [here](https://www.tensorflow.org/install/lang_c).
+You can install the library system wide or place it in 'libtensorflow' in home directory.
+The [CMakeList.txt](bindings/CMakeList.txt) assumes 'libtensorflow' is in home directory.
+
+## Steps to create a custom handler
+1. Create pybind module
+Use commands below to create a pybind python module which can be imported in python program.
+This will create a python module called "tf_c_inference".
+
+```bash
+git clone --recursive https://github.com/awslabs/multi-model-server.git
+
+# If you haven't used recursive option while cloning, use below command to get updated submodule
+git submodule update --init
+
+pip install ./bindings
+```
+2. Create a python handler
+Create a python custom handler which invokes 'load_model' and 'run_model' API of "tf_c_inference".
+Here is the example [handler](handler.py).
+
+3. Create a MAR file
+Use model-archiver utility to create a MAR file using handler.py. The handler uses
+[mobilenet model](https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz). However you can use
+any other TensorFlow model.
+If you want to ship the model to different machine. Either ensure that you install "tf_c_inference" using step 1 or ship the .so file to that machine and ensure it's in python path.command
+
+## Details about pybind code:
+The binding code is specified in [tf_c_inference.cpp](bindings/src/tf_c_inference.cpp) file.
+The APIs exposed are 'load_model' and 'run_model'. The singleton is used so that model is loaded only once.
+The API invokes CppFlow C++ API which is a wrapper over TensorFlow C API. Feel free to modify the code as per your needs.
+
+## Reference Links
+1. [pybind11](https://github.com/pybind/pybind11)
+2. [cmake_example](https://github.com/pybind/cmake_example)
+3. [CppFlow](https://github.com/serizba/cppflow)
+4. [Tensorflow C API](https://www.tensorflow.org/install/lang_c)

examples/tesnorflow_c_handler/bindings/CMakeLists.txt

@@ -0,0 +1,12 @@
+cmake_minimum_required(VERSION 3.10)
+project(tf_c_inference)
+
+find_library(TENSORFLOW_LIB tensorflow HINT $ENV{HOME}/libtensorflow/lib)
+
+set(CMAKE_CXX_STANDARD 17)
+add_subdirectory(pybind11)
+pybind11_add_module(tf_c_inference src/tf_c_inference.cpp src/Model.cpp src/Tensor.cpp)
+
+target_include_directories(tf_c_inference PRIVATE ../../include $ENV{HOME}/libtensorflow/include)
+
+target_link_libraries (tf_c_inference PRIVATE "${TENSORFLOW_LIB}")

examples/tesnorflow_c_handler/bindings/include/Model.h

@@ -0,0 +1,69 @@
+//
+// Created by sergio on 12/05/19.
+//
+
+#ifndef CPPFLOW_MODEL_H
+#define CPPFLOW_MODEL_H
+
+#include <cstring>
+#include <algorithm>
+#include <string>
+#include <vector>
+#include <iostream>
+#include <fstream>
+#include <tuple>
+#include <tensorflow/c/c_api.h>
+#include "Tensor.h"
+
+class Tensor;
+
+class Model {
+public:
+    // Pass a path to the model file and optional Tensorflow config options. See examples/load_model/main.cpp.
+    explicit Model(const std::string& model_filename, const std::vector<uint8_t>& config_options = {});
+
+    // Rule of five, moving is easy as the pointers can be copied, copying not as i have no idea how to copy
+    // the contents of the pointer (i guess dereferencing won't do a deep copy)
+    Model(const Model &model) = delete;
+    Model(Model &&model) = default;
+    Model& operator=(const Model &model) = delete;
+    Model& operator=(Model &&model) = default;
+
+    ~Model();
+
+    void init();
+    void restore(const std::string& ckpt);
+    void save(const std::string& ckpt);
+    std::vector<std::string> get_operations() const;
+
+    // Original Run
+    void run(const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs);
+
+    // Run with references
+    void run(Tensor& input, const std::vector<Tensor*>& outputs);
+    void run(const std::vector<Tensor*>& inputs, Tensor& output);
+    void run(Tensor& input, Tensor& output);
+
+    // Run with pointers
+    void run(Tensor* input, const std::vector<Tensor*>& outputs);
+    void run(const std::vector<Tensor*>& inputs, Tensor* output);
+    void run(Tensor* input, Tensor* output);
+
+private:
+    TF_Graph* graph;
+    TF_Session* session;
+    TF_Status* status;
+
+    // Read a file from a string
+    static TF_Buffer* read(const std::string&);
+
+    bool status_check(bool throw_exc) const;
+    void error_check(bool condition, const std::string &error) const;
+
+public:
+    friend class Tensor;
+};
+
+
+
+#endif //CPPFLOW_MODEL_H

examples/tesnorflow_c_handler/bindings/include/Tensor.h

@@ -0,0 +1,66 @@
+//
+// Created by sergio on 13/05/19.
+//
+
+#ifndef CPPFLOW_TENSOR_H
+#define CPPFLOW_TENSOR_H
+
+#include <vector>
+#include <memory>
+#include <string>
+#include <algorithm>
+#include <numeric>
+#include <cstring>
+#include <tensorflow/c/c_api.h>
+#include "Model.h"
+
+class Model;
+
+class Tensor {
+public:
+    Tensor(const Model& model, const std::string& operation);
+
+    // Rule of five, moving is easy as the pointers can be copied, copying not as i have no idea how to copy
+    // the contents of the pointer (i guess dereferencing won't do a deep copy)
+    Tensor(const Tensor &tensor) = delete;
+    Tensor(Tensor &&tensor) = default;
+    Tensor& operator=(const Tensor &tensor) = delete;
+    Tensor& operator=(Tensor &&tensor) = default;
+
+    ~Tensor();
+
+    void clean();
+
+    template<typename T>
+    void set_data(std::vector<T> new_data);
+
+    template<typename T>
+    void set_data(std::vector<T> new_data, const std::vector<int64_t>& new_shape);
+
+    template<typename T>
+    std::vector<T> get_data();
+
+	std::vector<int64_t> get_shape();
+
+private:
+    TF_Tensor* val;
+    TF_Output op;
+    TF_DataType type;
+    std::vector<int64_t> shape;
+    std::unique_ptr<std::vector<int64_t>> actual_shape;
+    void* data;
+    int flag;
+
+    // Aux functions
+    void error_check(bool condition, const std::string& error);
+
+    template <typename T>
+    static TF_DataType deduce_type();
+
+    void deduce_shape();
+
+public:
+    friend class Model;
+};
+
+#endif //CPPFLOW_TENSOR_H

examples/tesnorflow_c_handler/bindings/setup.py

@@ -0,0 +1,73 @@
+import os
+import re
+import sys
+import platform
+import subprocess
+
+from setuptools import setup, Extension
+from setuptools.command.build_ext import build_ext
+from distutils.version import LooseVersion
+
+
+class CMakeExtension(Extension):
+    def __init__(self, name, sourcedir=''):
+        Extension.__init__(self, name, sources=[])
+        self.sourcedir = os.path.abspath(sourcedir)
+
+
+class CMakeBuild(build_ext):
+    def run(self):
+        try:
+            out = subprocess.check_output(['cmake', '--version'])
+        except OSError:
+            raise RuntimeError("CMake must be installed to build the following extensions: " +
+                               ", ".join(e.name for e in self.extensions))
+
+        if platform.system() == "Windows":
+            cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', out.decode()).group(1))
+            if cmake_version < '3.1.0':
+                raise RuntimeError("CMake >= 3.1.0 is required on Windows")
+
+        for ext in self.extensions:
+            self.build_extension(ext)
+
+    def build_extension(self, ext):
+        extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
+        # required for auto-detection of auxiliary "native" libs
+        if not extdir.endswith(os.path.sep):
+            extdir += os.path.sep
+
+        cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
+                      '-DPYTHON_EXECUTABLE=' + sys.executable]
+
+        cfg = 'Debug' if self.debug else 'Release'
+        build_args = ['--config', cfg]
+
+        if platform.system() == "Windows":
+            cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir)]
+            if sys.maxsize > 2**32:
+                cmake_args += ['-A', 'x64']
+            build_args += ['--', '/m']
+        else:
+            cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
+            build_args += ['--', '-j2']
+
+        env = os.environ.copy()
+        env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''),
+                                                              self.distribution.get_version())
+        if not os.path.exists(self.build_temp):
+            os.makedirs(self.build_temp)
+        subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env)
+        subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp)
+
+setup(
+    name='tf_c_inference',
+    version='0.0.1',
+    author='',
+    author_email='',
+    description='Pybind module to use Tensorflow C API',
+    long_description='',
+    ext_modules=[CMakeExtension('tf_c_inference')],
+    cmdclass=dict(build_ext=CMakeBuild),
+    zip_safe=False,
+)