diff --git a/demos/image_classification/go/README.md b/demos/image_classification/go/README.md index 705556b2d7..4e78fc401c 100644 --- a/demos/image_classification/go/README.md +++ b/demos/image_classification/go/README.md @@ -11,25 +11,17 @@ cd model_server/demos/image_classification/go ## Get the model -To run end to end flow and get correct results, please download `resnet-50-tf` model and convert it to IR format by following [instructions available on the OpenVINO Model Zoo page](https://github.com/openvinotoolkit/open_model_zoo/blob/master/models/public/resnet-50-tf/README.md) +To run end to end flow and get correct results, please download `resnet50-v1-12` model from ONNX repository. -Place converted model files (XML and BIN) under the following path: `/resnet-50-tf/1` +Place downloaded model file (.onnx) under the following path: `/1` Where `PATH_TO_MODELS` is the path to the directory with models on the host filesystem. For example: ```bash -mkdir models -docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models openvino/ubuntu20_dev:2024.6.0 omz_downloader --name resnet-50-tf --output_dir /models -docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models:rw openvino/ubuntu20_dev:2024.6.0 omz_converter --name resnet-50-tf --download_dir /models --output_dir /models --precisions FP32 -mv ${PWD}/models/public/resnet-50-tf/FP32 ${PWD}/models/public/resnet-50-tf/1 - -tree models/public/resnet-50-tf -models/public/resnet-50-tf -├── 1 -│   ├── resnet-50-tf.bin -│   └── resnet-50-tf.xml -└── resnet_v1-50.pb +mkdir -p model/1 +wget -P model/1 https://github.com/onnx/models/raw/refs/heads/main/validated/vision/classification/resnet/model/resnet50-v1-12.onnx + ``` ## Build Go client docker image @@ -49,11 +41,9 @@ docker build . -t ovmsclient Before running the client launch OVMS with prepared ResNet model. You can do that with a command similar to: ```bash -docker run -d --rm -p 9000:9000 -v ${PWD}/models/public/resnet-50-tf:/models/resnet openvino/model_server:latest --model_name resnet --model_path /models/resnet --port 9000 +docker run -d --rm -p 9000:9000 -v ${PWD}/model:/models openvino/model_server:latest --model_name resnet --model_path /models --port 9000 --layout NHWC:NCHW ``` -**Note** Layout for downloaded resnet model is NHWC. It ensures that the model will accept binary input generated by the client. See [binary inputs](../../../docs/binary_input.md) doc if you want to learn more about this feature. - ## Run prediction with Go client In order to run prediction on the model served by the OVMS using Go client run the following command: @@ -69,15 +59,4 @@ Classification confidence: 98.914299% Command explained: - `--net=host` option is required so the container with the client can access container with the model server via host network (localhost), - `--serving-address` parameter defines the address of the model server gRPC endpoint, -- the last part in the command is a path to the image that will be send to OVMS for prediction. The image must be accessible from the inside of the container (could be mounted). Single zebra picture - `zebra.jpeg` - has been embedded in the docker image to simplify the example, so above command would work out of the box. If you wish to use other image you need to provide it to the container and change the path. - -You can also choose if the image should be sent as binary input (raw JPG or PNG bytes) or should be converted on the client side to the data array accepted by the model. -To send raw bytes just add `--binary-input` flag like this: - -```bash -docker run --net=host --rm ovmsclient --serving-address localhost:9000 --binary-input zebra.jpeg -# exemplary output -2022/06/15 13:46:53 Request sent successfully -Predicted class: zebra -Classification confidence: 98.914299% -``` +- the last part in the command is a path to the image that will be send to OVMS for prediction. The image must be accessible from the inside of the container (could be mounted). Single zebra picture - `zebra.jpeg` - has been embedded in the docker image to simplify the example, so above command would work out of the box. If you wish to use other image you need to provide it to the container and change the path. \ No newline at end of file diff --git a/demos/image_classification/go/resnet_predict.go b/demos/image_classification/go/resnet_predict.go index f613150dc9..1ab5fa9015 100644 --- a/demos/image_classification/go/resnet_predict.go +++ b/demos/image_classification/go/resnet_predict.go @@ -17,6 +17,8 @@ package main import ( + "bytes" + "encoding/binary" "context" "flag" "fmt" @@ -28,97 +30,81 @@ import ( "path/filepath" framework "tensorflow/core/framework" pb "tensorflow_serving" - + "math" google_protobuf "github.com/golang/protobuf/ptypes/wrappers" "github.com/nfnt/resize" "gocv.io/x/gocv" "google.golang.org/grpc" ) -func run_binary_input(servingAddress string, imgPath string) { - // Read the image in binary form - imgBytes, err := ioutil.ReadFile(imgPath) - if err != nil { - log.Fatalln(err) - } - - // Target model specification - const MODEL_NAME string = "resnet" - const INPUT_NAME string = "map/TensorArrayStack/TensorArrayGatherV3" - const OUTPUT_NAME string = "softmax_tensor:0" - - // Create Predict Request to OVMS - predictRequest := &pb.PredictRequest{ - ModelSpec: &pb.ModelSpec{ - Name: MODEL_NAME, - SignatureName: "serving_default", - VersionChoice: &pb.ModelSpec_Version{ - Version: &google_protobuf.Int64Value{ - Value: int64(0), - }, - }, - }, - Inputs: map[string]*framework.TensorProto{ - INPUT_NAME: &framework.TensorProto{ - Dtype: framework.DataType_DT_STRING, - TensorShape: &framework.TensorShapeProto{ - Dim: []*framework.TensorShapeProto_Dim{ - &framework.TensorShapeProto_Dim{ - Size: int64(1), - }, - }, - }, - StringVal: [][]byte{imgBytes}, - }, - }, - } - - // Setup connection with the model server via gRPC - conn, err := grpc.Dial(servingAddress, grpc.WithInsecure()) - if err != nil { - log.Fatalf("Cannot connect to the grpc server: %v\n", err) - } - defer conn.Close() - - // Create client instance to prediction service - client := pb.NewPredictionServiceClient(conn) - - // Send predict request and receive response - predictResponse, err := client.Predict(context.Background(), predictRequest) - if err != nil { - log.Fatalln(err) - } +// Target model specification +const MODEL_NAME string = "resnet" +const INPUT_NAME string = "data" +const OUTPUT_NAME string = "resnetv17_dense0_fwd" +const IMG_RESIZE_SIZE uint = 224 + +// Convert slice of 4 bytes to float32 (assumes Little Endian) +func readFloat32(fourBytes []byte) float32 { + buf := bytes.NewBuffer(fourBytes) + var retval float32 + binary.Read(buf, binary.LittleEndian, &retval) + return retval +} - log.Println("Request sent successfully") +//change logits array to softmax values +func makeSoftmaxArr(x []float32) []float32 { + max := x[0] + for _, v := range x { + if v > max { + max = v + } + } + + var expSum float32 = 0.0 + exps := make([]float32, len(x)) + for i, v := range x { + exps[i] = float32(math.Exp(float64(v - max))) + expSum += exps[i] + } + + for i := range exps { + exps[i] /= expSum + } + return exps +} - // Read prediction results - responseProto, ok := predictResponse.Outputs[OUTPUT_NAME] - if !ok { - log.Fatalf("Expected output: %s does not exist in the response", OUTPUT_NAME) - } +func printPredictionFromResponse(responseProto *framework.TensorProto){ responseContent := responseProto.GetTensorContent() - // Get details about output shape outputShape := responseProto.GetTensorShape() dim := outputShape.GetDim() classesNum := dim[1].GetSize() - // Convert bytes to matrix - outMat, err := gocv.NewMatFromBytes(1, int(classesNum), gocv.MatTypeCV32FC1, responseContent) - outMat = outMat.Reshape(1, 1) - - // Find maximum value along with its index in the output - _, maxVal, _, maxLoc := gocv.MinMaxLoc(outMat) + //Convert response to array of float32 + outArr := make([]float32, int(classesNum)) + for i := 0; i < int(classesNum); i++ { + outArr[i] = readFloat32(responseContent[i*4:i*4+4]) + } + softmaxArr := makeSoftmaxArr(outArr) + //Get max value and index + maxVal := softmaxArr[0] + maxLoc := 0 + for i := 0; i < int(classesNum); i++ { + if maxVal < softmaxArr[i] { + maxVal = softmaxArr[i] + maxLoc = i + } + } // Get label of the class with the highest confidence var label string if classesNum == 1000 { - label = labels[maxLoc.X] + label = labels[maxLoc] } else if classesNum == 1001 { - label = labels[maxLoc.X-1] + label = labels[maxLoc-1] } else { fmt.Printf("Unexpected class number in the output") - return + return } fmt.Printf("Predicted class: %s\nClassification confidence: %f%%\n", label, maxVal*100) @@ -140,7 +126,7 @@ func run_with_conversion(servingAddress string, imgPath string) { } // Resize image to match ResNet input - resizedImg := resize.Resize(224, 224, decodedImg, resize.Lanczos3) + resizedImg := resize.Resize(IMG_RESIZE_SIZE, IMG_RESIZE_SIZE, decodedImg, resize.Lanczos3) // Convert image to gocv.Mat type (HWC layout) imgMat, err := gocv.ImageToMatRGB(resizedImg) @@ -149,18 +135,33 @@ func run_with_conversion(servingAddress string, imgPath string) { os.Exit(1) } - newMat := gocv.NewMat() + floatMat := gocv.NewMat() // Convert type so each value is represented by float32 // as in Mat generated by ImageToMatRGB values are represented with 8 bit precision - imgMat.ConvertTo(&newMat, gocv.MatTypeCV32FC2) - - // Having right layout and precision, convert Mat to []bytes - imgBytes := newMat.ToBytes() - - // Target model specification - const MODEL_NAME string = "resnet" - const INPUT_NAME string = "map/TensorArrayStack/TensorArrayGatherV3" - const OUTPUT_NAME string = "softmax_tensor:0" + imgMat.ConvertTo(&floatMat, gocv.MatTypeCV32FC2) + + // Split channels + channels := gocv.Split(floatMat) + + var means = [3]float32{123.675, 116.28, 103.53} + var scales = [3]float32{58.395, 57.12, 57.375} + for i := 0; i < 3; i++ { + // Subtract mean + meanScalar := gocv.NewScalar(float64(means[i]), 0, 0, 0) + meanMat := gocv.NewMatWithSizeFromScalar(meanScalar, channels[i].Rows(), channels[i].Cols(), gocv.MatTypeCV32F) + gocv.Subtract(channels[i], meanMat, &channels[i]) + // Divide by scale + scaleScalar :=gocv.NewScalar(float64(scales[i]), 0, 0, 0) + scaleMat := gocv.NewMatWithSizeFromScalar(scaleScalar, channels[i].Rows(), channels[i].Cols(), gocv.MatTypeCV32F) + gocv.Divide(channels[i], scaleMat, &channels[i]) + } + // Merge channels back in BGR format + normalizedMat := gocv.NewMat() + defer normalizedMat.Close() + gocv.Merge([]gocv.Mat{channels[2], channels[1], channels[0]}, &normalizedMat) + + // Having right layout and precision, convert Mat to []byte + imgBytes := normalizedMat.ToBytes() // Create Predict Request to OVMS predictRequest := &pb.PredictRequest{ @@ -219,37 +220,12 @@ func run_with_conversion(servingAddress string, imgPath string) { if !ok { log.Fatalf("Expected output: %s does not exist in the response", OUTPUT_NAME) } - responseContent := responseProto.GetTensorContent() - - // Get details about output shape - outputShape := responseProto.GetTensorShape() - dim := outputShape.GetDim() - classesNum := dim[1].GetSize() - - // Convert bytes to matrix - outMat, err := gocv.NewMatFromBytes(1, int(classesNum), gocv.MatTypeCV32FC1, responseContent) - outMat = outMat.Reshape(1, 1) - - // Find maximum value along with its index in the output - _, maxVal, _, maxLoc := gocv.MinMaxLoc(outMat) - // Get label of the class with the highest confidence - var label string - if classesNum == 1000 { - label = labels[maxLoc.X] - } else if classesNum == 1001 { - label = labels[maxLoc.X-1] - } else { - fmt.Printf("Unexpected class number in the output") - return - } - - fmt.Printf("Predicted class: %s\nClassification confidence: %f%%\n", label, maxVal*100) + printPredictionFromResponse(responseProto) } func main() { servingAddress := flag.String("serving-address", "localhost:8500", "The tensorflow serving address") - binaryInput := flag.Bool("binary-input", false, "Send JPG/PNG raw bytes") flag.Parse() if flag.NArg() > 2 { @@ -263,9 +239,5 @@ func main() { os.Exit(1) } - if *binaryInput { - run_binary_input(*servingAddress, imgPath) - } else { - run_with_conversion(*servingAddress, imgPath) - } + run_with_conversion(*servingAddress, imgPath) }