Added files for rnn-serving benchmark.

Signed-off-by: L Lakshmanan <[email protected]>

Author: Lakshman

.github/workflows/e2e-rnn-serving.yml

@@ -175,10 +175,11 @@ jobs:
         if: ${{ always() }}
         run: |
           set -x
-          container_list=$(kubectl get pods -n default -o jsonpath="{.items[*].spec.containers[*].name}")
-          for container_name in $container_list
+
+          pod_list=$(kubectl get pods -n default -o jsonpath="{.items[*].name}")
+          for pod in $pod_list
           do
-            kubectl logs -n default -c $container_name -l serving.knative.dev/service=${{ matrix.service }}
+            kubectl logs $pod
           done
 
       - name: Down

benchmarks/rnn-serving/Makefile

@@ -0,0 +1,40 @@
+# MIT License
+
+# Copyright (c) 2024 EASE lab
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+ROOT = ../../
+
+all-image: rnn-serving-python
+
+rnn-serving-python: docker/Dockerfile python/server.py python/rnn.py python/requirements.txt model/rnn_model.pth
+	DOCKER_BUILDKIT=1 docker build \
+	--tag vhiveease/rnn-serving-python:latest \
+	--target rnnServingPython \
+	-f docker/Dockerfile \
+	$(ROOT) --load
+
+## Push images
+push:
+	docker push docker.io/vhiveease/rnn-serving-python:latest 
+
+## Pull images from docker hub
+pull:
+	docker pull docker.io/vhiveease/rnn-serving-python:latest

benchmarks/rnn-serving/docker/Dockerfile

@@ -0,0 +1,43 @@
+# MIT License
+
+# Copyright (c) 2024 EASE lab
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+    
+#---------- PYTHON -----------#
+# First stage (Builder):
+# Install gRPC and all other dependencies
+FROM vhiveease/python-slim:latest as rnnServingPythonBuilder
+WORKDIR /py
+COPY ./benchmarks/rnn-serving/python/requirements.txt ./requirements.txt
+RUN pip3 install --user -r requirements.txt
+COPY ./utils/tracing/python/tracing.py ./
+COPY ./benchmarks/rnn-serving/python/server.py ./
+COPY ./benchmarks/rnn-serving/python/rnn.py ./
+ADD https://raw.githubusercontent.com/vhive-serverless/vSwarm-proto/main/proto/rnn_serving/rnn_serving_pb2_grpc.py ./
+ADD https://raw.githubusercontent.com/vhive-serverless/vSwarm-proto/main/proto/rnn_serving/rnn_serving_pb2.py ./proto/rnn_serving/
+
+# Second stage (Runner):
+FROM vhiveease/python-slim:latest as rnnServingPython
+COPY --from=rnnServingPythonBuilder /root/.local /root/.local
+COPY --from=rnnServingPythonBuilder /py /app
+COPY ./benchmarks/rnn-serving/model/ /app/model
+WORKDIR /app
+# ENV PATH=/root/.local/bin:$PATH
+ENTRYPOINT [ "python3", "/app/server.py" ]

benchmarks/rnn-serving/model/rnn_model.pth

@@ -0,0 +1,12 @@
+numpy
+torch
+grpcio==1.45.0
+grpcio-tools==1.45.0
+opentelemetry-api==1.3.0
+opentelemetry-exporter-zipkin==1.3.0
+opentelemetry-exporter-zipkin-json==1.3.0
+opentelemetry-exporter-zipkin-proto-http==1.3.0
+opentelemetry-instrumentation==0.22b0
+opentelemetry-instrumentation-grpc==0.22b0
+opentelemetry-sdk==1.3.0
+opentelemetry-semantic-conventions==0.22b0

benchmarks/rnn-serving/model/rnn_params.pkl

@@ -0,0 +1,77 @@
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+
+
+class RNN(nn.Module):
+    def __init__(self, input_size, hidden_size, output_size, all_categories, n_categories, all_letters, n_letters):
+        super(RNN, self).__init__()
+        self.hidden_size = hidden_size
+
+        self.all_categories = all_categories
+        self.n_categories = n_categories
+        self.all_letters = all_letters
+        self.n_letters = n_letters
+
+        self.i2h = nn.Linear(n_categories + input_size + hidden_size, hidden_size)
+        self.i2o = nn.Linear(n_categories + input_size + hidden_size, output_size)
+        self.o2o = nn.Linear(hidden_size + output_size, output_size)
+        self.dropout = nn.Dropout(0.1)
+        self.softmax = nn.LogSoftmax(dim=1)
+
+    def forward(self, category, input_tensor, hidden):
+        input_combined = torch.cat((category, input_tensor, hidden), 1)
+        hidden = self.i2h(input_combined)
+        output = self.i2o(input_combined)
+        output_combined = torch.cat((hidden, output), 1)
+        output = self.o2o(output_combined)
+        output = self.dropout(output)
+        output = self.softmax(output)
+        return output, hidden
+
+    def init_hidden(self):
+        return Variable(torch.zeros(1, self.hidden_size))
+
+    @staticmethod
+    def gen_input_tensor(all_letters, n_letters, line):
+        tensor = torch.zeros(len(line), 1, n_letters)
+        for li in range(len(line)):
+            letter = line[li]
+            tensor[li][0][all_letters.find(letter)] = 1
+        return tensor
+
+    @staticmethod
+    def gen_category_tensor(all_categories, n_categories, category):
+        li = all_categories.index(category)
+        tensor = torch.zeros(1, n_categories)
+        tensor[0][li] = 1
+        return tensor
+
+    # Sample from a category and starting letter
+    def sample(self, category, start_letter='A'):
+        category_tensor = Variable(self.gen_category_tensor(self.all_categories, self.n_categories, category))
+        input_tensor = Variable(self.gen_input_tensor(self.all_letters, self.n_letters, start_letter))
+        hidden = self.init_hidden()
+
+        output_name = start_letter
+
+        max_length = 20
+        for i in range(max_length):
+            output, hidden = self.forward(category_tensor, input_tensor[0], hidden)
+            topv, topi = output.data.topk(1)
+            topi = topi[0][0]
+
+            if topi == self.n_letters - 1:
+                break
+            else:
+                letter = self.all_letters[topi]
+                output_name += letter
+
+            input_tensor = Variable(self.gen_input_tensor(self.all_letters, self.n_letters, letter))
+
+        return output_name
+
+    # Get multiple samples from one category and multiple starting letters
+    def samples(self, category, start_letters='ABC'):
+        for start_letter in start_letters:
+            yield self.sample(category, start_letter)

benchmarks/rnn-serving/python/requirements.txt

@@ -0,0 +1,85 @@
+import os
+import sys
+
+import pickle
+import numpy as np
+import torch
+import rnn
+import string
+import random
+
+import tracing
+
+import grpc
+import argparse
+
+from proto.rnn_serving import rnn_serving_pb2
+import rnn_serving_pb2_grpc
+
+from concurrent import futures 
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-a", "--addr", dest="addr", default="0.0.0.0", help="IP address")
+parser.add_argument("-p", "--port", dest="port", default="50051", help="serve port")
+parser.add_argument("-zipkin", "--zipkin", dest="url", default="http://0.0.0.0:9411/api/v2/spans", help="Zipkin endpoint url")
+parser.add_argument("--default_language", default="French", help="Default Language in which string will be generated")
+parser.add_argument("--num_strings", default=3, help="Number of strings to be generated")
+
+args = parser.parse_args()
+args.num_strings = int(args.num_strings)
+
+torch.set_num_threads(1)
+
+rnn_model_path = "model/rnn_model.pth"
+
+all_categories =['French', 'Czech', 'Dutch', 'Polish', 'Scottish', 'Chinese', 'English', 'Italian', 'Portuguese', 'Japanese', 'German', 'Russian', 'Korean', 'Arabic', 'Greek', 'Vietnamese', 'Spanish', 'Irish']
+n_categories = len(all_categories)
+all_letters = string.ascii_letters + " .,;'-"
+n_letters = len(all_letters) + 1 
+
+rnn_model = rnn.RNN(n_letters, 128, n_letters, all_categories, n_categories, all_letters, n_letters)
+rnn_model.load_state_dict(torch.load(rnn_model_path))
+rnn_model.eval()
+
+all_start_letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+if tracing.IsTracingEnabled():
+    tracing.initTracer("rnn-serving-python", url=args.url)
+    tracing.grpcInstrumentClient()
+    tracing.grpcInstrumentServer()
+
+def GenerateStringThroughRNN(language, num_strings):
+    try:
+        start_letters = "".join(random.choices(all_start_letters, k=num_strings))
+        output_names = list(rnn_model.samples(language, start_letters))
+        return f"fn: RNN-Serving | language:{language}.n:{num_strings} | output:{output_names} | runtime: Python"
+    except Exception as e:
+        return f"fn: RNN-Serving | language:{language}.n:{num_strings} | RNNServingFailed.Error:{e} | runtime: Python"
+
+class RNNServing(rnn_serving_pb2_grpc.RNNServingServicer):
+    def GenerateString(self, request, context):
+            
+        if request.language not in all_categories:
+            language = args.default_language
+        else:
+            language = request.language
+            
+        if request.numSamples <= 0:
+            num_strings = args.num_strings
+        else:
+            num_strings = request.numSamples
+
+        msg = GenerateStringThroughRNN(language, num_strings)
+        return rnn_serving_pb2.GetString(message=msg)
+                
+def serve():
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
+    rnn_serving_pb2_grpc.add_RNNServingServicer_to_server(RNNServing(), server)
+    address = (args.addr + ":" + args.port)
+    server.add_insecure_port(address)
+    print("Start RNNServing-python server. Addr: " + address)
+    server.start()
+    server.wait_for_termination()
+
+if __name__ == '__main__':
+    serve()

benchmarks/rnn-serving/python/rnn.py

@@ -0,0 +1,51 @@
+# MIT License
+
+# Copyright (c) 2024 EASE lab
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+version: "3.3"
+services:
+  rnn-serving-python:
+    image: vhiveease/rnn-serving-python:latest
+    container_name: rnn-serving-python
+    entrypoint:
+      - python
+      - /app/server.py
+      - --addr=0.0.0.0
+      - --port=50051
+      - --default_language=French
+      - --num_strings=15
+    ports:
+      - target: 50051
+  relay:
+    image: vhiveease/relay:latest
+    entrypoint:
+      - /app/server
+      - --addr=0.0.0.0:50000
+      - --function-endpoint-url=rnn-serving-python
+      - --function-endpoint-port=50051
+      - --function-name=rnn-serving-python
+      - --value=French
+      - --generator=random
+      - --lowerBound=10
+      - --upperBound=15
+    ports:
+        - published: 50000
+          target: 50000

benchmarks/rnn-serving/python/server.py

@@ -0,0 +1,50 @@
+# MIT License
+
+# Copyright (c) 2024 EASE lab
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+apiVersion: serving.knative.dev/v1
+kind: Service
+metadata:
+  name: rnn-serving-python
+  namespace: default
+spec:
+  template:
+    spec:
+      containers:
+        - image: docker.io/vhiveease/relay:latest
+          ports:
+            - name: h2c
+              containerPort: 50000
+          args:
+            - --addr=0.0.0.0:50000
+            - --function-endpoint-url=0.0.0.0
+            - --function-endpoint-port=50051
+            - --function-name=rnn-serving-python
+            - --value=French
+            - --generator=random
+            - --lowerBound=10
+            - --upperBound=20
+        - image: docker.io/vhiveease/rnn-serving-python:latest
+          args:
+            - --addr=0.0.0.0
+            - --port=50051
+            - --default_language=French
+            - --num_strings=15