added token counter

Author: Gautam-Rajeev

token_counter/openai/local/Dockerfile

@@ -0,0 +1,15 @@
+# Use an official Python runtime as a parent image
+FROM python:3.9-slim
+
+WORKDIR /app
+
+
+#install requirements
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+
+# Copy the rest of the application code to the working directory
+COPY . /app/
+EXPOSE 8000
+# Set the entrypoint for the container
+CMD ["hypercorn", "--bind", "0.0.0.0:8000", "api:app"]

token_counter/openai/local/README.md

@@ -0,0 +1,22 @@
+## Grievance classification:
+
+
+### Purpose :
+Model to classify grievances into 3 buckets :
+- Label 0: 'Agri scheme'
+- Label 1: 'Other agri content'
+- Label 2: 'pest flow'
+- Label 3: 'seed flow'
+
+
+### Testing the model deployment :  
+To run for testing just the Hugging Face deployment for grievence recognition, you can follow the following steps : 
+
+- Git clone the repo
+- Go to current folder location i.e. ``` cd /src/text_classification/flow_classification/local ```
+- Create docker image file and test the api:  
+```
+docker build -t testmodel .
+docker run -p 8000:8000 testmodel
+curl -X POST -H "Content-Type: application/json" -d '{"text": "Where is my money? "}' http://localhost:8000/
+```

token_counter/openai/local/__init__.py

@@ -0,0 +1,2 @@
+from .request import ModelRequest
+from .request import Model

token_counter/openai/local/api.py

@@ -0,0 +1,24 @@
+from model import Model
+from request import ModelRequest
+from quart import Quart, request
+import aiohttp
+
+app = Quart(__name__)
+
+model = None
+
+@app.before_serving
+async def startup():
+    app.client = aiohttp.ClientSession()
+    global model
+    model = Model(app)
+
+@app.route('/', methods=['POST'])
+async def embed():
+    global model
+    data = await request.get_json()
+    req = ModelRequest(**data)
+    return await model.inference(req)
+
+if __name__ == "__main__":
+    app.run()

token_counter/openai/local/model.py

@@ -0,0 +1,16 @@
+import tiktoken
+from request import ModelRequest
+
+class Model():
+    def __new__(cls, context):
+        cls.context = context
+        if not hasattr(cls, 'instance'):
+            cls.instance = super(Model, cls).__new__(cls)
+        model_name = "gpt-3.5-turbo"
+        cls.encoding = tiktoken.encoding_for_model(model_name)
+        return cls.instance
+
+
+    async def inference(self,  request: ModelRequest):
+        num_tokens = len(self.encoding.encode(request.text))
+        return str(num_tokens)

token_counter/openai/local/request.py

@@ -0,0 +1,11 @@
+import requests
+import json
+
+
+class ModelRequest():
+    def __init__(self, text):
+        self.text = text
+
+    def to_json(self):
+        return json.dumps(self, default=lambda o: o.__dict__,
+                          sort_keys=True, indent=4)

token_counter/openai/local/requirements.txt

@@ -0,0 +1,3 @@
+tiktoken
+quart
+aiohttp