Merge branch 'Samagra-Development:restructure' into restructure

Author: Gautam-Rajeev

config.json

@@ -1,13 +1,22 @@
 {
   "models": [
         {
+      "serviceName": "ner",
+      "modelBasePath": "src/ner/agri_ner_akai/local/.",
+      "apiBasePath": "ner/agri_ner_akai/local/",
+      "containerPort": 8000,
+      "environment": {},
+      "nginx": [],
+      "build": true
+    },    
+        {
       "serviceName": "word_score",
       "modelBasePath": "src/search/word_score/local/.",
-      "apiBasePath": "/search/word_score/local",
+      "apiBasePath": "/search/word_score/local/",
       "containerPort": 8000,
       "environment": {},
       "nginx": [],
-      "build": false
+      "build": true
     },
     {
       "serviceName": "spell_check",

src/ner/README.md

@@ -0,0 +1,15 @@
+# Use an official Python runtime as a parent image
+FROM python:3.9-slim
+
+WORKDIR /app
+
+
+#install requirements
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+
+# Copy the rest of the application code to the working directory
+COPY . /app/
+EXPOSE 8000
+# Set the entrypoint for the container
+CMD ["hypercorn", "--bind", "0.0.0.0:8000", "api:app"]

src/ner/agri_ner_akai/README.md

@@ -0,0 +1,21 @@
+## NER:
+
+
+### Purpose :
+Model to detect
+- crops
+- pests
+- seed type 
+
+
+### Testing the model deployment :  
+To run for testing just the Hugging Face deployment for grievence recognition, you can follow the following steps : 
+
+- Git clone the repo
+- Go to current folder location i.e. ``` cd /src/ner/agri_ner_akai/local ```
+- Create docker image file and test the api:  
+```
+docker build -t testmodel .
+docker run -p 8000:8000 testmodel
+curl -X POST -H "Content-Type: application/json" -d '{"text": "What are tomatoes and potaotes that are being attacked by aphids? "}' http://localhost:8000/
+```

src/ner/agri_ner_akai/local/Dockerfile

@@ -0,0 +1,2 @@
+from .request import ModelRequest
+from .request import Model

src/ner/agri_ner_akai/local/README.md

@@ -0,0 +1,25 @@
+from model import Model
+from request import ModelRequest
+from quart import Quart, request, jsonify
+import aiohttp
+
+app = Quart(__name__)
+
+model = None
+
+@app.before_serving
+async def startup():
+    app.client = aiohttp.ClientSession()
+    global model
+    model = Model(app)
+
+@app.route('/', methods=['POST'])
+async def embed():
+    global model
+    data = await request.get_json()
+    req = ModelRequest(**data)
+    entities = await model.inference(req)
+    return jsonify(entities)  # Convert the list of entities to JSON format
+
+if __name__ == "__main__":
+    app.run()

src/ner/agri_ner_akai/local/__init__.py

@@ -0,0 +1,68 @@
+from transformers import pipeline
+from request import ModelRequest
+
+class Model():
+    def __new__(cls, context):
+        cls.context = context
+        if not hasattr(cls, 'instance'):
+            cls.instance = super(Model, cls).__new__(cls)
+        cls.nlp_ner = pipeline("ner", model="GautamR/akai_ner", tokenizer="GautamR/akai_ner")
+        return cls.instance
+
+    async def inference(self, request: ModelRequest):
+        entities = self.nlp_ner(request.text)
+        return self.aggregate_entities(request.text, entities)
+
+    @staticmethod
+    def aggregate_entities(sentence, entity_outputs):
+        aggregated_entities = []
+        current_entity = None
+
+        for entity in entity_outputs:
+            entity_type = entity["entity"].split("-")[-1]
+
+            # Handle subwords
+            if entity["word"].startswith("##"):
+                # If we encounter an I-PEST or any other I- entity
+                if "I-" in entity["entity"]:
+                    if current_entity:  # Add previous entity
+                        aggregated_entities.append(current_entity)
+                
+                    word_start = sentence.rfind(" ", 0, entity["start"]) + 1
+                    word_end = sentence.find(" ", entity["end"])
+                    if word_end == -1:
+                        word_end = len(sentence)
+
+                    current_entity = {
+                        "entity_group": entity_type,
+                        "score": float(entity["score"]),
+                        "word": sentence[word_start:word_end].replace('.','').replace('?',''),
+                        "start": float(word_start),
+                        "end": float(word_end)
+                    }
+                    aggregated_entities.append(current_entity)
+                    current_entity = None
+
+                else:
+                    # If it's a subword but not an I- entity
+                    current_entity["word"] += entity["word"][2:]
+                    current_entity["end"] = entity["end"]
+                    current_entity["score"] = float((current_entity["score"] + entity["score"]) / 2)  # averaging scores
+
+            # Handle full words
+            else:
+                if current_entity:
+                    aggregated_entities.append(current_entity)
+
+                current_entity = {
+                    "entity_group": entity_type,
+                    "score": float(entity["score"]),
+                    "word": entity["word"],
+                    "start": float(entity["start"]),
+                    "end": float(entity["end"])
+                }
+
+        if current_entity:
+            aggregated_entities.append(current_entity)
+
+        return aggregated_entities

src/ner/agri_ner_akai/local/api.py

@@ -0,0 +1,11 @@
+import requests
+import json
+
+
+class ModelRequest():
+    def __init__(self, text):
+        self.text = text
+
+    def to_json(self):
+        return json.dumps(self, default=lambda o: o.__dict__,
+                          sort_keys=True, indent=4)

src/ner/agri_ner_akai/local/model.py

@@ -0,0 +1,4 @@
+torch==2.0.1 --index-url https://download.pytorch.org/whl/cpu
+transformers
+quart
+aiohttp