To update this repo to latest commit

.gitignore

@@ -4,6 +4,10 @@ node_modules/
 #backend
 gcloud/
 keys/
+data/
+
+#intent-management
+dist/
 
 #dataConstruction directory
 permutations/
@@ -14,10 +18,17 @@ training_data
 
 #python directory
 __pycache__/
+bin/
+!backend/bin/
+include/
 lib/
+share/
 others/
+pyvenv.cfg
+lib64
 
 #optional
 .env
 .vscode/
+.idea/
 .DS_Store

README.md

@@ -1,7 +1,7 @@
 # Chatbot Web Framework for Response Comparisons and Performance Analysis in FAQs
 
 ## Project Summary
-The objective of this project is to design a full stack web application to compare a variable set of FAQ Chatbot API endpoints. The chatbot platform includes a text input and a speech input for intuitiveness and to cater for real live scenarios. For the context of this project, MSF's Baby Bonus is used as a test bed for FAQ question and answer matching.<br/>
+The objective of this project is to design a full stack web application to compare a variable set of FAQ Chatbot API endpoints. The chatbot platform includes a text input and a speech input for intuitiveness and to cater for real live scenarios. For the context of this project, MSF's Baby Bonus and other topics are used as a test bed for FAQ question and answer matching.<br/>
 
 ## Features
 1. Text and Speech based input methods<br/>
@@ -13,24 +13,21 @@ The objective of this project is to design a full stack web application to compa
 1. Govtech's askJamie (Benchmark for accuracy comparison)<br/>
 2. MICL lab's QA Matching Model<br/>
 3. Google's Dialogflow<br/>
-4. Text Classification Model<br/>
-5. Rajat QA Matching Model<br/>
+4. Rajat QA Matching Model<br/>
+5. Rushi's QA Matching Model<br/>
 
 ### Speech to Text API used
 1. AISG's Speech to Text<br/>
 2. Google's Speech API<br/>
-3. Twilio Speech Lab (To be implemented)<br/>
 
 
 ## Deployment Considerations
-Docker is used to set up 3 microservices React Frontend, Node Backend and Flask Server for response comparison. A docker-compose file is used to start up all microservices for deployment usage. Docker deployment resources can be found in the Docker branch of the repository.
+Docker is used to set up 3 microservices React Frontend and NodeJs Backend. A docker-compose file is used to start up all microservices for deployment usage. Docker deployment resources can be found in the Docker branch of the repository.
 
 ## Project organization:
 **frontend directory**: Written on ReactJS, provides the view of the application<br/>
 **backend directory**: Written on NodeJS, provides API endpoints for frontend<br/>
-**comparison directory**: Written on Flask, provides API service for response comparisons<br/>
 **dialogflowfunctions**: Written on NodeJS, used to upload intentions to dialogflow for NLP training<br/>
-**flaskservice directory**: Written on Flask, provides Text Classification Model Training and response comparisons<br/>
 
 ## Master Branch
 
@@ -39,16 +36,37 @@ Following directories must be executed in seperate terminals to run application
 
 1. Frontend Directory<br/>
 2. Backend Directory<br/>
-3. flaskservice Directory(Deep Neural Network + Similarity) or comparison Directory(Similarity only)<br/>
 
 **Additional Requirement**<br/>
 Create a `.env` file in the Backend Directory with the following:<br/>
 
 ```
-DIALOGFLOW_PROJECT_ID= XXX
-AISG_TOKEN= XXX
-SPEECH_API= ws://40.90.170.182:8001/client/ws/speech
-SPEECH_HTTP_API= http://40.90.170.182:8001/client/dynamic/recognize 
+DIALOGFLOW_KEYFILENAME_COVID19=
+DIALOGFLOW_KEYFILENAME_BABYBONUS=
+
+MICL_ENDPOINT=
+RAJAT_ENDPOINT_BABYBONUS=
+RAJAT_ENDPOINT_COVID19=
+RUSHI_ENDPOINT=
+BANI_ENDPOINT=
+
+AISG_CREDENTIALS=
+SPEECH_API=
+SPEECH_HTTP_API=
+SPEECH_HTTP_AUTH=
+SPEECH_ENDPOINT=
+
+DB_HOST=localhost
+DB_PORT=27017
+DB_USER=
+DB_PASS=
+DB_NAME=faqdatastore
+```
+
+Create a `.env` file in the Frontend Directory with the following:<br/>
+
+```
+REACT_APP_API=
 ```
 
 # Installation/setup instructions will be provided in each directory.

backend/.dockerignore

@@ -0,0 +1,15 @@
+.docker
+.dockerignore
+coverage
+dist
+node_modules
+test
+.env*
+.eslintrc.js
+.prettierrc
+Dockerfile
+README.md
+LICENSE
+nest-cli.json
+.DS_Store
+jest.config.js

backend/Dockerfile

@@ -0,0 +1,24 @@
+FROM node:12.2.0-alpine
+
+RUN mkdir -p /srv/app/server
+WORKDIR /srv/app/server
+
+COPY package.json /srv/app/server
+COPY package-lock.json /srv/app/server
+COPY requirements.txt /srv/app/server
+
+RUN apk add --no-cache --virtual .build-deps g++ python3-dev libffi-dev openssl-dev && \
+    apk add --no-cache --update python3 && \
+    pip3 install --upgrade pip setuptools
+
+RUN npm install --production --slient && \
+    npm install -g pm2 && \
+    npm cache clean --force
+
+RUN pip3 install -r requirements.txt
+
+COPY . /srv/app/server
+
+EXPOSE 3001
+
+CMD ["pm2-runtime","ecosystem.config.js"]

backend/app.js

@@ -1,32 +1,56 @@
+require("dotenv").config()
 const express = require('express');
 const dialogflowRouter = require('./routes/dialogflow');
 const miclRouter = require('./routes/micl');
-const flaskRouter = require('./routes/flask');
+const similarityRouter = require('./routes/similarity');
 const askJamieRouter = require('./routes/askJamie');
 const rajatRouter = require('./routes/rajat.js');
+const rushiRouter = require('./routes/rushi.js')
+const baniRouter = require('./routes/bani')
+const modelsRouter = require('./routes/generalModel')
 const STT = require('./controllers/MainController');
 const upload = require('./upload');
 const AC = require('./controllers/AudioController')
+const FAQ = require('./controllers/FAQDataController')
 
 const app = express();
 
 app.use(require('cors')({ origin: true, credentials: true }))
-app.use(express.json())
+app.use(express.json({limit: "50mb"}))
 app.use(express.urlencoded({ extended: false }))
 
+app.use('/similarity', similarityRouter);
 app.use('/dialog', dialogflowRouter);
 app.use('/micl', miclRouter);
-app.use('/flask', flaskRouter);
 app.use('/jamie', askJamieRouter);
 app.use('/rajat', rajatRouter);
+app.use('/rushi', rushiRouter);
+app.use('/bani', baniRouter);
+app.use('/models', modelsRouter())
+
+app.get('/', (req, res, next)=>{res.json({'status':'success'})})
 
 app.post('/stream/google', STT.streamByRecordingGoogle)
 app.post('/stream/aisg', STT.streamByRecordingAISG)
-app.post('/stream/import', upload.single('file'), STT.streamByImport)
+app.post('/stream/import', upload.audio.single('file'), STT.streamByImport)
 
-app.post('/api/upload', upload.single('file'), AC.convertToWAV)
-app.post('/api/speechlabs', upload.none(), STT.speechLabsHTTPRequest)
-app.post('/api/google', upload.none(), STT.googleHTTPRequest)
+app.post('/api/upload', upload.audio.single('file'), AC.convertToWAV)
+app.post('/api/speechlabs', upload.audio.none(), STT.speechLabsHTTPRequest)
+app.post('/api/google', upload.audio.none(), STT.googleHTTPRequest)
 app.get('/api/deletestorage', AC.deleteFiles)
 
+
+app.get('/faqdata/topic/:topic', FAQ.getFAQData)
+app.delete('/faqdata/topic/:topic', FAQ.deleteAllIntents)
+
+app.get('/faqdata/id/:id', FAQ.getIntent)
+app.delete('/faqdata/id/:id', FAQ.deleteIntent)
+app.patch('/faqdata/id/:id', FAQ.updateIntent)
+app.post('/faqdata/id', FAQ.createIntent)
+
+app.post('/faqdata/csv', upload.csv.single('file'), FAQ.processCSV)
+app.get('/faqtopics', FAQ.getFAQTopics)
+app.post('/faqtopics/create', FAQ.addNewTopic)
+app.delete('/faqtopics/:topic', FAQ.deleteTopic)
+
 module.exports = app

backend/ask_jamie.py

@@ -6,6 +6,12 @@
 import requests
 from bs4 import BeautifulSoup
 
+
+"""
+This script is no longer required or in use
+but will remain in the project for archiving and referencing if needed.
+"""
+
 if __name__ == '__main__':
     arg_parser = argparse.ArgumentParser()
     arg_parser.add_argument("--test_questions", type=str, help="Path to newline separated questions stored in txt file")

backend/bin/www

@@ -5,9 +5,10 @@
  */
 
 const app = require('../app')
-const debug = require('debug')('online-asr-backend:server')
+const debug = require('debug')('chatbot-backend:server')
 const http = require('http')
 const io = require('../io')
+const mongoose = require('mongoose')
 
 /**
  * Get port from environment and store in Express.
@@ -31,11 +32,25 @@ server.listen(port)
 server.on('error', onError)
 server.on('listening', onListening)
 
+/**
+ * Connect to mongo database
+ */
+
+mongoose.connect(`mongodb://${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`, {
+  useNewUrlParser: true,
+  useUnifiedTopology: true
+})
+const db = mongoose.connection
+db.on('error', console.error.bind(console, 'connection error:'))
+db.once('open', () => {
+  console.log('Mongodb connected')
+})
+
 /**
  * Normalize a port into a number, string, or false.
  */
 
-function normalizePort (val) {
+function normalizePort(val) {
   const port = parseInt(val, 10)
 
   if (isNaN(port)) {
@@ -55,7 +70,7 @@ function normalizePort (val) {
  * Event listener for HTTP server "error" event.
  */
 
-function onError (error) {
+function onError(error) {
   if (error.syscall !== 'listen') {
     throw error
   }
@@ -83,7 +98,7 @@ function onError (error) {
  * Event listener for HTTP server "listening" event.
  */
 
-function onListening () {
+function onListening() {
   const addr = server.address()
   const bind = typeof addr === 'string'
     ? 'pipe ' + addr