Merge pull request #132 from evan-cao-wb/re-format-the-code

re format the code

Author: Oceania2018

src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs

@@ -2,7 +2,6 @@
 using System.IO;
 using System.Text;
 using System.Collections.Generic;
-using Tensorflow;
 using static Tensorflow.KerasApi;
 using Tensorflow.Keras.Engine;
 using Tensorflow.NumPy;
@@ -16,11 +15,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using System.Linq;
 using Tensorflow.Keras;
-using System.Numerics;
-using Newtonsoft.Json;
-using Tensorflow.Keras.Layers;
 using BotSharp.Abstraction.Agents;
-using BotSharp.Abstraction.Knowledges;
 
 namespace BotSharp.Plugin.RoutingSpeeder.Providers;
 
@@ -33,11 +28,8 @@ public class IntentClassifier
     private bool _isModelReady;
     public bool isModelReady => _isModelReady;
     private ClassifierSetting _settings;
-
     private string[] _labels;
-
     public string[] Labels => GetLabels();
-
     private int _numLabels
     {
         get
@@ -67,7 +59,7 @@ private void Build()
         }
 
         var vector = _services.GetServices<ITextEmbedding>()
-                                 .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding));
+            .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding));
 
         var layers = new List<ILayer>
         {
@@ -89,28 +81,29 @@ private void Fit(NDArray x, NDArray y, TrainingParams trainingParams)
     {
         _model.compile(optimizer: keras.optimizers.Adam(trainingParams.LearningRate),
             loss: keras.losses.SparseCategoricalCrossentropy(),
-            metrics: new[] { "accuracy" }
-            );
+            metrics: new[] { "accuracy" });
 
-        CallbackParams callback_parameters = new CallbackParams
+        var callback_parameters = new CallbackParams
         {
             Model = _model,
             Epochs = trainingParams.Epochs,
             Verbose = 1,
             Steps = 10
         };
 
-        ICallback earlyStop = new EarlyStopping(callback_parameters, "accuracy");
+        var earlyStop = new EarlyStopping(callback_parameters, "accuracy");
 
-        var callbacks = new List<ICallback>() { earlyStop };
+        var callbacks = new List<ICallback>()
+        {
+            earlyStop
+        };
 
         var weights = LoadWeights(trainingParams.Inference);
 
         _model.fit(x, y,
             batch_size: trainingParams.BatchSize,
             epochs: trainingParams.Epochs,
             callbacks: callbacks,
-            // validation_split: 0.1f,
             shuffle: true);
 
         _model.save_weights(weights);
@@ -120,7 +113,9 @@ private void Fit(NDArray x, NDArray y, TrainingParams trainingParams)
 
     public string LoadWeights(bool inference = true)
     {
-        var agentService = _services.CreateScope().ServiceProvider.GetRequiredService<IAgentService>();
+        var agentService = _services.CreateScope()
+            .ServiceProvider
+            .GetRequiredService<IAgentService>();
 
         var weightsFile = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, $"intent-classifier.h5");
 
@@ -129,13 +124,13 @@ public string LoadWeights(bool inference = true)
             _model.load_weights(weightsFile);
             _isModelReady = true;
             Console.WriteLine($"Successfully load the weights!");
-
         }
         else
         {
             var logInfo = inference ? "No available weights." : "Will implement model training process and write trained weights into local";
             Console.WriteLine(logInfo);
         }
+
         return weightsFile;
     }
 
@@ -152,24 +147,33 @@ public NDArray GetTextEmbedding(string text)
 
     public (NDArray, NDArray) PrepareLoadData()
     {
-        var agentService = _services.CreateScope().ServiceProvider.GetRequiredService<IAgentService>();
-        string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.RAW_DATA_DIR);
-        string saveLabelDirectory = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, _settings.LABEL_FILE_NAME);
+        var agentService = _services.CreateScope()
+            .ServiceProvider
+            .GetRequiredService<IAgentService>();
+        string rootDirectory = Path.Combine(
+            agentService.GetDataDir(), 
+            _settings.RAW_DATA_DIR);
+        string saveLabelDirectory = Path.Combine(
+            agentService.GetDataDir(), 
+            _settings.MODEL_DIR, 
+            _settings.LABEL_FILE_NAME);
 
         if (!Directory.Exists(rootDirectory))
         {
             throw new Exception($"No training data found! Please put training data in this path: {rootDirectory}");
         }
 
+        // Do embedding and store results
         var vector = _services.GetRequiredService<ITextEmbedding>();
-
         var vectorList = new List<float[]>();
-
         var labelList = new List<string>();
 
         foreach (var filePath in GetFiles())
         {
-            var texts = File.ReadAllLines(filePath, Encoding.UTF8).Select(x => TextClean(x)).ToList();
+            var texts = File.ReadAllLines(filePath, Encoding.UTF8)
+                .Select(x => TextClean(x))
+                .ToList();
+
             vectorList.AddRange(vector.GetVectors(texts));
             string fileName = Path.GetFileNameWithoutExtension(filePath);
             labelList.AddRange(Enumerable.Repeat(fileName, texts.Count).ToList());
@@ -185,25 +189,39 @@ public NDArray GetTextEmbedding(string text)
         for (int i = 0; i < vectorList.Count; i++)
         {
             x[i] = vectorList[i];
-            // y[i] = (float)uniqueLabelList.IndexOf(labelList[i]);
             y[i] = (float)Array.IndexOf(uniqueLabelList, labelList[i]);
         }
+
         return (x, y);
     }
 
     public string[] GetFiles(string prefix = "intent")
     {
-        var agentService = _services.CreateScope().ServiceProvider.GetRequiredService<IAgentService>();
+        var agentService = _services.CreateScope()
+            .ServiceProvider
+            .GetRequiredService<IAgentService>();
         string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.RAW_DATA_DIR);
-        return Directory.GetFiles(rootDirectory).Where(x => Path.GetFileNameWithoutExtension(x).StartsWith(prefix)).OrderBy(x => x).ToArray();
+
+        return Directory.GetFiles(rootDirectory)
+            .Where(x => Path.GetFileNameWithoutExtension(x)
+            .StartsWith(prefix))
+            .OrderBy(x => x)
+            .ToArray();
     }
 
     public string[] GetLabels()
     {
         if (_labels == null)
         {
-            var agentService = _services.CreateScope().ServiceProvider.GetRequiredService<IAgentService>();
-            string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, _settings.LABEL_FILE_NAME);
+            var agentService = _services.CreateScope()
+                .ServiceProvider
+                .GetRequiredService<IAgentService>();
+            string rootDirectory = Path.Combine(
+                agentService.GetDataDir(), 
+                _settings.MODEL_DIR,
+                _settings.LABEL_FILE_NAME
+                );
+
             var labelText = File.ReadAllLines(rootDirectory);
             _labels = labelText.OrderBy(x => x).ToArray();
         }
@@ -217,9 +235,11 @@ public string TextClean(string text)
         // Remove digits
         // To lowercase
         var processedText = Regex.Replace(text, "[AB0-9]", " ");
-        processedText = string.Join("", processedText.Select(c => char.IsPunctuation(c) ? ' ' : c).ToList());
-        processedText = processedText.Replace("  ", " ").ToLower();
-        return processedText;
+        var replacedTextList = processedText.Select(c => char.IsPunctuation(c) ? ' ' : c).ToList();
+
+        return string.Join("", replacedTextList)
+            .Replace("  ", " ")
+            .ToLower();
     }
 
     public string Predict(NDArray vector, float confidenceScore = 0.9f)
@@ -229,8 +249,8 @@ public string Predict(NDArray vector, float confidenceScore = 0.9f)
             InitClassifer();
         }
 
+        // Generate and post-process prediction
         var prob = _model.predict(vector).numpy();
-
         var probLabel = tf.arg_max(prob, -1).numpy().ToArray<long>();
         prob = np.squeeze(prob, axis: 0);
 
@@ -239,9 +259,9 @@ public string Predict(NDArray vector, float confidenceScore = 0.9f)
             return string.Empty;
         }
 
-        var prediction = _labels[probLabel[0]];
+        var labelIndex = probLabel[0];
 
-        return prediction;
+        return _labels[labelIndex];
     }
     public void InitClassifer(bool inference = true)
     {