Ticket #14 : Add monitoring on function

Author: habarthierry-hue

PrometheusDockerfile

@@ -0,0 +1,3 @@
+FROM prom/prometheus
+
+COPY conf/docker/prometheus/ /etc/prometheus/

conf/docker/prometheus/prometheus.yml

@@ -0,0 +1,26 @@
+# my global config
+global:
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: "prometheus"
+    file_sd_configs:
+    - files:
+      - 'targets.json'

conf/docker/prometheus/targets.json

@@ -0,0 +1,10 @@
+[
+  {
+    "labels": {
+      "job": "node"
+    },
+    "targets": [
+      "test-entry.test.svc.cluster.local"
+    ]
+  }
+]

default.ps1

@@ -76,11 +76,13 @@ task buildLocalDockerImage -depends publish {
 	exec { docker build -f KubernetesDockerfile -t localhost:5000/faaskubernetes . }
 	exec { docker build -f GatewayDockerfile -t localhost:5000/faasgateway . }
 	exec { docker build -f WebsiteDockerfile -t localhost:5000/faaswebsite . }
+	exec { docker build -f PrometheusDockerfile -t localhost:5000/faasprometheus . }
 	exec { docker push localhost:5000/faasgetsql }
 	exec { docker push localhost:5000/faastransform }
 	exec { docker push localhost:5000/faaskubernetes }
 	exec { docker push localhost:5000/faasgateway }
 	exec { docker push localhost:5000/faaswebsite }
+	exec { docker push localhost:5000/faasprometheus }
 }
 
 task initLocalKubernetes {
@@ -94,6 +96,8 @@ task initLocalKubernetes {
 	exec { kubectl apply -f ./kubernetes/faas-gateway-svc.yml --namespace=faas }
 	exec { kubectl apply -f ./kubernetes/run-website.yml --namespace=faas }
 	exec { kubectl apply -f ./kubernetes/faas-website-svc.yml --namespace=faas }
+	exec { kubectl apply -f ./kubernetes/run-prometheus.yml --namespace=faas }
+	exec { kubectl apply -f ./kubernetes/faas-prometheus-svc.yml --namespace=faas }
 }
 
 task builderDockerImage -depends publish {

kubernetes/faas-prometheus-svc.yml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: faas-prometheus-entry
+spec:
+  type: NodePort
+  selector:
+    run: faas-prometheus
+  ports:
+  - port: 9090
+    protocol: TCP
+    targetPort: 9090
+    nodePort: 30004

kubernetes/run-prometheus.yml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: faas-prometheus
+spec:
+  selector:
+    matchLabels:
+      run: faas-prometheus
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        run: faas-prometheus
+    spec:
+      containers:
+      - name: faas-website
+        image: localhost:5000/faasprometheus
+        ports:
+        - containerPort: 9090

src/FaasNet.Runtime.GetSql/FaasNet.Runtime.GetSql.csproj

@@ -12,5 +12,6 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.11.1" />
     <PackageReference Include="System.Data.SqlClient" Version="4.8.3" />
+    <PackageReference Include="prometheus-net.AspNetCore" Version="5.0.1" />
   </ItemGroup>
 </Project>

src/FaasNet.Runtime.GetSql/Middlewares/FunctionMiddleware.cs

@@ -0,0 +1,53 @@
+using FaasNet.Runtime.Parameters;
+using Microsoft.AspNetCore.Http;
+using Newtonsoft.Json;
+using System;
+using System.IO;
+using System.Net;
+using System.Threading.Tasks;
+
+namespace FaasNet.Runtime.GetSql.Middlewares
+{
+    public class FunctionMiddleware
+    {
+        private readonly RequestDelegate _request;
+
+        public FunctionMiddleware(RequestDelegate request)
+        {
+            _request = request;
+        }
+
+        public async Task Invoke(HttpContext context)
+        {
+            if (context.Request.Method != HttpMethods.Post)
+            {
+                await _request.Invoke(context);
+                return;
+            }
+
+            var functionHandler = new FunctionHandler();
+            try
+            {
+                var requestBody = await GetRequest(context.Request.Body);
+                var parameter = JsonConvert.DeserializeObject<FunctionParameter<GetSqlConfiguration>>(requestBody);
+                var result = await functionHandler.Handle(parameter);
+                context.Response.StatusCode = 200;
+                context.Response.ContentType = "application/json";
+                await context.Response.WriteAsync(result.ToString());
+            }
+            catch (Exception ex)
+            {
+                context.Response.StatusCode = (int)HttpStatusCode.InternalServerError;
+                await context.Response.WriteAsync(ex.Message);
+            }
+        }
+
+        private Task<string> GetRequest(Stream inputBody)
+        {
+            using (var reader = new StreamReader(inputBody))
+            {
+                return reader.ReadToEndAsync();
+            }
+        }
+    }
+}

src/FaasNet.Runtime.GetSql/Middlewares/ResponseMetricMiddleware.cs

@@ -0,0 +1,39 @@
+using Microsoft.AspNetCore.Http;
+using System.Diagnostics;
+using System.Threading.Tasks;
+
+namespace FaasNet.Runtime.GetSql.Middlewares
+{
+    public class ResponseMetricMiddleware
+    {
+        private readonly RequestDelegate _request;
+
+        public ResponseMetricMiddleware(RequestDelegate request)
+        {
+            _request = request;
+        }
+
+        public async Task Invoke(HttpContext httpContext, MetricReporter reporter)
+        {
+            var path = httpContext.Request.Path.Value;
+            if (httpContext.Request.Method != HttpMethods.Post)
+            {
+                await _request.Invoke(httpContext);
+                return;
+            }
+
+
+            var sw = Stopwatch.StartNew();
+            try
+            {
+                await _request.Invoke(httpContext);
+            }
+            finally
+            {
+                sw.Stop();
+                reporter.RegisterRequest();
+                reporter.RegisterResponseTime(httpContext.Response.StatusCode, httpContext.Request.Method, sw.Elapsed);
+            }
+        }
+    }
+}

src/FaasNet.Runtime.GetSql/Startup.cs

@@ -1,19 +1,24 @@
 using FaasNet.Runtime.Factories;
-using FaasNet.Runtime.Parameters;
+using FaasNet.Runtime.GetSql.Middlewares;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.DependencyInjection;
 using Newtonsoft.Json;
-using System;
+using Prometheus;
 using System.IO;
-using System.Net;
-using System.Threading.Tasks;
 
 namespace FaasNet.Runtime.GetSql
 {
     public class Startup
     {
+        public void ConfigureServices(IServiceCollection services)
+        {
+            services.AddSingleton<MetricReporter>();
+        }
+
         public void Configure(IApplicationBuilder app)
         {
+            app.UseMetricServer();
             app.Map("/configuration", (c) =>
             {
                 c.Run(async (httpContext) =>
@@ -25,38 +30,8 @@ public void Configure(IApplicationBuilder app)
                     await httpContext.Response.WriteAsync(json);
                 });
             });
-            app.Use(async (context, next) =>
-            {
-                if (context.Request.Method != HttpMethods.Post)
-                {
-                    await next.Invoke();
-                    return;
-                }
-
-                var functionHandler = new FunctionHandler();
-                try
-                {
-                    var requestBody = await GetRequest(context.Request.Body);
-                    var parameter = JsonConvert.DeserializeObject<FunctionParameter<GetSqlConfiguration>>(requestBody);
-                    var result = await functionHandler.Handle(parameter);
-                    context.Response.StatusCode = 200;
-                    context.Response.ContentType = "application/json";
-                    await context.Response.WriteAsync(result.ToString());
-                }
-                catch (Exception ex)
-                {
-                    context.Response.StatusCode = (int)HttpStatusCode.InternalServerError;
-                    await context.Response.WriteAsync(ex.Message);
-                }
-            });
-        }
-
-        private Task<string> GetRequest(Stream inputBody)
-        {
-            using (var reader = new StreamReader(inputBody))
-            {
-                return reader.ReadToEndAsync();
-            }
+            app.UseMiddleware<ResponseMetricMiddleware>();
+            app.UseMiddleware<FunctionMiddleware>();
         }
     }
 }