- Ingestion Apis Support for python sdk

monger/handlers/ingest.go

@@ -1,12 +1,9 @@
 package handlers
 
 import (
-	"encoding/json"
+	"github.com/NumexaHQ/monger/model"
 	"io/ioutil"
 	"net/http"
-	"time"
-
-	"github.com/NumexaHQ/monger/model"
 
 	"github.com/sirupsen/logrus"
 )
@@ -26,15 +23,13 @@ func (h *Handler) IngestLogs(w http.ResponseWriter, r *http.Request) {
 	// if request, build proxy request
 	// if response, build proxy response
 	if r.Header.Get("X-Numexa-Log-Type") == "request" {
-		var embededRequest *http.Request
-		err = json.Unmarshal(body, &embededRequest)
+		err, newReq, requestTime := model.CreateNewRequest(err, body, r)
 		if err != nil {
-			logrus.Errorf("Error unmarshalling body: %v", err)
-			http.Error(w, "Error unmarshalling body", http.StatusBadRequest)
+			logrus.Errorf("Error creating New Request: %v", err)
+			http.Error(w, "Error creating New Request", http.StatusInternalServerError)
 			return
 		}
-
-		pr, err := model.ProxyRequestBuilderForHTTPRequest(embededRequest, time.Now(), h.AuthDB, r.URL.String(), apiKey)
+		pr, err := model.ProxyRequestBuilderForHTTPRequest(newReq, requestTime, h.AuthDB, newReq.URL.String(), apiKey)
 		if err != nil {
 			logrus.Errorf("Error building proxy request: %v", err)
 		}
@@ -44,15 +39,13 @@ func (h *Handler) IngestLogs(w http.ResponseWriter, r *http.Request) {
 		}()
 		return
 	} else if r.Header.Get("X-Numexa-Log-Type") == "response" {
-		var embededResponse *http.Response
-		err = json.Unmarshal(body, &embededResponse)
+		err, newResponse, initiatedTime, responseTime := model.CreateNewResponse(r, body)
 		if err != nil {
-			logrus.Errorf("Error unmarshalling body: %v", err)
-			http.Error(w, "Error unmarshalling body", http.StatusBadRequest)
+			logrus.Errorf("Error building New Response: %v", err)
+			http.Error(w, "Error building New Response", http.StatusInternalServerError)
 			return
 		}
-
-		pr, err := model.ProxyResponseBuilderForHTTPResponse(r.Context(), embededResponse, h.AuthDB, time.Now(), 0, time.Now(), apiKey)
+		pr, err := model.ProxyResponseBuilderForHTTPResponse(r.Context(), newResponse, h.AuthDB, initiatedTime, 0, responseTime, apiKey)
 		if err != nil {
 			logrus.Errorf("Error building proxy response: %v", err)
 		}

monger/model/python_sdk_request.go

@@ -0,0 +1,60 @@
+package model
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+type PythonSdkRequest struct {
+	RequestTime    string                 `json:"request_time"`
+	SourceIp       string                 `json:"source_ip"`
+	RequestMethod  string                 `json:"request_method"`
+	RequestUrl     string                 `json:"request_url"`
+	RequestHeaders map[string]string      `json:"request_headers"`
+	RequestBody    map[string]interface{} `json:"request_body"`
+}
+
+// CreateNewRequest Helper function to Create new Request received from python Sdk
+func CreateNewRequest(err error, body []byte, r *http.Request) (error, *http.Request, time.Time) {
+	bodyFromPythonSdk := &PythonSdkRequest{}
+	err = json.Unmarshal(body, bodyFromPythonSdk)
+	if err != nil {
+		return err, r, time.Now()
+	}
+	// Creating Body Bytes for New Request
+	newRequestBodyBytes, err := json.Marshal(bodyFromPythonSdk.RequestBody)
+	if err != nil {
+		return err, r, time.Now()
+	}
+	// Creating Body From Body Bytes For New Request
+	newRequestBody := io.NopCloser(bytes.NewBuffer(newRequestBodyBytes))
+
+	u, err := url.Parse(bodyFromPythonSdk.RequestUrl)
+	if err != nil {
+		return err, r, time.Now()
+	}
+	requestTimeStamp, err := time.Parse("2006-01-02 15:04:05.000", bodyFromPythonSdk.RequestTime)
+	if err != nil {
+		return err, r, time.Now()
+	}
+	// Setting up New Request With Existing Request Context
+	newReq, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), newRequestBody)
+	if err != nil {
+		return err, newReq, time.Now()
+	}
+	//Setting Up new Request URl and RemoteAddress
+	newReq.URL = u
+	newReq.RemoteAddr = bodyFromPythonSdk.SourceIp
+
+	// Copying Existing Request Headers into New Request
+	for key, values := range r.Header {
+		for _, value := range values {
+			newReq.Header.Add(key, value)
+		}
+	}
+	return nil, newReq, requestTimeStamp
+}

monger/model/python_sdk_response.go

@@ -0,0 +1,66 @@
+package model
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/NumexaHQ/monger/utils"
+	"io"
+	"net/http"
+	"time"
+)
+
+type PythonSdkResponse struct {
+	InitiatedTimestamp string                 `json:"initiated_timestamp"`
+	ResponseTimestamp  string                 `json:"response_timestamp"`
+	ResponseStatusCode uint16                 `json:"response_status_code"`
+	ResponseBody       map[string]interface{} `json:"response_body"`
+}
+
+// CreateNewResponse Helper function to Create new Response received from python Sdk
+func CreateNewResponse(r *http.Request, body []byte) (error, *http.Response, time.Time, time.Time) {
+	pythonSdkResponse := &PythonSdkResponse{}
+	err := json.Unmarshal(body, pythonSdkResponse)
+	if err != nil {
+		return err, &http.Response{}, time.Now(), time.Now()
+	}
+	// Creating Body Bytes for New Response
+	newResponseBodyBytes, err := json.Marshal(pythonSdkResponse.ResponseBody)
+	if err != nil {
+		return err, &http.Response{}, time.Now(), time.Now()
+	}
+	contentEncoding := r.Header.Get("Content-Encoding")
+	initiatedTimeStamp, err := time.Parse("2006-01-02 15:04:05.000", pythonSdkResponse.InitiatedTimestamp)
+	if err != nil {
+		return err, &http.Response{}, time.Now(), time.Now()
+	}
+	responseTimestamp, err := time.Parse("2006-01-02 15:04:05.000", pythonSdkResponse.ResponseTimestamp)
+	if err != nil {
+		return err, &http.Response{}, time.Now(), time.Now()
+	}
+	var bodyBytes []byte
+	if contentEncoding == "br" {
+		// Encoding Into Brotli
+		brotliBytes, err := utils.EncodeIntoBrotli(newResponseBodyBytes)
+		if err != nil {
+			return err, &http.Response{}, time.Now(), time.Now()
+		}
+		bodyBytes = brotliBytes
+
+	} else if contentEncoding == "gzip" {
+		// Encoding Into Gzip
+		gzipBytes, err := utils.EncodeIntoGzip(newResponseBodyBytes)
+		if err != nil {
+			return err, &http.Response{}, time.Now(), time.Now()
+		}
+		bodyBytes = gzipBytes
+	} else {
+		bodyBytes = newResponseBodyBytes
+	}
+	proxyResponse := &http.Response{
+		StatusCode:    int(pythonSdkResponse.ResponseStatusCode),
+		Body:          io.NopCloser(bytes.NewBuffer(bodyBytes)),
+		ContentLength: int64(len(body)),
+		Header:        r.Header,
+	}
+	return nil, proxyResponse, initiatedTimeStamp, responseTimestamp
+}

monger/utils/http.go

@@ -1,8 +1,12 @@
 package utils
 
 import (
+	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"errors"
+	"fmt"
+	"github.com/andybalholm/brotli"
 	"io"
 )
 
@@ -47,3 +51,31 @@ func ExtractContentFromRequestBody(rb io.ReadCloser) (string, error) {
 
 	return content, nil
 }
+
+// EncodeIntoBrotli helper function to encode the marshal data into brotli format
+func EncodeIntoBrotli(newResponseBodyBytes []byte) ([]byte, error) {
+	brotliBytes := &bytes.Buffer{}
+	writer := brotli.NewWriter(brotliBytes)
+	_, err := writer.Write(newResponseBodyBytes)
+	if err != nil {
+		return nil, fmt.Errorf("error Encoding into brotli %s", err)
+	}
+	if err := writer.Close(); err != nil {
+		return nil, fmt.Errorf("error while closing brotli Writer %s", err)
+	}
+	return brotliBytes.Bytes(), err
+}
+
+// EncodeIntoGzip helper function to encode the marshal data into gzip format
+func EncodeIntoGzip(data []byte) ([]byte, error) {
+	gzipBytes := &bytes.Buffer{}
+	writer := gzip.NewWriter(gzipBytes)
+	_, err := writer.Write(data)
+	if err != nil {
+		return nil, fmt.Errorf("error Encoding into Gzip %s", err)
+	}
+	if err := writer.Close(); err != nil {
+		return nil, fmt.Errorf("error while closing gzip Writer %s", err)
+	}
+	return gzipBytes.Bytes(), nil
+}