Bugfix/10.6.2/update sophos integration guide (#1058)

JocLRojas · mjabascal10 · osmontero · web-flow · commit 4873ccb996fb · 2025-03-04T18:07:10.000-05:00
* feat: add compliance reports

* fix: update sophos guide

* chore: update version and changelog

* fix: update sophos guide

* fix: update sophos guide

* Enhance log alert message with key details.

Include the specific blocklisted key in the alert message to provide clearer context and improve debugging efficiency. This update ensures more actionable and informative alerts.

* Update how events are retrieved using the Sophos-Central API.

---------

Co-authored-by: Manuel Abascal &lt;mjabascal10@gmail.com&gt;
Co-authored-by: Osmany Montero &lt;osmontero@icloud.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,5 @@
-# UTMStack 10.6.1 Release Notes
-## Bug Fixes
-- Fixed ISM policy to ensure snapshots include only indices older than 24 hours.
+# UTMStack 10.6.2 Release Notes
+
+## Features
+- Added additional compliance reports.
+- Updated the Sophos Central integration guide.
diff --git a/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSophos.java b/backend/src/main/java/com/park/utmstack/domain/application_modules/factory/impl/ModuleSophos.java
@@ -41,32 +41,22 @@ public List<ModuleRequirement> checkRequirements(Long serverId) throws Exception
     public List<ModuleConfigurationKey> getConfigurationKeys(Long groupId) throws Exception {
         List<ModuleConfigurationKey> keys = new ArrayList<>();
 
-        // sophos_api_url
+        // sophos_api_client_id
         keys.add(ModuleConfigurationKey.builder()
             .withGroupId(groupId)
-            .withConfKey("sophos_api_url")
-            .withConfName("API Url")
-            .withConfDescription("Configure Sophos Central api url")
+            .withConfKey("sophos_client_id")
+            .withConfName("Client Id")
+            .withConfDescription("Configure Sophos Central Client Id")
             .withConfDataType("text")
             .withConfRequired(true)
             .build());
 
-        // sophos_authorization
-        keys.add(ModuleConfigurationKey.builder()
-            .withGroupId(groupId)
-            .withConfKey("sophos_authorization")
-            .withConfName("Authorization")
-            .withConfDescription("Configure Sophos Central Authorization")
-            .withConfDataType("password")
-            .withConfRequired(true)
-            .build());
-
-        // sophos_x_api_key
+        // sophos_x_client_secret
         keys.add(ModuleConfigurationKey.builder()
             .withGroupId(groupId)
             .withConfKey("sophos_x_api_key")
-            .withConfName("X-API-KEY")
-            .withConfDescription("Configure Sophos Central api key")
+            .withConfName("Client Secret")
+            .withConfDescription("Configure Sophos Central Client Secret")
             .withConfDataType("password")
             .withConfRequired(true)
             .build());
diff --git a/backend/src/main/resources/config/liquibase/changelog/20250227001_add_compliance_report.xml b/backend/src/main/resources/config/liquibase/changelog/20250227001_add_compliance_report.xml
diff --git a/backend/src/main/resources/config/liquibase/changelog/20250303001_udpate_sophos_guide.xml b/backend/src/main/resources/config/liquibase/changelog/20250303001_udpate_sophos_guide.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
+
+    <changeSet id="20250303001" author="Manuel">
+        <sql>
+            <![CDATA[
+            DELETE FROM utm_module_group
+            WHERE module_id = (SELECT id FROM utm_module WHERE module_name = 'SOPHOS');
+            ]]>
+        </sql>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml
@@ -73,6 +73,8 @@
 
     <include file="/config/liquibase/changelog/20250127001_add_compliance_data.xml" relativeToChangelogFile="false"/>
 
+    <include file="/config/liquibase/changelog/20250227001_add_compliance_report.xml" relativeToChangelogFile="false"/>
 
+    <include file="/config/liquibase/changelog/20250303001_udpate_sophos_guide.xml" relativeToChangelogFile="false"/>
 
 </databaseChangeLog>
diff --git a/correlation/ti/ti.go b/correlation/ti/ti.go
@@ -96,7 +96,7 @@ func IsBlocklisted() {
 
 					if strings.Contains(log, key) {
 						correlation.Alert(
-							fmt.Sprintf("Maliciuos %s found in log", value),
+							fmt.Sprintf("Malicious %s found in log: %s", value, key),
 							"Low",
 							"A blocklisted element has been identified in the logs. Further investigation is recommended.",
 							"",
diff --git a/sophos/configuration/const.go b/sophos/configuration/const.go
@@ -4,6 +4,8 @@ import "github.com/utmstack/UTMStack/sophos/utils"
 
 const (
 	CORRELATIONURL = "http://correlation:8080/v1/newlog"
+	AUTHURL        = "https://id.sophos.com/api/v2/oauth2/token"
+	WHOAMIURL      = "https://api.central.sophos.com/whoami/v1"
 )
 
 func GetInternalKey() string {
diff --git a/sophos/processor/processor.go b/sophos/processor/processor.go
@@ -4,65 +4,181 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"time"
 
 	"github.com/threatwinds/logger"
+	"github.com/utmstack/UTMStack/sophos/configuration"
 	"github.com/utmstack/UTMStack/sophos/utils"
 	"github.com/utmstack/config-client-go/types"
 )
 
 type SophosCentralProcessor struct {
-	XApiKey       string
-	Authorization string
-	ApiUrl        string
+	ClientID     string
+	ClientSecret string
+	TenantID     string
+	DataRegion   string
+	AccessToken  string
+	ExpiresAt    time.Time
 }
 
-func GetSophosCentralProcessor(group types.ModuleGroup) SophosCentralProcessor {
+func getSophosCentralProcessor(group types.ModuleGroup) SophosCentralProcessor {
 	sophosProcessor := SophosCentralProcessor{}
 
 	for _, cnf := range group.Configurations {
 		switch cnf.ConfName {
-		case "X-API-KEY":
-			sophosProcessor.XApiKey = cnf.ConfValue
-		case "Authorization":
-			sophosProcessor.Authorization = cnf.ConfValue
-		case "API Url":
-			sophosProcessor.ApiUrl = cnf.ConfValue
+		case "ClientID":
+			sophosProcessor.ClientID = cnf.ConfValue
+		case "ClientSecret":
+			sophosProcessor.ClientSecret = cnf.ConfValue
 		}
 	}
 	return sophosProcessor
 }
 
-type EventAggregate struct {
-	HasMore    bool                     `json:"has_more"`
-	Items      []map[string]interface{} `json:"items"`
-	NextCursor string                   `json:"next_cursor"`
-}
+func (p *SophosCentralProcessor) getAccessToken() (string, *logger.Error) {
+	data := url.Values{}
+	data.Set("grant_type", "client_credentials")
+	data.Set("client_id", p.ClientID)
+	data.Set("client_secret", p.ClientSecret)
+	data.Set("scope", "token")
 
-func (p *SophosCentralProcessor) GetLogs(group types.ModuleGroup, fromTime int) ([]TransformedLog, *logger.Error) {
-	baseURL := p.ApiUrl + "/siem/v1/events"
+	headers := map[string]string{
+		"Content-Type": "application/x-www-form-urlencoded",
+	}
 
-	u, parseerr := url.Parse(baseURL)
-	if parseerr != nil {
-		return nil, utils.Logger.ErrorF("error parsing URL params: %v", parseerr)
+	response, _, err := utils.DoReq[map[string]any](configuration.AUTHURL, []byte(data.Encode()), http.MethodPost, headers)
+	if err != nil {
+		return "", utils.Logger.ErrorF("error making auth request: %v", err)
 	}
 
-	params := url.Values{}
-	params.Add("limit", "1000")
-	params.Add("from_date", fmt.Sprintf("%d", fromTime))
+	accessToken, ok := response["access_token"].(string)
+	if !ok || accessToken == "" {
+		return "", utils.Logger.ErrorF("access_token not found in response")
+	}
 
-	u.RawQuery = params.Encode()
+	expiresIn, ok := response["expires_in"].(float64)
+	if !ok {
+		return "", utils.Logger.ErrorF("expires_in not found in response")
+	}
+
+	p.AccessToken = accessToken
+	p.ExpiresAt = time.Now().Add(time.Duration(expiresIn) * time.Second)
+
+	return accessToken, nil
+}
+
+type WhoamiResponse struct {
+	ID       string   `json:"id"`
+	ApiHosts ApiHosts `json:"apiHosts"`
+}
+type ApiHosts struct {
+	Global     string `json:"global"`
+	DataRegion string `json:"dataRegion"`
+}
 
+func (p *SophosCentralProcessor) getTenantInfo(accessToken string) *logger.Error {
 	headers := map[string]string{
 		"accept":        "application/json",
-		"Authorization": p.Authorization,
-		"x-api-key":     p.XApiKey,
+		"Authorization": "Bearer " + accessToken,
 	}
 
-	response, _, err := utils.DoReq[EventAggregate](u.String(), nil, http.MethodGet, headers)
+	response, _, err := utils.DoReq[WhoamiResponse](configuration.WHOAMIURL, nil, http.MethodGet, headers)
 	if err != nil {
-		return nil, err
+		return utils.Logger.ErrorF("error making whoami request: %v", err)
+	}
+
+	if response.ID == "" {
+		return utils.Logger.ErrorF("tenant ID not found in whoami response")
+	}
+	p.TenantID = response.ID
+
+	if response.ApiHosts.DataRegion == "" {
+		return utils.Logger.ErrorF("dataRegion not found in whoami response")
+	}
+	p.DataRegion = response.ApiHosts.DataRegion
+
+	return nil
+}
+
+func (p *SophosCentralProcessor) getValidAccessToken() (string, *logger.Error) {
+	if p.AccessToken != "" && time.Now().Before(p.ExpiresAt) {
+		return p.AccessToken, nil
 	}
+	return p.getAccessToken()
+}
 
-	logs := ETLProcess(response, group)
-	return logs, nil
+type EventAggregate struct {
+	Pages Pages            `json:"pages"`
+	Items []map[string]any `json:"items"`
+}
+
+type Pages struct {
+	FromKey string `json:"fromKey"`
+	NextKey string `json:"nextKey"`
+	Size    int64  `json:"size"`
+	MaxSize int64  `json:"maxSize"`
+}
+
+func (p *SophosCentralProcessor) getLogs(fromTime int, nextKey string, group types.ModuleGroup) ([]TransformedLog, string, *logger.Error) {
+	accessToken, err := p.getValidAccessToken()
+	if err != nil {
+		return nil, "", utils.Logger.ErrorF("error getting access token: %v", err)
+	}
+
+	if p.TenantID == "" || p.DataRegion == "" {
+		if err := p.getTenantInfo(accessToken); err != nil {
+			return nil, "", utils.Logger.ErrorF("error getting tenant information: %v", err)
+		}
+	}
+
+	var aggregatedEvents EventAggregate
+	aggregatedEvents.Items = make([]map[string]any, 0)
+	currentNextKey := nextKey
+
+	for {
+		u, err := p.buildURL(fromTime, currentNextKey)
+		if err != nil {
+			return nil, "", utils.Logger.ErrorF("error building URL: %v", err)
+		}
+
+		headers := map[string]string{
+			"Content-Type":  "application/json",
+			"Authorization": "Bearer " + accessToken,
+			"X-Tenant-ID":   p.TenantID,
+		}
+
+		response, _, err := utils.DoReq[EventAggregate](u.String(), nil, http.MethodGet, headers)
+		if err != nil {
+			return nil, "", err
+		}
+
+		aggregatedEvents.Items = append(aggregatedEvents.Items, response.Items...)
+
+		if response.Pages.NextKey == "" {
+			break
+		}
+		currentNextKey = response.Pages.NextKey
+	}
+
+	transformedLogs := ETLProcess(aggregatedEvents, group)
+
+	return transformedLogs, currentNextKey, nil
+}
+
+func (p *SophosCentralProcessor) buildURL(fromTime int, nextKey string) (*url.URL, *logger.Error) {
+	baseURL := p.DataRegion + "/siem/v1/events"
+	u, parseErr := url.Parse(baseURL)
+	if parseErr != nil {
+		return nil, utils.Logger.ErrorF("error parsing url: %v", parseErr)
+	}
+
+	params := url.Values{}
+	if nextKey != "" {
+		params.Set("pageFromKey", nextKey)
+	} else {
+		params.Set("from_date", fmt.Sprintf("%d", fromTime))
+	}
+
+	u.RawQuery = params.Encode()
+	return u, nil
 }
diff --git a/sophos/processor/pull.go b/sophos/processor/pull.go
@@ -11,6 +11,7 @@ import (
 const delayCheck = 300
 
 var timeGroups = make(map[int]int)
+var nextKeys = make(map[int]string)
 
 func PullLogs(group types.ModuleGroup) *logger.Error {
 	utils.Logger.Info("starting log sync for : %s", group.GroupName)
@@ -26,13 +27,15 @@ func PullLogs(group types.ModuleGroup) *logger.Error {
 		timeGroups[group.ModuleID] = epoch + 1
 	}()
 
-	agent := GetSophosCentralProcessor(group)
+	agent := getSophosCentralProcessor(group)
 
-	logs, err := agent.GetLogs(group, timeGroups[group.ModuleID])
+	logs, newNextKey, err := agent.getLogs(timeGroups[group.ModuleID], nextKeys[group.ModuleID], group)
 	if err != nil {
 		return err
 	}
 
+	nextKeys[group.ModuleID] = newNextKey
+
 	err = SendToCorrelation(logs)
 	if err != nil {
 		return err
diff --git a/version.yml b/version.yml
@@ -1 +1 @@
-version: 10.6.1
+version: 10.6.2

Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,8 @@ import "github.com/utmstack/UTMStack/sophos/utils"`
`4`	`4`
`5`	`5`	`const (`
`6`	`6`	`CORRELATIONURL = "http://correlation:8080/v1/newlog"`
	`7`	`+ AUTHURL = "https://id.sophos.com/api/v2/oauth2/token"`
	`8`	`+ WHOAMIURL = "https://api.central.sophos.com/whoami/v1"`
`7`	`9`	`)`
`8`	`10`
`9`	`11`	`func GetInternalKey() string {`