Skip to content

Commit 29f47b0

Browse files
v-atulyadavv-atulyadav
authored
Merge pull request #11177 from niralishah-crest/BitSightDataConnectorIssueFix
Updated Data Connector code with fix of jsondecoder error and upgraded python version from 3.9 to 3.11 for BitSight
2 parents 277abb6 + cc8af15 commit 29f47b0

File tree

13 files changed

+255
-56
lines changed

13 files changed

+255
-56
lines changed

‎Solutions/BitSight/Data Connectors/BitSightDataConnector/AlertsGraphStatisticsDetails/bitsight_statistics.py‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
ALERTS_DATA_TABLE,
2020
COMPANIES,
2121
ENDPOINTS,
22-
ALERT_GRAPH_STATISTICS_FUNC_NAME,
22+
ALERT_GRAPH_STATISTICS_FUNC_NAME
2323
)
2424

2525

@@ -96,7 +96,7 @@ def get_risk_vector_data(
9696
data_to_post = []
9797
risk_vector_data = []
9898
checkpoint_key = "{}".format(company_guid)
99-
checkpoint_data = self.checkpoint_obj.get_last_data(state)
99+
checkpoint_data = self.checkpoint_obj.get_last_data(state, table_name=table_name)
100100
last_data = self.checkpoint_obj.get_endpoint_last_data(
101101
checkpoint_data, endpoint, checkpoint_key
102102
)
@@ -124,9 +124,8 @@ def get_risk_vector_data(
124124
self.send_data_to_sentinel(
125125
risk_vector_data, table_name, company_name, endpoint
126126
)
127-
# data_to_post = str(data_to_post)
128127
self.checkpoint_obj.save_checkpoint(
129-
state, checkpoint_data, endpoint, checkpoint_key, data_to_post
128+
state, checkpoint_data, endpoint, "{}_{}".format(table_name, "Checkpoint"), checkpoint_key, data_to_post
130129
)
131130
except BitSightException:
132131
raise BitSightException()
@@ -183,7 +182,7 @@ def get_diligence_historical_statistics_details(self, company_name, company_guid
183182
post_data = []
184183
checkpoint_key = "{}".format(company_guid)
185184
checkpoint_data = self.checkpoint_obj.get_last_data(
186-
self.diligence_historical_statistics_state
185+
self.diligence_historical_statistics_state, table_name=DILIGENCE_HISTORICAL_STATISTICS_TABLE
187186
)
188187
last_data = self.checkpoint_obj.get_endpoint_last_data(
189188
checkpoint_data, "diligence_historical-statistics", company_guid
@@ -218,11 +217,11 @@ def get_diligence_historical_statistics_details(self, company_name, company_guid
218217
company_name,
219218
"diligence historical statistics",
220219
)
221-
# checkpoint_data_to_post = str(checkpoint_data_to_post)
222220
self.checkpoint_obj.save_checkpoint(
223221
self.diligence_historical_statistics_state,
224222
checkpoint_data,
225223
"diligence_historical-statistics",
224+
"{}_{}".format(DILIGENCE_HISTORICAL_STATISTICS_TABLE, "Checkpoint"),
226225
checkpoint_key,
227226
checkpoint_data_to_post,
228227
)
@@ -254,7 +253,7 @@ def get_graph_data(self, company_name, company_guid):
254253
rating_diff = None
255254
last_date = None
256255
checkpoint_key = "{}".format(company_guid)
257-
checkpoint_data = self.checkpoint_obj.get_last_data(self.graph_state)
256+
checkpoint_data = self.checkpoint_obj.get_last_data(self.graph_state, table_name=GRAPH_DATA_TABLE)
258257
last_data = self.checkpoint_obj.get_endpoint_last_data(
259258
checkpoint_data, "graph_data", company_guid
260259
)
@@ -307,6 +306,7 @@ def get_graph_data(self, company_name, company_guid):
307306
self.graph_state,
308307
checkpoint_data,
309308
"graph_data",
309+
"{}_{}".format(GRAPH_DATA_TABLE, "Checkpoint"),
310310
checkpoint_key,
311311
data_to_post,
312312
)
@@ -340,7 +340,7 @@ def get_alerts_details(self, company_name, company_guid):
340340
try:
341341
data_to_post = None
342342
checkpoint_key = "{}".format(company_guid)
343-
checkpoint_data = self.checkpoint_obj.get_last_data(self.alerts_state)
343+
checkpoint_data = self.checkpoint_obj.get_last_data(self.alerts_state, table_name=ALERTS_DATA_TABLE)
344344
last_date = self.checkpoint_obj.get_endpoint_last_data(
345345
checkpoint_data, "alerts_data", company_guid
346346
)
@@ -403,6 +403,7 @@ def get_alerts_details(self, company_name, company_guid):
403403
self.alerts_state,
404404
checkpoint_data,
405405
"alerts_data",
406+
"{}_{}".format(ALERTS_DATA_TABLE, "Checkpoint"),
406407
checkpoint_key,
407408
data_to_post,
408409
)
@@ -422,7 +423,7 @@ def get_all_copmanies_alerts_graph_statisctics_details(
422423
company_names (list): List of company names.
423424
"""
424425
fetching_index = self.get_last_data_index(
425-
company_names, self.checkpoint_obj, self.company_state
426+
company_names, self.checkpoint_obj, self.company_state, table_name="{}_{}".format(ALERTS_DATA_TABLE, "Statistics")
426427
)
427428
for company_index in range(fetching_index + 1, len(logs_data)):
428429
company_name = logs_data[company_index].get("name_s")
@@ -443,6 +444,7 @@ def get_all_copmanies_alerts_graph_statisctics_details(
443444
self.company_state,
444445
company_name,
445446
"statisctics_company",
447+
"{}_{}".format(ALERTS_DATA_TABLE, "Statistics_Company_Checkpoint"),
446448
company_name_flag=True,
447449
)
448450

‎Solutions/BitSight/Data Connectors/BitSightDataConnector/BitSight.zip‎

-17.6 MB
Binary file not shown.

‎Solutions/BitSight/Data Connectors/BitSightDataConnector/BreachesDetails/bitsight_breaches.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ def get_all_companies_breaches_details(self, company_names, logs_data):
7878
"""
7979
count_companies = 0
8080
fetching_index = self.get_last_data_index(
81-
company_names, self.checkpoint_obj, self.breach_company_state
81+
company_names, self.checkpoint_obj, self.breach_company_state, table_name=consts.BREACHES_TABLE_NAME
8282
)
8383
for company_index in range(fetching_index + 1, len(logs_data)):
8484
company_name = logs_data[company_index].get("name_s")
@@ -98,6 +98,7 @@ def get_all_companies_breaches_details(self, company_names, logs_data):
9898
self.breach_company_state,
9999
company_name,
100100
"breaches",
101+
"{}_{}".format(consts.BREACHES_TABLE_NAME, "Company_Checkpoint"),
101102
company_name_flag=True,
102103
)
103104
applogger.info(
@@ -171,7 +172,7 @@ def get_breaches_data(self, company_name, company_guid):
171172
)
172173
)
173174
return
174-
last_data = self.checkpoint_obj.get_last_data(self.breaches_details_state)
175+
last_data = self.checkpoint_obj.get_last_data(self.breaches_details_state, table_name=consts.BREACHES_TABLE_NAME)
175176
last_checkpoint_company = self.checkpoint_obj.get_endpoint_last_data(
176177
last_data, "breaches", company_guid
177178
)
@@ -188,6 +189,7 @@ def get_breaches_data(self, company_name, company_guid):
188189
self.breaches_details_state,
189190
last_data,
190191
"breaches",
192+
"{}_{}".format(consts.BREACHES_TABLE_NAME, "Checkpoint"),
191193
checkpoint_key,
192194
checkpoint_date,
193195
)

‎Solutions/BitSight/Data Connectors/BitSightDataConnector/CompaniesDetails/bitsight_companies.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
COMPANIES_RATING_DETAILS_TABLE_NAME,
1212
COMPANY_DETAIL_TABLE_NAME,
1313
ENDPOINTS,
14-
COMPANY_DETAILS_FUNC_NAME,
14+
COMPANY_DETAILS_FUNC_NAME
1515
)
1616
from ..SharedCode.get_logs_data import get_logs_data
1717
from ..SharedCode.logger import applogger
@@ -61,10 +61,10 @@ def get_company_details(self, company_name, company_guid):
6161
post_data_ratings = []
6262
checkpoint_key = "{}".format(company_guid)
6363
checkpoint_data_company = self.checkpoint_obj.get_last_data(
64-
self.company_detail_state
64+
self.company_detail_state, table_name=COMPANY_DETAIL_TABLE_NAME
6565
)
6666
checkpoint_data_company_ratings = self.checkpoint_obj.get_last_data(
67-
self.company_rating_state
67+
self.company_rating_state, table_name=COMPANIES_RATING_DETAILS_TABLE_NAME
6868
)
6969
last_data_company_details = self.checkpoint_obj.get_endpoint_last_data(
7070
checkpoint_data_company, "companies_details", checkpoint_key
@@ -114,6 +114,7 @@ def get_company_details(self, company_name, company_guid):
114114
self.company_rating_state,
115115
checkpoint_data_company_ratings,
116116
"companies_ratings_details",
117+
"{}_{}".format(COMPANIES_RATING_DETAILS_TABLE_NAME, "Checkpoint"),
117118
checkpoint_key,
118119
data_to_post,
119120
)
@@ -137,6 +138,7 @@ def get_company_details(self, company_name, company_guid):
137138
self.company_detail_state,
138139
checkpoint_data_company,
139140
"companies_details",
141+
"{}_{}".format(COMPANY_DETAIL_TABLE_NAME, "Checkpoint"),
140142
checkpoint_key,
141143
data_to_post,
142144
)
@@ -166,7 +168,7 @@ def get_all_copmanies_details(self, logs_data, company_names):
166168
"""
167169
count_companies = 0
168170
fetching_index = self.get_last_data_index(
169-
company_names, self.checkpoint_obj, self.company_state
171+
company_names, self.checkpoint_obj, self.company_state, table_name=COMPANY_DETAIL_TABLE_NAME
170172
)
171173
for company_index in range(fetching_index + 1, len(logs_data)):
172174
company_name = logs_data[company_index].get("name_s")
@@ -186,6 +188,7 @@ def get_all_copmanies_details(self, logs_data, company_names):
186188
self.company_state,
187189
company_name,
188190
"portfolio_company",
191+
"{}_{}".format(COMPANY_DETAIL_TABLE_NAME, "Company_Checkpoint"),
189192
company_name_flag=True,
190193
)
191194
applogger.info(

‎Solutions/BitSight/Data Connectors/BitSightDataConnector/FindingsDetails/bitsight_findings.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
FINDINGS_TABLE_NAME,
1414
COMPANIES,
1515
ENDPOINTS,
16+
FINDING_DETAILS_QUERY
1617
)
1718

1819

@@ -40,7 +41,7 @@ def __init__(self, start_time) -> None:
4041
def get_all_copmanies_findings_details(self, logs_data, company_names):
4142
count_companies = 0
4243
fetching_index = self.get_last_data_index(
43-
company_names, self.checkpoint_obj, self.company_state
44+
company_names, self.checkpoint_obj, self.company_state, table_name=FINDINGS_TABLE_NAME
4445
)
4546
for company_index in range(fetching_index + 1, len(logs_data)):
4647
company_name = logs_data[company_index].get("name_s")
@@ -63,6 +64,7 @@ def get_all_copmanies_findings_details(self, logs_data, company_names):
6364
self.company_state,
6465
company_name,
6566
"findings_company",
67+
"{}_{}".format(FINDINGS_TABLE_NAME, "Company_Checkpoint"),
6668
company_name_flag=True,
6769
)
6870

@@ -125,7 +127,7 @@ def get_findings_details(self, company_name, company_guid):
125127
{"risk_category": "Compromised Systems"},
126128
{"risk_category": "User Behavior"},
127129
]
128-
last_data = self.checkpoint_obj.get_last_data(self.findings_state)
130+
last_data = self.checkpoint_obj.get_last_data(self.findings_state, table_name=FINDINGS_TABLE_NAME, checkpoint_query=FINDING_DETAILS_QUERY)
129131
findings_url = self.base_url + self.findings_endpoint_path.format(
130132
company_guid
131133
)
@@ -173,6 +175,7 @@ def get_findings_details(self, company_name, company_guid):
173175
self.findings_state,
174176
last_data,
175177
"findings_details",
178+
"{}_{}".format(FINDINGS_TABLE_NAME, "Checkpoint"),
176179
checkpoint_key,
177180
str(data_to_post.date()),
178181
)
@@ -196,6 +199,13 @@ def get_findings_details(self, company_name, company_guid):
196199
c_data["next1"] = self.get_bitsight_data(findings_url, params)
197200
next_link = c_data["next1"].get("links").get("next")
198201
length_results = len(c_data.get("next1").get("results"))
202+
if length_results == 0:
203+
applogger.info(
204+
'BitSight: No new findings found for {} on page {} ({})'.format(
205+
company_name, page, risk
206+
)
207+
)
208+
break
199209
applogger.info(
200210
"BitSight: Got {} findings for {} on page {}".format(
201211
length_results, company_name, page
@@ -222,6 +232,7 @@ def get_findings_details(self, company_name, company_guid):
222232
self.findings_state,
223233
last_data,
224234
"findings_details",
235+
"{}_{}".format(FINDINGS_TABLE_NAME, "Checkpoint"),
225236
checkpoint_key,
226237
str(data_to_post.date()),
227238
)

‎Solutions/BitSight/Data Connectors/BitSightDataConnector/FindingsSummaryDetails/bitsight_findings_summary.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ def get_all_companies_findings_summary_details(self, company_names, logs_data):
8383
"""
8484
count_companies = 0
8585
fetching_index = self.get_last_data_index(
86-
company_names, self.checkpoint_obj, self.findings_summary_company_state
86+
company_names, self.checkpoint_obj, self.findings_summary_company_state, table_name=consts.FINDINGS_SUMMARY_TABLE_NAME
8787
)
8888
for company_index in range(fetching_index + 1, len(logs_data)):
8989
company_name = logs_data[company_index].get("name_s")
@@ -103,6 +103,7 @@ def get_all_companies_findings_summary_details(self, company_names, logs_data):
103103
self.findings_summary_company_state,
104104
company_name,
105105
"findings_summary",
106+
"{}_{}".format(consts.FINDINGS_SUMMARY_TABLE_NAME, "Company_Checkpoint"),
106107
company_name_flag=True,
107108
)
108109
applogger.info(
@@ -232,7 +233,7 @@ def create_findings_summary_data(
232233
company_guid (str): GUID of the company.
233234
"""
234235
last_data = self.checkpoint_obj.get_last_data(
235-
self.findings_summary_details_state
236+
self.findings_summary_details_state, table_name=consts.FINDINGS_SUMMARY_TABLE_NAME
236237
)
237238
last_checkpoint_company = self.checkpoint_obj.get_endpoint_last_data(
238239
last_data, "findings_summary", company_guid
@@ -276,6 +277,7 @@ def create_findings_summary_data(
276277
self.findings_summary_details_state,
277278
last_data,
278279
"findings_summary",
280+
"{}_{}".format(consts.FINDINGS_SUMMARY_TABLE_NAME, "Checkpoint"),
279281
checkpoint_key,
280282
last_checkpoint_company,
281283
)

‎Solutions/BitSight/Data Connectors/BitSightDataConnector/PortFolioCompanies/bitsight_portfolio.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
COMPANIES_TABLE_NAME,
1010
ENDPOINTS,
1111
LOGS_STARTS_WITH,
12-
PORTFOLIO_PAGE_SIZE,
12+
PORTFOLIO_PAGE_SIZE
1313
)
1414
from ..SharedCode.get_logs_data import get_logs_data
1515
from ..SharedCode.logger import applogger

‎Solutions/BitSight/Data Connectors/BitSightDataConnector/SharedCode/bitsight_client.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from ..SharedCode.azure_sentinel import MicrosoftSentinel
99
from .bitsight_exception import BitSightException
1010
from .utils import CheckpointManager
11-
from .consts import API_TOKEN, BASE_URL, LOGS_STARTS_WITH
11+
from .consts import API_TOKEN, BASE_URL, LOGS_STARTS_WITH, COMPANY_FETCH_QUERY
1212
from .logger import applogger
1313

1414

@@ -36,7 +36,6 @@ def check_environment_var_exist(self, environment_var):
3636
Returns:
3737
bool: True if all environment variables are set, False otherwise.
3838
"""
39-
__method_name = inspect.currentframe().f_code.co_name
4039
try:
4140
applogger.debug(
4241
"BitSight: check_environment_var_exist: started checking existence of all custom environment variable"
@@ -86,20 +85,21 @@ def generate_auth_token(self):
8685
raise BitSightException()
8786

8887
def get_last_data_index(
89-
self, company_names, checkpoint_obj: CheckpointManager, company_state
88+
self, company_names, checkpoint_obj: CheckpointManager, company_state, table_name
9089
):
9190
"""Get the index for fetching last data.
9291
9392
Args:
9493
company_names (list): List of company names.
9594
checkpoint_obj (CheckpointManager): CheckpointManager object.
9695
company_state (str): State of the company.
96+
table_name (str): Table name from which data should be fetched in case of checkpoint file corrupted.
9797
9898
Returns:
9999
int: Index for fetching last data.
100100
"""
101101
last_company_name = checkpoint_obj.get_last_data(
102-
company_state, company_name_flag=True
102+
company_state, company_name_flag=True, table_name=table_name, checkpoint_query=COMPANY_FETCH_QUERY
103103
)
104104
fetching_index = -1
105105
if last_company_name is not None:

‎Solutions/BitSight/Data Connectors/BitSightDataConnector/SharedCode/consts.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,20 @@
5252
FINDINGS_FUNC_NAME = "Findings:"
5353
COMPANY_DETAILS_FUNC_NAME = "Company Details:"
5454
ALERT_GRAPH_STATISTICS_FUNC_NAME = "Alerts-Graph-statistics Details:"
55+
PORTFOLIO_COMPANY_QUERY = """{}_CL
56+
| summarize arg_max(TimeGenerated, *) by guid_g
57+
| sort by name_s asc
58+
| project name_s, guid_g""".format(
59+
COMPANIES_TABLE_NAME
60+
)
61+
FINDING_DETAILS_QUERY = """{}_CL
62+
| summarize arg_max(TimeGenerated, *) by Key_s
63+
| sort by Key_s asc
64+
| project Key_s, Value_s"""
65+
CHECKPOINT_DATA_QUERY = """{}_CL
66+
| summarize arg_max(TimeGenerated, *) by Key_g
67+
| sort by Key_g asc
68+
| project Key_g, Value_s"""
69+
COMPANY_FETCH_QUERY = """{}_CL
70+
| summarize arg_max(TimeGenerated, *) by Key_s
71+
| project Key_s, Value_s"""

0 commit comments

Comments
 (0)