Skip to content

Commit 74d67da

Browse files
aalva500-progaaarone90RyanL1997
authored
Adding IT suite for PPL-based dashboards in Neo for CloudWatch Lake (opensearch-project#4695)
Co-authored-by: Aaron Alvarez <[email protected]> Co-authored-by: Jialiang Liang <[email protected]>
1 parent 3d548b9 commit 74d67da

17 files changed

+4167
-0
lines changed

‎integ-test/src/test/java/org/opensearch/sql/ppl/dashboard/CloudTrailPplDashboardIT.java‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.ppl.dashboard;
7+
8+
import static org.opensearch.sql.util.MatcherUtils.rows;
9+
import static org.opensearch.sql.util.MatcherUtils.schema;
10+
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
11+
import static org.opensearch.sql.util.MatcherUtils.verifySchema;
12+
13+
import java.io.IOException;
14+
import org.json.JSONObject;
15+
import org.junit.jupiter.api.Test;
16+
import org.opensearch.sql.legacy.TestUtils;
17+
import org.opensearch.sql.ppl.PPLIntegTestCase;
18+
19+
/**
20+
* Integration tests for CloudTrail PPL dashboard queries using exact original dashboard query
21+
* format. These tests ensure that CloudTrail-related PPL queries work correctly with actual test
22+
* data.
23+
*/
24+
public class CloudTrailPplDashboardIT extends PPLIntegTestCase {
25+
26+
private static final String CLOUDTRAIL_LOGS_INDEX = "cloudtrail_logs";
27+
28+
@Override
29+
public void init() throws Exception {
30+
super.init();
31+
enableCalcite();
32+
loadCloudTrailLogsIndex();
33+
}
34+
35+
private void loadCloudTrailLogsIndex() throws IOException {
36+
if (!TestUtils.isIndexExist(client(), CLOUDTRAIL_LOGS_INDEX)) {
37+
String mapping = TestUtils.getMappingFile("mappings/cloudtrail_logs_index_mapping.json");
38+
TestUtils.createIndexByRestClient(client(), CLOUDTRAIL_LOGS_INDEX, mapping);
39+
TestUtils.loadDataByRestClient(
40+
client(),
41+
CLOUDTRAIL_LOGS_INDEX,
42+
"src/test/java/org/opensearch/sql/ppl/dashboard/testdata/cloudtrail_logs.json");
43+
}
44+
}
45+
46+
@Test
47+
public void testTotalEventsCount() throws IOException {
48+
String query =
49+
String.format("source=%s | stats count() as `Event Count`", CLOUDTRAIL_LOGS_INDEX);
50+
51+
JSONObject response = executeQuery(query);
52+
verifySchema(response, schema("Event Count", null, "bigint"));
53+
verifyDataRows(response, rows(100));
54+
}
55+
56+
@Test
57+
public void testEventsOverTime() throws IOException {
58+
String query =
59+
String.format("source=%s | stats count() by span(start_time, 30d)", CLOUDTRAIL_LOGS_INDEX);
60+
61+
JSONObject response = executeQuery(query);
62+
verifySchema(
63+
response,
64+
schema("count()", null, "bigint"),
65+
schema("span(start_time,30d)", null, "timestamp"));
66+
}
67+
68+
@Test
69+
public void testEventsByAccountIds() throws IOException {
70+
String query =
71+
String.format(
72+
"source=%s | where isnotnull(userIdentity.accountId) | stats count() as Count by"
73+
+ " userIdentity.accountId | sort - Count | head 10",
74+
CLOUDTRAIL_LOGS_INDEX);
75+
76+
JSONObject response = executeQuery(query);
77+
verifySchema(
78+
response,
79+
schema("Count", null, "bigint"),
80+
schema("userIdentity.accountId", null, "string"));
81+
}
82+
83+
@Test
84+
public void testEventsByCategory() throws IOException {
85+
String query =
86+
String.format(
87+
"source=%s | stats count() as Count by eventCategory | sort - Count | head 5",
88+
CLOUDTRAIL_LOGS_INDEX);
89+
90+
JSONObject response = executeQuery(query);
91+
verifySchema(
92+
response, schema("Count", null, "bigint"), schema("eventCategory", null, "string"));
93+
verifyDataRows(response, rows(100, "Management"));
94+
}
95+
96+
@Test
97+
public void testEventsByRegion() throws IOException {
98+
String query =
99+
String.format(
100+
"source=%s | stats count() as Count by `awsRegion` | sort - Count | head 10",
101+
CLOUDTRAIL_LOGS_INDEX);
102+
103+
JSONObject response = executeQuery(query);
104+
verifySchema(response, schema("Count", null, "bigint"), schema("awsRegion", null, "string"));
105+
verifyDataRows(
106+
response,
107+
rows(12, "us-west-1"),
108+
rows(12, "ca-central-1"),
109+
rows(9, "us-west-2"),
110+
rows(8, "ap-southeast-1"),
111+
rows(8, "ap-northeast-1"),
112+
rows(7, "us-east-2"),
113+
rows(7, "sa-east-1"),
114+
rows(7, "eu-north-1"),
115+
rows(7, "ap-south-1"),
116+
rows(6, "ap-southeast-2"));
117+
}
118+
119+
@Test
120+
public void testTop10EventAPIs() throws IOException {
121+
String query =
122+
String.format(
123+
"source=%s | stats count() as Count by `eventName` | sort - Count | head 10",
124+
CLOUDTRAIL_LOGS_INDEX);
125+
126+
JSONObject response = executeQuery(query);
127+
verifySchema(response, schema("Count", null, "bigint"), schema("eventName", null, "string"));
128+
}
129+
130+
@Test
131+
public void testTop10Services() throws IOException {
132+
String query =
133+
String.format(
134+
"source=%s | stats count() as Count by `eventSource` | sort - Count | head 10",
135+
CLOUDTRAIL_LOGS_INDEX);
136+
137+
JSONObject response = executeQuery(query);
138+
verifySchema(response, schema("Count", null, "bigint"), schema("eventSource", null, "string"));
139+
verifyDataRows(
140+
response,
141+
rows(15, "ec2.amazonaws.com"),
142+
rows(14, "s3.amazonaws.com"),
143+
rows(13, "rds.amazonaws.com"),
144+
rows(10, "dynamodb.amazonaws.com"),
145+
rows(9, "cloudwatch.amazonaws.com"),
146+
rows(8, "sts.amazonaws.com"),
147+
rows(8, "lambda.amazonaws.com"),
148+
rows(8, "iam.amazonaws.com"),
149+
rows(8, "cloudformation.amazonaws.com"),
150+
rows(7, "logs.amazonaws.com"));
151+
}
152+
153+
@Test
154+
public void testTop10SourceIPs() throws IOException {
155+
String query =
156+
String.format(
157+
"source=%s | WHERE NOT (sourceIPAddress LIKE '%%amazon%%.com%%') | STATS count() as"
158+
+ " Count by sourceIPAddress| SORT - Count| HEAD 10",
159+
CLOUDTRAIL_LOGS_INDEX);
160+
161+
JSONObject response = executeQuery(query);
162+
verifySchema(
163+
response, schema("Count", null, "bigint"), schema("sourceIPAddress", null, "string"));
164+
}
165+
166+
@Test
167+
public void testTop10UsersGeneratingEvents() throws IOException {
168+
String query =
169+
String.format(
170+
"source=%s | where ISNOTNULL(`userIdentity.accountId`)| STATS count() as Count by"
171+
+ " `userIdentity.sessionContext.sessionIssuer.userName`, `userIdentity.accountId`,"
172+
+ " `userIdentity.sessionContext.sessionIssuer.type` | rename"
173+
+ " `userIdentity.sessionContext.sessionIssuer.userName` as `User Name`,"
174+
+ " `userIdentity.accountId` as `Account Id`,"
175+
+ " `userIdentity.sessionContext.sessionIssuer.type` as `Type` | SORT - Count |"
176+
+ " HEAD 1000",
177+
CLOUDTRAIL_LOGS_INDEX);
178+
179+
JSONObject response = executeQuery(query);
180+
verifySchema(
181+
response,
182+
schema("Count", null, "bigint"),
183+
schema("User Name", null, "string"),
184+
schema("Account Id", null, "string"),
185+
schema("Type", null, "string"));
186+
}
187+
188+
@Test
189+
public void testEC2ChangeEventCount() throws IOException {
190+
String query =
191+
String.format(
192+
"source=%s | where eventSource like \\\"ec2%%\\\" and (eventName = \\\"RunInstances\\\""
193+
+ " or eventName = \\\"TerminateInstances\\\" or eventName = \\\"StopInstances\\\")"
194+
+ " and not (eventName like \\\"Get%%\\\" or eventName like \\\"Describe%%\\\" or"
195+
+ " eventName like \\\"List%%\\\" or eventName like \\\"Head%%\\\") | stats count()"
196+
+ " as Count by eventName | sort - Count | head 5",
197+
CLOUDTRAIL_LOGS_INDEX);
198+
199+
JSONObject response = executeQuery(query);
200+
verifySchema(response, schema("Count", null, "bigint"), schema("eventName", null, "string"));
201+
verifyDataRows(response, rows(1, "TerminateInstances"), rows(1, "RunInstances"));
202+
}
203+
204+
@Test
205+
public void testEC2UsersBySessionIssuer() throws IOException {
206+
String query =
207+
String.format(
208+
"source=%s | where isnotnull(`userIdentity.sessionContext.sessionIssuer.userName`) and"
209+
+ " `eventSource` like 'ec2%%' and not (`eventName` like 'Get%%' or `eventName`"
210+
+ " like 'Describe%%' or `eventName` like 'List%%' or `eventName` like 'Head%%') |"
211+
+ " stats count() as Count by `userIdentity.sessionContext.sessionIssuer.userName`"
212+
+ " | sort - Count | head 10",
213+
CLOUDTRAIL_LOGS_INDEX);
214+
215+
JSONObject response = executeQuery(query);
216+
verifySchema(
217+
response,
218+
schema("Count", null, "bigint"),
219+
schema("userIdentity.sessionContext.sessionIssuer.userName", null, "string"));
220+
verifyDataRows(
221+
response, rows(1, "Analyst"), rows(1, "DataEngineer"), rows(1, "ec2-service"), rows(1, ""));
222+
}
223+
224+
@Test
225+
public void testEC2EventsByName() throws IOException {
226+
String query =
227+
String.format(
228+
"source=%s | where `eventSource` like \\\"ec2%%\\\" and not (`eventName` like"
229+
+ " \\\"Get%%\\\" or `eventName` like \\\"Describe%%\\\" or `eventName` like"
230+
+ " \\\"List%%\\\" or `eventName` like \\\"Head%%\\\") | stats count() as Count by"
231+
+ " `eventName` | rename `eventName` as `Event Name` | sort - Count | head 10",
232+
CLOUDTRAIL_LOGS_INDEX);
233+
234+
JSONObject response = executeQuery(query);
235+
verifySchema(response, schema("Count", null, "bigint"), schema("Event Name", null, "string"));
236+
verifyDataRows(
237+
response,
238+
rows(2, "CreateSecurityGroup"),
239+
rows(1, "RunInstances"),
240+
rows(1, "TerminateInstances"));
241+
}
242+
}

0 commit comments

Comments
 (0)