Skip to content

Commit 55a9ee3

Browse files
nitin-ebinitin-ebi
authored
EVA-4079 validate call home json with schema before registering event (#36)
1 parent 4dd2666 commit 55a9ee3

File tree

10 files changed

+313
-40
lines changed

10 files changed

+313
-40
lines changed

‎pom.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,15 @@
9292
<artifactId>semver4j</artifactId>
9393
<version>3.1.0</version>
9494
</dependency>
95+
<dependency>
96+
<groupId>com.networknt</groupId>
97+
<artifactId>json-schema-validator</artifactId>
98+
<version>2.0.1</version>
99+
</dependency>
100+
<dependency>
101+
<groupId>com.fasterxml.jackson.core</groupId>
102+
<artifactId>jackson-databind</artifactId>
103+
</dependency>
95104
</dependencies>
96105
<dependencyManagement>
97106
<dependencies>

‎src/main/java/uk/ac/ebi/eva/submission/SubmissionApplication.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@
33
import org.springframework.boot.SpringApplication;
44
import org.springframework.boot.autoconfigure.SpringBootApplication;
55
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
6+
import org.springframework.cache.annotation.EnableCaching;
67
import org.springframework.retry.annotation.EnableRetry;
78
import org.springframework.scheduling.annotation.EnableScheduling;
89

910
@SpringBootApplication
1011
@EnableScheduling
12+
@EnableCaching
1113
@EnableRetry
1214
public class SubmissionApplication extends SpringBootServletInitializer {
1315

‎src/main/java/uk/ac/ebi/eva/submission/controller/callhome/CallHomeController.java‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package uk.ac.ebi.eva.submission.controller.callhome;
22

33
import com.fasterxml.jackson.databind.JsonNode;
4+
import org.slf4j.Logger;
5+
import org.slf4j.LoggerFactory;
6+
import org.springframework.http.HttpStatus;
47
import org.springframework.http.ResponseEntity;
58
import org.springframework.web.bind.annotation.PostMapping;
69
import org.springframework.web.bind.annotation.RequestBody;
@@ -11,15 +14,26 @@
1114
@RestController
1215
@RequestMapping("/v1/call-home")
1316
public class CallHomeController {
17+
private final Logger logger = LoggerFactory.getLogger(CallHomeController.class);
1418
private final CallHomeService callHomeService;
1519

1620
public CallHomeController(CallHomeService callHomeService) {
1721
this.callHomeService = callHomeService;
1822
}
1923

2024
@PostMapping("/events")
21-
public ResponseEntity<Void> ingest(@RequestBody JsonNode callHomeEventJson) {
22-
callHomeService.registerCallHomeEvent(callHomeEventJson);
23-
return ResponseEntity.ok().build();
25+
public ResponseEntity<?> ingest(@RequestBody JsonNode callHomeEventJson) {
26+
try {
27+
boolean valid = callHomeService.validateJson(callHomeEventJson);
28+
if (valid) {
29+
callHomeService.registerCallHomeEvent(callHomeEventJson);
30+
return ResponseEntity.ok().build();
31+
} else {
32+
return new ResponseEntity<>("Could not register event as the event json is invalid", HttpStatus.BAD_REQUEST);
33+
}
34+
} catch (Exception ex) {
35+
logger.error("Could not register event as an exception occurred: {}", ex.toString());
36+
return new ResponseEntity<>("Could not register event as an exception occurred", HttpStatus.INTERNAL_SERVER_ERROR);
37+
}
2438
}
2539
}

‎src/main/java/uk/ac/ebi/eva/submission/service/CallHomeService.java‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,56 @@
33
import com.fasterxml.jackson.databind.JsonNode;
44
import com.fasterxml.jackson.databind.ObjectMapper;
55
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
6+
import com.networknt.schema.InputFormat;
7+
import com.networknt.schema.Schema;
8+
import com.networknt.schema.SchemaRegistry;
9+
import com.networknt.schema.SpecificationVersion;
10+
import org.slf4j.Logger;
11+
import org.slf4j.LoggerFactory;
612
import org.springframework.stereotype.Service;
713
import uk.ac.ebi.eva.submission.entity.CallHomeEventEntity;
814
import uk.ac.ebi.eva.submission.repository.CallHomeEventRepository;
15+
import uk.ac.ebi.eva.submission.util.SchemaDownloader;
916

1017
import java.time.LocalDateTime;
18+
import java.util.List;
1119
import java.util.stream.Collectors;
1220
import java.util.stream.StreamSupport;
1321

1422
@Service
1523
public class CallHomeService {
24+
private final Logger logger = LoggerFactory.getLogger(CallHomeService.class);
25+
1626
private final CallHomeEventRepository callHomeEventRepository;
1727

18-
private static final ObjectMapper MAPPER = new ObjectMapper()
19-
.registerModule(new JavaTimeModule());
28+
private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
29+
private final SchemaRegistry schemaRegistry = SchemaRegistry.withDefaultDialect(SpecificationVersion.DRAFT_2020_12);
30+
private final SchemaDownloader schemaDownloader;
2031

21-
public CallHomeService(CallHomeEventRepository callHomeEventRepository) {
32+
public CallHomeService(CallHomeEventRepository callHomeEventRepository, SchemaDownloader schemaDownloader) {
2233
this.callHomeEventRepository = callHomeEventRepository;
34+
this.schemaDownloader = schemaDownloader;
2335
}
2436

37+
public boolean validateJson(JsonNode jsonPayload) {
38+
String latestTag = schemaDownloader.getLatestTag(SchemaDownloader.TAG_URL);
39+
String schemaURLWithLatestTag = SchemaDownloader.SCHEMA_URL.replace("{tag}", latestTag);
40+
String schemaContent = schemaDownloader.loadSchemaFromGitHub(schemaURLWithLatestTag);
41+
Schema schema = schemaRegistry.getSchema(schemaContent, InputFormat.JSON);
42+
List<com.networknt.schema.Error> errorList = schema.validate(jsonPayload.toString(), InputFormat.JSON,
43+
executionContext -> executionContext
44+
.executionConfig(config -> config.formatAssertionsEnabled(true))
45+
);
46+
47+
boolean schemaValidationPassed = errorList.isEmpty();
48+
if (!schemaValidationPassed) {
49+
logger.error("Schema validation failed: {}", errorList);
50+
}
51+
52+
return schemaValidationPassed;
53+
}
54+
55+
2556
public void registerCallHomeEvent(JsonNode callHomeEventJson) {
2657
CallHomeEventEntity callHomeEventEntity = getCallHomeEventEntity(callHomeEventJson);
2758
callHomeEventRepository.save(callHomeEventEntity);

‎src/main/java/uk/ac/ebi/eva/submission/util/SchemaDownloader.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package uk.ac.ebi.eva.submission.util;
2+
3+
import com.fasterxml.jackson.databind.JsonNode;
4+
import org.springframework.cache.annotation.CacheEvict;
5+
import org.springframework.cache.annotation.Cacheable;
6+
import org.springframework.retry.annotation.Backoff;
7+
import org.springframework.retry.annotation.Retryable;
8+
import org.springframework.scheduling.annotation.Scheduled;
9+
import org.springframework.stereotype.Component;
10+
import org.springframework.web.client.RestTemplate;
11+
12+
@Component
13+
public class SchemaDownloader {
14+
public static String TAG_URL = "https://api.github.com/repos/EBIvariation/eva-sub-cli/tags";
15+
public static String SCHEMA_URL = "https://raw.githubusercontent.com/EBIvariation/eva-sub-cli/{tag}/eva_sub_cli/etc/call_home_payload_schema.json";
16+
17+
private final RestTemplate restTemplate;
18+
19+
public SchemaDownloader(RestTemplate restTemplate) {
20+
this.restTemplate = restTemplate;
21+
}
22+
23+
@Cacheable(value = "latestTagCache", key = "#tagURL")
24+
@Retryable(value = Exception.class, maxAttempts = 5, backoff = @Backoff(delay = 2000, multiplier = 2))
25+
public String getLatestTag(String tagURL) {
26+
JsonNode tagJson = restTemplate.getForObject(tagURL, JsonNode.class);
27+
return tagJson.get(0).get("name").asText();
28+
}
29+
30+
@Cacheable(value = "schemaCache", key = "#schemaUrl")
31+
@Retryable(value = Exception.class, maxAttempts = 5, backoff = @Backoff(delay = 2000, multiplier = 2))
32+
public String loadSchemaFromGitHub(String schemaUrl) {
33+
return restTemplate.getForObject(schemaUrl, String.class);
34+
}
35+
36+
@CacheEvict(value = "latestTagCache", allEntries = true)
37+
@Scheduled(fixedRate = 12 * 60 * 60 * 1000)
38+
public void evictLatestTagCache() {}
39+
40+
@CacheEvict(value = "schemaCache", allEntries = true)
41+
@Scheduled(fixedRate = 48 * 60 * 60 * 1000)
42+
public void evictSchemaCache() {}
43+
44+
}

‎src/main/resources/application.properties‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,6 @@ lsri.token.url=|lsri.token-url|
3131
webin.userinfo.url=|webin.userinfo-url|
3232
server.servlet.context-path=/eva/webservices/submission-ws
3333

34-
management.metrics.binders.jvm.enabled=false
34+
management.metrics.binders.jvm.enabled=false
35+
36+
spring.cache.type=simple

‎src/test/java/uk/ac/ebi/eva/submission/integration/CallHomeWSIntegrationTest.java‎

Lines changed: 93 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
package uk.ac.ebi.eva.submission.integration;
22

3+
import com.fasterxml.jackson.databind.JsonNode;
34
import com.fasterxml.jackson.databind.ObjectMapper;
5+
import com.fasterxml.jackson.databind.node.ArrayNode;
46
import com.fasterxml.jackson.databind.node.ObjectNode;
7+
import org.junit.jupiter.api.BeforeAll;
8+
import org.junit.jupiter.api.BeforeEach;
59
import org.junit.jupiter.api.Test;
610
import org.springframework.beans.factory.annotation.Autowired;
711
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
@@ -12,21 +16,32 @@
1216
import org.springframework.test.context.DynamicPropertySource;
1317
import org.springframework.test.web.servlet.MockMvc;
1418
import org.springframework.transaction.annotation.Transactional;
19+
import org.springframework.web.client.RestTemplate;
1520
import org.testcontainers.containers.PostgreSQLContainer;
1621
import org.testcontainers.junit.jupiter.Container;
1722
import org.testcontainers.junit.jupiter.Testcontainers;
1823
import uk.ac.ebi.eva.submission.entity.CallHomeEventEntity;
1924
import uk.ac.ebi.eva.submission.repository.CallHomeEventRepository;
2025
import uk.ac.ebi.eva.submission.service.GlobusDirectoryProvisioner;
2126
import uk.ac.ebi.eva.submission.service.GlobusTokenRefreshService;
22-
23-
import java.time.LocalDateTime;
27+
import uk.ac.ebi.eva.submission.util.SchemaDownloader;
28+
29+
import java.io.BufferedReader;
30+
import java.io.IOException;
31+
import java.io.InputStream;
32+
import java.io.InputStreamReader;
33+
import java.net.URL;
34+
import java.nio.charset.StandardCharsets;
35+
import java.time.ZonedDateTime;
2436
import java.util.List;
2537
import java.util.stream.Collectors;
2638
import java.util.stream.StreamSupport;
2739

2840
import static org.assertj.core.api.Assertions.assertThat;
41+
import static org.mockito.ArgumentMatchers.eq;
42+
import static org.mockito.Mockito.when;
2943
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
44+
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
3045
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
3146

3247
@SpringBootTest
@@ -36,6 +51,9 @@ public class CallHomeWSIntegrationTest {
3651
@Autowired
3752
private CallHomeEventRepository callHomeEventRepository;
3853

54+
@Autowired
55+
private SchemaDownloader schemaDownloader;
56+
3957
@MockBean
4058
private GlobusTokenRefreshService globusTokenRefreshService;
4159

@@ -45,6 +63,35 @@ public class CallHomeWSIntegrationTest {
4563
@Autowired
4664
private MockMvc mvc;
4765

66+
@MockBean
67+
private static RestTemplate restTemplate;
68+
69+
private static String mockVersion = "v1.0.0";
70+
private static String mockSchemaUrl = SchemaDownloader.SCHEMA_URL.replace("{tag}", mockVersion);
71+
private static String schema = "";
72+
73+
@BeforeAll
74+
public static void downloadSchema() throws IOException {
75+
// get schema from main
76+
String schemaURL = "https://raw.githubusercontent.com/EBIvariation/eva-sub-cli/main/eva_sub_cli/etc/call_home_payload_schema.json";
77+
try (InputStream in = new URL(schemaURL).openStream();
78+
BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
79+
schema = reader.lines().collect(Collectors.joining("\n"));
80+
}
81+
}
82+
83+
@BeforeEach
84+
public void setup() throws IOException {
85+
schemaDownloader.evictSchemaCache();
86+
87+
// mock tag
88+
ObjectMapper mapper = new ObjectMapper();
89+
ArrayNode arrayNode = mapper.createArrayNode();
90+
ObjectNode tagNode = mapper.createObjectNode();
91+
tagNode.put("name", mockVersion);
92+
arrayNode.add(tagNode);
93+
when(restTemplate.getForObject(eq(SchemaDownloader.TAG_URL), eq(JsonNode.class))).thenReturn(arrayNode);
94+
}
4895

4996
@Container
5097
static PostgreSQLContainer<?> postgreSQLContainer = new PostgreSQLContainer<>("postgres:9.6")
@@ -67,6 +114,7 @@ static void dataSourceProperties(DynamicPropertyRegistry registry) {
67114
@Test
68115
@Transactional
69116
public void testRegisterCallHomeEvent() throws Exception {
117+
when(restTemplate.getForObject(eq(mockSchemaUrl), eq(String.class))).thenReturn(schema);
70118
ObjectMapper mapper = new ObjectMapper();
71119

72120
ObjectNode callHomeJsonRootNode = getCallHomeJson(mapper);
@@ -84,21 +132,26 @@ public void testRegisterCallHomeEvent() throws Exception {
84132
assertThat(callHomeEventEntityList.size()).isEqualTo(1);
85133

86134
CallHomeEventEntity callHomeEventEntity = callHomeEventEntityList.get(0);
87-
assertThat(callHomeEventEntity.getDeploymentId()).isEqualTo("test-deployment-id");
88-
assertThat(callHomeEventEntity.getRunId()).isEqualTo("test-run-id");
89-
assertThat(callHomeEventEntity.getEventType()).isEqualTo("test-event-type");
90-
assertThat(callHomeEventEntity.getCliVersion()).isEqualTo("test-cli-version");
91-
assertThat(callHomeEventEntity.getCreatedAt()).isEqualTo(LocalDateTime.parse("2020-01-01T00:00:00"));
92-
assertThat(callHomeEventEntity.getRuntimeSeconds()).isEqualTo(123);
93-
assertThat(callHomeEventEntity.getExecutor()).isEqualTo("Native");
94-
assertThat(callHomeEventEntity.getTasks()).isEqualTo("VALIDATION,SUBMIT");
135+
assertThat(callHomeEventEntity.getDeploymentId()).isEqualTo(callHomeJsonRootNode.get("deploymentId").asText());
136+
assertThat(callHomeEventEntity.getRunId()).isEqualTo(callHomeJsonRootNode.get("runId").asText());
137+
assertThat(callHomeEventEntity.getEventType()).isEqualTo(callHomeJsonRootNode.get("eventType").asText());
138+
assertThat(callHomeEventEntity.getCliVersion()).isEqualTo(callHomeJsonRootNode.get("cliVersion").asText());
139+
assertThat(callHomeEventEntity.getCreatedAt())
140+
.isEqualTo(ZonedDateTime.parse(callHomeJsonRootNode.get("createdAt").asText()).toLocalDateTime());
141+
assertThat(callHomeEventEntity.getRuntimeSeconds()).isEqualTo(callHomeJsonRootNode.get("runtimeSeconds").asInt());
142+
assertThat(callHomeEventEntity.getExecutor()).isEqualTo(callHomeJsonRootNode.get("executor").asText());
143+
assertThat(callHomeEventEntity.getTasks()).isEqualTo(StreamSupport
144+
.stream(callHomeJsonRootNode.get("tasks").spliterator(), false)
145+
.map(JsonNode::asText)
146+
.collect(Collectors.joining(",")));
95147

96148
assertThat(callHomeEventEntity.getRawPayload().toString()).isEqualTo(callHomeJsonRootNode.toString());
97149
}
98150

99151
@Test
100152
@Transactional
101-
public void testRegisterCallHomeEvent_someFieldsAreNull() throws Exception {
153+
public void testRegisterCallHomeEvent_BadRequestAsFieldIsMissing() throws Exception {
154+
when(restTemplate.getForObject(eq(mockSchemaUrl), eq(String.class))).thenReturn(schema);
102155
ObjectMapper mapper = new ObjectMapper();
103156

104157
ObjectNode callHomeJsonRootNode = getCallHomeJson(mapper);
@@ -107,40 +160,50 @@ public void testRegisterCallHomeEvent_someFieldsAreNull() throws Exception {
107160
mvc.perform(post("/v1/call-home/events")
108161
.content(mapper.writeValueAsString(callHomeJsonRootNode))
109162
.contentType(MediaType.APPLICATION_JSON))
110-
.andExpect(status().isOk());
163+
.andExpect(status().isBadRequest())
164+
.andExpect(content().string("Could not register event as the event json is invalid"));
111165

112166
Iterable<CallHomeEventEntity> iterable = callHomeEventRepository.findAll();
113167
List<CallHomeEventEntity> callHomeEventEntityList = StreamSupport
114168
.stream(iterable.spliterator(), false)
115169
.collect(Collectors.toList());
116170

117-
assertThat(callHomeEventEntityList.size()).isEqualTo(1);
171+
assertThat(callHomeEventEntityList.size()).isEqualTo(0);
172+
}
118173

119-
CallHomeEventEntity callHomeEventEntity = callHomeEventEntityList.get(0);
120-
assertThat(callHomeEventEntity.getDeploymentId()).isEqualTo("test-deployment-id");
121-
assertThat(callHomeEventEntity.getRunId()).isEqualTo("test-run-id");
122-
assertThat(callHomeEventEntity.getEventType()).isNull();
123-
assertThat(callHomeEventEntity.getCliVersion()).isEqualTo("test-cli-version");
124-
assertThat(callHomeEventEntity.getCreatedAt()).isEqualTo(LocalDateTime.parse("2020-01-01T00:00:00"));
125-
assertThat(callHomeEventEntity.getRuntimeSeconds()).isEqualTo(123);
126-
assertThat(callHomeEventEntity.getExecutor()).isEqualTo("Native");
127-
assertThat(callHomeEventEntity.getTasks()).isEqualTo("VALIDATION,SUBMIT");
174+
@Test
175+
@Transactional
176+
public void testRegisterCallHomeEvent_InternalServerError() throws Exception {
177+
when(restTemplate.getForObject(eq(mockSchemaUrl), eq(String.class))).thenReturn(null);
178+
ObjectMapper mapper = new ObjectMapper();
128179

129-
assertThat(callHomeEventEntity.getRawPayload().toString()).isEqualTo(callHomeJsonRootNode.toString());
180+
ObjectNode callHomeJsonRootNode = getCallHomeJson(mapper);
181+
182+
mvc.perform(post("/v1/call-home/events")
183+
.content(mapper.writeValueAsString(callHomeJsonRootNode))
184+
.contentType(MediaType.APPLICATION_JSON))
185+
.andExpect(status().isInternalServerError())
186+
.andExpect(content().string("Could not register event as an exception occurred"));
187+
188+
Iterable<CallHomeEventEntity> iterable = callHomeEventRepository.findAll();
189+
List<CallHomeEventEntity> callHomeEventEntityList = StreamSupport
190+
.stream(iterable.spliterator(), false)
191+
.collect(Collectors.toList());
192+
193+
assertThat(callHomeEventEntityList.size()).isEqualTo(0);
130194
}
131195

132196
private ObjectNode getCallHomeJson(ObjectMapper mapper) {
133197
ObjectNode rootNode = mapper.createObjectNode();
134-
rootNode.put("deploymentId", "test-deployment-id");
135-
rootNode.put("runId", "test-run-id");
136-
rootNode.put("eventType", "test-event-type");
198+
rootNode.put("deploymentId", "8f5bb4ea-9fc4-4117-91c6-9966d124e876");
199+
rootNode.put("runId", "8f5bb4ea-9fc4-4117-91c6-9966d124e876");
200+
rootNode.put("eventType", "VALIDATION_COMPLETED");
137201
rootNode.put("cliVersion", "test-cli-version");
138-
rootNode.put("createdAt", "2020-01-01T00:00:00");
202+
rootNode.put("createdAt", "2020-01-01T00:00:00Z");
139203
rootNode.put("runtimeSeconds", 123);
140-
rootNode.put("executor", "Native");
141-
rootNode.putArray("tasks").add("VALIDATION").add("SUBMIT");
204+
rootNode.put("executor", "native");
205+
rootNode.putArray("tasks").add("validate").add("submit");
142206

143207
return rootNode;
144208
}
145-
146209
}

‎src/test/java/uk/ac/ebi/eva/submission/unit/EnaUtilsAndEnaDownloaderTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package uk.ac.ebi.eva.submission.unit;
22

3-
import org.junit.jupiter.api.BeforeEach;
43
import org.junit.jupiter.api.Test;
54
import org.mockito.ArgumentMatchers;
65
import org.springframework.beans.factory.annotation.Autowired;

0 commit comments

Comments
 (0)