Skip to content

Commit cb578db

Browse files
aitkarraaitkarra
committed
feat: add endpoint raw-responses
1 parent 10b45fb commit cb578db

File tree

11 files changed

+163
-33
lines changed

11 files changed

+163
-33
lines changed

‎src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import fr.insee.genesis.domain.model.surveyunit.Mode;
1313
import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
1414
import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
15+
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse;
1516
import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
1617
import fr.insee.genesis.domain.ports.api.RawResponseApiPort;
1718
import fr.insee.genesis.exceptions.GenesisError;
@@ -270,7 +271,7 @@ public ResponseEntity<Map<String, List<String>>> getProcessedDataIdsSinceHours(
270271
@Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by start and end dates")
271272
@GetMapping(path = "/responses/raw/lunatic-json/{campaignId}")
272273
@PreAuthorize("hasRole('USER_BATCH_GENERIC')")
273-
public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getRawResponsesFromJsonBody(
274+
public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getLunaticJsonRawDataModelFromJsonBody(
274275
@PathVariable String campaignId,
275276
@RequestParam(value = "startDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant startDate,
276277
@RequestParam(value = "endDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant endDate,
@@ -284,5 +285,20 @@ public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getRawResponsesFromJs
284285
return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses));
285286
}
286287

287-
288+
@Operation(summary = "Get rawResponse JSON data from one campaign in Genesis Database, filtered by start and end dates")
289+
@GetMapping(path = "/raw-responses/{campaignId}")
290+
@PreAuthorize("hasRole('USER_BATCH_GENERIC')")
291+
public ResponseEntity<PagedModel<RawResponse>> getRawResponsesFromJsonBody(
292+
@PathVariable String campaignId,
293+
@RequestParam(value = "startDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant startDate,
294+
@RequestParam(value = "endDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant endDate,
295+
@RequestParam(value = "page", defaultValue = "0") int page,
296+
@RequestParam(value = "size", defaultValue = "1000") int size
297+
) {
298+
log.info("Try to read raw JSONs for campaign {}, with startDate={} and endDate={} - page={} - size={}", campaignId, startDate, endDate,page,size);
299+
Pageable pageable = PageRequest.of(page, size);
300+
Page<RawResponse> rawResponses = rawResponseApiPort.findRawResponseDataByCampaignIdAndDate(campaignId, startDate, endDate, pageable);
301+
log.info("rawResponses={}", rawResponses.getContent().size());
302+
return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses));
303+
}
288304
}

‎src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponse.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
package fr.insee.genesis.domain.model.surveyunit.rawdata;
22

33
import fr.insee.genesis.domain.model.surveyunit.Mode;
4+
import lombok.Builder;
45
import org.bson.types.ObjectId;
56

67
import java.time.LocalDateTime;
78
import java.util.Map;
89

10+
@Builder
911
public record RawResponse(
1012
ObjectId id,
1113
String interrogationId,

‎src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@
77
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse;
88
import fr.insee.genesis.exceptions.GenesisError;
99
import fr.insee.genesis.exceptions.GenesisException;
10+
import org.springframework.data.domain.Page;
11+
import org.springframework.data.domain.Pageable;
1012

13+
import java.time.Instant;
1114
import java.util.List;
1215

1316
public interface RawResponseApiPort {
@@ -18,5 +21,5 @@ public interface RawResponseApiPort {
1821
List<SurveyUnitModel> convertRawResponse(List<RawResponse> rawResponses, VariablesMap variablesMap);
1922
List<String> getUnprocessedCollectionInstrumentIds();
2023
void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
21-
24+
Page<RawResponse> findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable);
2225
}

‎src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@
22

33
import fr.insee.genesis.domain.model.surveyunit.Mode;
44
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse;
5+
import org.springframework.data.domain.Page;
6+
import org.springframework.data.domain.Pageable;
57

8+
import java.time.Instant;
69
import java.util.List;
710
import java.util.Set;
811

@@ -12,4 +15,5 @@ public interface RawResponsePersistencePort {
1215
void updateProcessDates(String collectionInstrumentId, Set<String> interrogationIds);
1316
List<String> getUnprocessedCollectionIds();
1417
Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
15-
}
18+
Page<RawResponse> findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable);
19+
}

‎src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,15 @@
2525
import fr.insee.genesis.infrastructure.utils.FileUtils;
2626
import lombok.extern.slf4j.Slf4j;
2727
import org.springframework.beans.factory.annotation.Qualifier;
28+
import org.springframework.data.domain.Page;
29+
import org.springframework.data.domain.Pageable;
2830
import org.springframework.http.ResponseEntity;
2931
import org.springframework.stereotype.Service;
3032

3133
import java.io.IOException;
34+
import java.time.Instant;
3235
import java.time.LocalDateTime;
33-
import java.util.ArrayList;
34-
import java.util.HashMap;
35-
import java.util.HashSet;
36-
import java.util.List;
37-
import java.util.Map;
38-
import java.util.Set;
36+
import java.util.*;
3937
import java.util.stream.Collectors;
4038

4139
@Service
@@ -444,4 +442,9 @@ private static void convertToExternalVar(SurveyUnitModel dstSurveyUnitModel, Var
444442
}
445443
}
446444
}
445+
446+
@Override
447+
public Page<RawResponse> findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) {
448+
return rawResponsePersistencePort.findByCampaignIdAndDate(campaignId,startDate, endDate,pageable);
449+
}
447450
}

‎src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,16 @@
99
import fr.insee.genesis.infrastructure.repository.RawResponseRepository;
1010
import lombok.extern.slf4j.Slf4j;
1111
import org.springframework.beans.factory.annotation.Qualifier;
12+
import org.springframework.data.domain.Page;
13+
import org.springframework.data.domain.PageImpl;
14+
import org.springframework.data.domain.Pageable;
1215
import org.springframework.data.mongodb.core.MongoTemplate;
1316
import org.springframework.data.mongodb.core.query.Criteria;
1417
import org.springframework.data.mongodb.core.query.Query;
1518
import org.springframework.data.mongodb.core.query.Update;
1619
import org.springframework.stereotype.Service;
1720

21+
import java.time.Instant;
1822
import java.time.LocalDateTime;
1923
import java.util.HashSet;
2024
import java.util.List;
@@ -58,4 +62,11 @@ public Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(Strin
5862
// We remove duplicate ids
5963
return new HashSet<>(repository.findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(collectionInstrumentId));
6064
}
65+
66+
@Override
67+
public Page<RawResponse> findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) {
68+
Page<RawResponseDocument> rawDataDocs = repository.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable);
69+
List<RawResponse> modelList = RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs.getContent());
70+
return new PageImpl<>(modelList, rawDataDocs.getPageable(), rawDataDocs.getTotalElements());
71+
}
6172
}

‎src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,7 @@ public record RawResponseDocument (
2121
Map<String,Object> payload,
2222
LocalDateTime recordDate,
2323
@Indexed(direction = IndexDirection.DESCENDING)
24-
LocalDateTime processDate
24+
LocalDateTime processDate,
25+
@Indexed
26+
String campaignId
2527
){}

‎src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
package fr.insee.genesis.infrastructure.repository;
22

33
import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument;
4+
import org.springframework.data.domain.Page;
5+
import org.springframework.data.domain.Pageable;
46
import org.springframework.data.mongodb.repository.Aggregation;
57
import org.springframework.data.mongodb.repository.MongoRepository;
68
import org.springframework.data.mongodb.repository.Query;
79
import org.springframework.stereotype.Repository;
810

11+
import java.time.Instant;
912
import java.util.List;
1013

1114
@Repository
@@ -26,4 +29,6 @@ public interface RawResponseRepository extends MongoRepository<RawResponseDocume
2629
})
2730
List<String> findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(String collectionInstrumentId);
2831

32+
@Query(value = "{ 'payload.campaignId' : ?0, 'recordDate' : { $gte: ?1, $lte: ?2 } }")
33+
Page<RawResponseDocument> findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable);
2934
}

‎src/test/java/cucumber/functional_tests/RawDataDefinitions.java‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,7 @@
2323
import fr.insee.genesis.exceptions.GenesisException;
2424
import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository;
2525
import fr.insee.genesis.infrastructure.utils.FileUtils;
26-
import fr.insee.genesis.stubs.ConfigStub;
27-
import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
28-
import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
29-
import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub;
30-
import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
31-
import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub;
26+
import fr.insee.genesis.stubs.*;
3227
import fr.insee.modelefiliere.RawResponseDto;
3328
import io.cucumber.java.Before;
3429
import io.cucumber.java.en.Given;
@@ -39,12 +34,9 @@
3934
import org.springframework.beans.factory.annotation.Autowired;
4035
import org.springframework.boot.test.web.client.TestRestTemplate;
4136
import org.springframework.boot.test.web.server.LocalServerPort;
42-
import org.springframework.http.HttpEntity;
43-
import org.springframework.http.HttpHeaders;
44-
import org.springframework.http.HttpMethod;
45-
import org.springframework.http.HttpStatus;
46-
import org.springframework.http.MediaType;
47-
import org.springframework.http.ResponseEntity;
37+
import org.springframework.data.domain.Page;
38+
import org.springframework.data.domain.Pageable;
39+
import org.springframework.http.*;
4840
import org.springframework.test.context.ContextConfiguration;
4941

5042
import java.io.IOException;
@@ -54,6 +46,7 @@
5446
import java.security.KeyPairGenerator;
5547
import java.security.NoSuchAlgorithmException;
5648
import java.security.interfaces.RSAPublicKey;
49+
import java.time.Instant;
5750
import java.util.Collections;
5851
import java.util.List;
5952

@@ -134,6 +127,11 @@ public List<String> getUnprocessedCollectionInstrumentIds() {
134127
public void updateProcessDates(List<SurveyUnitModel> surveyUnitModels) {
135128
// Do nothing - stub for test
136129
}
130+
131+
@Override
132+
public Page<RawResponse> findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) {
133+
return null;
134+
}
137135
};
138136

139137
RawResponseController rawResponseController = new RawResponseController(

‎src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java‎

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,17 @@
2222
import fr.insee.genesis.exceptions.GenesisError;
2323
import fr.insee.genesis.exceptions.GenesisException;
2424
import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument;
25+
import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument;
2526
import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper;
2627
import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository;
2728
import fr.insee.genesis.infrastructure.utils.FileUtils;
28-
import fr.insee.genesis.stubs.ConfigStub;
29-
import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
30-
import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
31-
import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub;
32-
import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
33-
import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub;
29+
import fr.insee.genesis.stubs.*;
3430
import fr.insee.modelefiliere.RawResponseDto;
3531
import lombok.extern.slf4j.Slf4j;
3632
import org.assertj.core.api.Assertions;
3733
import org.junit.jupiter.api.Test;
34+
import org.springframework.data.domain.Page;
35+
import org.springframework.data.domain.Pageable;
3836
import org.springframework.data.web.PagedModel;
3937
import org.springframework.http.ResponseEntity;
4038

@@ -49,6 +47,7 @@
4947
class RawResponseControllerTest {
5048
private final FileUtils fileUtils = new FileUtils(new ConfigStub());
5149
private final LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub();
50+
private final RawResponseDataPersistanceStub rawResponseDataPersistanceStub = new RawResponseDataPersistanceStub();
5251
private final SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub();
5352
private final SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub();
5453
private final DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub =
@@ -107,6 +106,10 @@ public List<String> getUnprocessedCollectionInstrumentIds() {
107106
public void updateProcessDates(List<SurveyUnitModel> surveyUnitModels) {
108107

109108
}
109+
@Override
110+
public Page<RawResponse> findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) {
111+
return rawResponseDataPersistanceStub.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable);
112+
}
110113
};
111114

112115
private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort, rawResponseApiPortStub, rawResponseInputRepositoryStub);
@@ -309,11 +312,10 @@ void processJsonRawDataV2Test(){
309312
.contains(interrogationId);
310313
}
311314

312-
313315
@Test
314-
void getRawResponsesFromJsonBody() {
316+
void getLunaticJsonRawDataModelFromJsonBody() {
315317
//GIVEN
316-
String campaignId = "getRawResponsesFromJsonBody";
318+
String campaignId = "getLunaticJsonRawDataModelFromJsonBody";
317319
String questionnaireId = campaignId + "_quest";
318320
String interrogationId = "getRawResponsesFromJsonBody_id1";
319321
String varName = "VARName1";
@@ -330,7 +332,29 @@ void getRawResponsesFromJsonBody() {
330332
int page=0, size= 10;
331333

332334
//WHEN
333-
ResponseEntity<PagedModel<LunaticJsonRawDataModel>> response = rawResponseController.getRawResponsesFromJsonBody(campaignId, starDate, endDate, page, size);
335+
ResponseEntity<PagedModel<LunaticJsonRawDataModel>> response = rawResponseController.getLunaticJsonRawDataModelFromJsonBody(campaignId, starDate, endDate, page, size);
336+
337+
//THEN
338+
Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
339+
org.junit.jupiter.api.Assertions.assertNotNull(response.getBody());
340+
Assertions.assertThat(response.getBody().getContent()).hasSize(1);
341+
}
342+
343+
@Test
344+
void getRawResponsesFromJsonBody() {
345+
//GIVEN
346+
String campaignId = "getRawResponsesFromJsonBody";
347+
String questionnaireId = campaignId + "_quest";
348+
String interrogationId = "getRawResponsesFromJsonBody_id1";
349+
Instant recordDate = Instant.parse("2025-01-01T01:00:00.000Z");
350+
351+
addJsonRawResponseDataDocumentToStub(campaignId, questionnaireId, interrogationId);
352+
353+
Instant starDate= recordDate.minusSeconds(86400),endDate = recordDate.plusSeconds(86400);
354+
int page=0, size= 10;
355+
356+
//WHEN
357+
ResponseEntity<PagedModel<RawResponse>> response = rawResponseController.getRawResponsesFromJsonBody(campaignId, starDate, endDate, page, size);
334358

335359
//THEN
336360
Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
@@ -380,4 +404,15 @@ private void addJsonRawDataDocumentToStub(String campaignId,
380404

381405
lunaticJsonRawDataPersistanceStub.getMongoStub().add(lunaticJsonDataDocument);
382406
}
407+
408+
private void addJsonRawResponseDataDocumentToStub(String campaignId, String questionnaireId, String interrogationId) {
409+
RawResponseDocument rawResponseDocument = RawResponseDocument.builder()
410+
.campaignId(campaignId)
411+
.collectionInstrumentId(questionnaireId)
412+
.interrogationId(interrogationId)
413+
.recordDate(LocalDateTime.now())
414+
.build();
415+
416+
rawResponseDataPersistanceStub.getMongoStub().add(rawResponseDocument);
417+
}
383418
}

0 commit comments

Comments
 (0)